def replace_numbers(self):
if self.numbers_idx > 0:
self._replace_numbers_re = _replace_numbers_re_selector[
self.numbers_idx]
self._replace_numbers_sub = _replace_numbers_sub_selector[
self.numbers_idx]
iterate_tokens(self.tokens, self.replace_numbers_token)
def remove_nonalnum(self):
if self.alnum_only:
iterate_tokens(self.tokens, self.remove_nonalnum_token)
def remove_nonascii(self):
if self.ascii_only:
iterate_tokens(self.tokens, self.remove_nonascii_token)
def process_lowercase(self):
if self.lowercase:
iterate_tokens(self.tokens, self.process_lowercase_token)
def split_numbers(self):
if self.numbers_split:
iterate_tokens(self.tokens, self.split_numbers_token)
def process_unicode_emoticons(self):
self._process_ascii_emotes_fct = _process_emotes_fct_selector[
self.ascii_emotes_idx]
if self._process_ascii_emotes_fct is not None:
iterate_tokens(self.tokens, self.process_unicode_emotes_token)
def process_urls(self):
self._process_urls_fct = _process_urls_fct_selector[self.urls_idx]
if self._process_urls_fct is not None:
iterate_tokens(self.tokens, self.process_urls_token)