def clean(text: str) -> str:
txt = text.strip()
txt = preprocessing.normalize_unicode(txt, form="NFKC")
# Collapse whitespaces
txt = preprocessing.normalize_whitespace(txt)
# Remove newlines
txt = preprocessing.normalize_repeating_chars(txt, chars="\n", maxn=1)
# fix hyphen-ated words
txt = preprocessing.normalize_hyphenated_words(txt)
txt = preprocessing.normalize_quotation_marks(txt)
txt = preprocessing.replace_urls(txt, replace_with="")
txt = preprocessing.replace_phone_numbers(txt, replace_with="")
txt = preprocessing.replace_emails(txt, replace_with="")
txt = preprocessing.replace_user_handles(txt, replace_with="")
txt = preprocessing.normalize_repeating_chars(txt, chars=".,;:-_ ", maxn=1)
txt = re.sub("\n ", " ", txt)
txt = re.sub(" \n", " ", txt)
txt = re.sub("\n", " ", txt)
txt = re.sub(" . ", " ", txt)
txt = re.sub(r"\.([A-Z])", r". \1", txt)
txt = re.sub(r"\. ([A-Z])", r".\n\1", txt)
# fix for some common abbreviations
for abv in ['Dr', 'St', 'Mr', 'Ms', 'mt', 'Inst', 'inc', 'est']:
txt = re.sub(abv + "\.\n", abv + ". ", txt)
return txt
def preprocess_siamese(doc):
# pre-process data
doc = tp.normalize.normalize_unicode(doc)
doc = tp.normalize_whitespace(doc)
doc = tp.normalize_quotation_marks(doc)
doc = tp.replace_emails(doc, replace_with="<EMAIL>")
doc = tp.replace_urls(doc, replace_with="<URL>")
doc = tp.replace_hashtags(doc, replace_with="<HASHTAG>")
doc = tp.replace_emojis(doc, replace_with="<EMOJI>")
doc = tp.replace_phone_numbers(doc, replace_with="<PHONE>")
# apply spaCy to tokenize doc
doc = nlp_token(doc)
# build new sentences for pre-processed doc
doc_new = []
for sent in doc.sents:
sent_new = ""
for token in sent:
token = token.text
token = token.replace("\n", "")
token = token.replace("\t", "")
token = token.strip()
sent_new += token + " "
doc_new.append(sent_new[:-1])
return doc_new
def test_normalize_quotation_marks():
in_outs = [
("These are ´funny single quotes´.",
"These are 'funny single quotes'."),
("These are ‘fancy single quotes’.",
"These are 'fancy single quotes'."),
("These are “fancy double quotes”.",
"These are \"fancy double quotes\"."),
]
for in_, out_ in in_outs:
assert preprocessing.normalize_quotation_marks(in_) == out_
def text_cleanup(text):
"cleanup our text"
text = preprocessing.replace_emails(text, replace_with='')
text = preprocessing.replace_urls(text, replace_with='')
text = preprocessing.replace_hashtags(text, replace_with='')
text = preprocessing.replace_phone_numbers(text, replace_with='')
text = preprocessing.replace_numbers(text, replace_with='')
text = preprocessing.remove_accents(text)
text = preprocessing.remove_punctuation(text)
text = preprocessing.normalize_quotation_marks(text)
text = preprocessing.normalize_hyphenated_words(text)
text = text.replace('\n', ' ').replace('\t', ' ')
text = text.lower()
text = preprocessing.normalize_whitespace(text)
return text
def textacy_preprocess(sentence):
"""Preprocess text."""
sentence = preprocessing.normalize_hyphenated_words(sentence)
sentence = preprocessing.normalize_quotation_marks(sentence)
#sentence = preprocessing.normalize_repeating_chars(sentence)
sentence = preprocessing.normalize_unicode(sentence)
sentence = preprocessing.normalize_whitespace(sentence)
sentence = preprocessing.remove_accents(sentence)
sentence = preprocessing.remove_punctuation(sentence)
sentence = preprocessing.replace_currency_symbols(sentence)
sentence = preprocessing.replace_emails(sentence)
sentence = preprocessing.replace_emojis(sentence)
sentence = preprocessing.replace_hashtags(sentence)
sentence = preprocessing.replace_numbers(sentence)
sentence = preprocessing.replace_phone_numbers(sentence)
sentence = preprocessing.replace_urls(sentence)
sentence = preprocessing.replace_user_handles(sentence)
return sentence
def preprocess_doc(self, doc):
# pre-process data
doc = tp.normalize.normalize_unicode(doc)
doc = tp.normalize_whitespace(doc)
doc = tp.normalize_quotation_marks(doc)
# apply spaCy to tokenize doc
doc = self.tokenizer(doc)
# build new sentences for pre-processed doc
doc_new = []
for sent in doc.sents:
sent_new = ''
for token in sent:
token = token.text
token = token.replace('\n', '')
token = token.replace('\t', '')
token = token.strip()
sent_new += token + ' '
doc_new.append(sent_new[:-1])
return doc_new
def _clean(self, text: str):
txt = text.strip()
#
txt = preprocessing.normalize_unicode(txt, form="NFKC")
# txt = preprocessing.remove_punctuation(txt)
# Collapse whitespaces
txt = preprocessing.normalize_whitespace(txt)
# Remove newlines
txt = preprocessing.normalize_repeating_chars(txt, chars="\n", maxn=1)
# fix hyphen-ated words
txt = preprocessing.normalize_hyphenated_words(txt)
txt = preprocessing.normalize_quotation_marks(txt)
txt = preprocessing.replace_urls(txt, replace_with="")
txt = preprocessing.replace_phone_numbers(txt, replace_with="")
txt = preprocessing.replace_emails(txt, replace_with="")
txt = preprocessing.replace_user_handles(txt, replace_with="")
txt = preprocessing.normalize_repeating_chars(txt,
chars=".,;:-_ ",
maxn=1)
txt = re.sub("\n ", " ", txt)
txt = re.sub(" \n", " ", txt)
txt = re.sub("\n", " ", txt)
txt = re.sub(" . ", " ", txt)
# txt = text.encode().decode("unicode-escape")
# Used ftfy for "fixing" broken text, e.g. Unicode
# txt = fix_text(txt.strip(), normalization="NFKC")
# re- minissence => reminissence
# txt = re.sub(r"([a-z])\-\s{,2}([a-z])", r"\1\2", txt)
# collapse two+ newlines into single whitespace
# txt = re.sub(r"\s+\n{1,}\s*(\w)", r" \1", txt)
# collapse two+ newlines into single whitespace
# txt = re.sub("\n+", " ", txt)
"""
# collapse two+ newlines into single whitespace
txt = re.sub(r"\s+\n{2,}\s*(\w)", r" \1", txt)
# double-newlines to dots
txt = re.sub(r"\n\n", ". ", txt)
# collapse whitespace
txt = re.sub(r"(\s){2,}", r"\1", txt)
# collapse dots
txt = re.sub(r"\.{2,}", ".", txt)
# newline to whitespace between word characters
txt = re.sub(r"(\w)\n(\w)", r"\1 \2", txt)
# newline + open brace to whitespace
txt = re.sub(r"(\w)\n(\()", r"\1 \2", txt)
# comma + newline to whitespace
txt = re.sub(r"(\w)\,\n(\w)", r"\1 \2", txt)
# Number end of sentence, followed by sentence that starts with number + dot
txt = re.sub(r"(\d+)\.(\d\.\s+)", r"\1. ", txt)
# remove decimals + dot after whitespace followed by whitespace
txt = re.sub(r"(\.\s*)\d+\.\s+", r"\1", txt)
# collapse backslashes
txt = re.sub(r"\\{2,}", r"\\", txt)
# remove 'escaped backslash' artefacts
txt = re.sub(r"\\\\", "", txt)
# remove lowdash artifacts ("lines")
txt = re.sub(r"_{2,}", r"", txt)
# normalize newline
txt = re.sub(r"\r\n", r"\n", txt)
# Linebreaks starting with numbers \n77\n
txt = re.sub(r"\n\d+\n", r"\n", txt)
# remove quotes + decimals on beginning of sentences
txt = re.sub(r"\.([\"']?)\d+\s+", r".\1", txt)
# remove quotes + decimals on beginning of sentences
txt = re.sub(r"\.([\"']?)\d+\s+", r".\1", txt)
# collapse dots
txt = re.sub(r"\.\s+\.", ". ", txt)
# collapse whitespace
txt = re.sub(r"(\w+)\s{2,}(\w+)", r"\1 \2", txt)
# Add space+ dot with double quotes
txt = re.sub(r"\.\"(\w+)", r'.". \1', txt)
# Add space+ between two sentences
txt = re.sub(r"([a-z])\.([A-Z])", r"\1. \2", txt)
"""
return txt
def preprocess_text(text,
char_count_filter=True,
stopwords=None,
min_len=2,
max_len=15):
"""
Pre-processing steps prior to spaCy nlp pipeline. Optional filtering of
tokens based on character length.
Parameters
----------
text : str
char_count_filter : bool
stopwords : iterable, None
min_len : int
max_len : int
Returns
-------
text : str
pre-processed text
"""
# 1) convert to lower case for robust stop-word recognition
text = text.lower()
# 2) normalise
text = preprocessing.normalize_quotation_marks(text)
# text = preprocessing.normalize_repeating_chars(text)
text = preprocessing.normalize_hyphenated_words(text)
text = preprocessing.normalize_whitespace(text)
# 3) replace
text = preprocessing.replace_currency_symbols(text)
text = preprocessing.replace_emails(text)
text = preprocessing.replace_emojis(text)
text = preprocessing.replace_hashtags(text)
text = preprocessing.replace_numbers(text)
text = preprocessing.replace_phone_numbers(text)
text = preprocessing.replace_urls(text)
text = preprocessing.replace_user_handles(text)
# 4) remove
text = preprocessing.remove_accents(text)
text = preprocessing.remove_punctuation(text)
text = re.sub("[^A-Za-z0-9]+", " ", text) # keep text and numbers
# 5) optionally remove tokens based on length
if char_count_filter & (stopwords is not None):
# filter based on token length
tokens = gensim.utils.simple_preprocess(doc=text,
min_len=min_len,
max_len=max_len)
# filter case-specific words
tokens = [token for token in tokens if token not in stopwords]
# convert processed list of tokens back to one string
text = " ".join(tokens)
else:
raise NotImplementedError("Not implemented.")
return text
def process_line(self, line: str) -> Optional[str]:
return normalize_quotation_marks(line)