def _normalize_sentence(cls, single_sentence):
""" Preprocess text and normalize for parsing network """
return [
tok.txt
for tok in bintokenizer.tokenize(single_sentence)
if BIN_Token.is_understood(tok)
]
def _normalize_text(cls, text):
""" Preprocess text and normalize for parsing network """
pgs = text.split("\n")
normalized_pgs = [[
tok.txt for tok in list(bintokenizer.tokenize(pg))
if BIN_Token.is_understood(tok)
] for pg in pgs]
return [" ".join(tok for tok in npg if tok) for npg in normalized_pgs]