def tokenize_message(message, tokenizer_type, spacer_annotate,
preserve_placeholders, spacer_new):
tokenizer = Tokenizer(tokenizer_type,
spacer_annotate=spacer_annotate,
preserve_placeholders=preserve_placeholders,
spacer_new=spacer_new)
return tokenizer.tokenize(message)[0]
def __init__(self, config_path):
self.config = json.loads(jsonnet_evaluate_file(config_path))
self.lang_detect_model_path = self.config["lang_detect_model_path"]
self.cat_detect_model_path = self.config["cat_detect_model_path"]
self.max_tokens = self.config.get("max_tokens")
self.is_lower = self.config["is_lower"]
self.languages = self.config.get("languages", ["ru", "en"])
self.is_news_only = self.config.get("is_news_only", False)
assert os.path.exists(
self.lang_detect_model_path), "No language detection model found"
assert os.path.exists(
self.cat_detect_model_path), "No category detection model found"
self.lang_detect_model = ft_load_model(self.lang_detect_model_path)
self.cat_detect_model = ft_load_model(self.cat_detect_model_path)
self.tokenizer = Tokenizer("conservative", joiner_annotate=False)
def __init__(self, messages, tokenizer_type):
self.messages = messages
Tokens.TOKENIZER = Tokenizer(tokenizer_type,
spacer_annotate=True,
preserve_placeholders=True,
spacer_new=True)
class DocumentsCleaner:
def __init__(self, config_path):
self.config = json.loads(jsonnet_evaluate_file(config_path))
self.lang_detect_model_path = self.config["lang_detect_model_path"]
self.cat_detect_model_path = self.config["cat_detect_model_path"]
self.max_tokens = self.config.get("max_tokens")
self.is_lower = self.config["is_lower"]
self.languages = self.config.get("languages", ["ru", "en"])
self.is_news_only = self.config.get("is_news_only", False)
assert os.path.exists(
self.lang_detect_model_path), "No language detection model found"
assert os.path.exists(
self.cat_detect_model_path), "No category detection model found"
self.lang_detect_model = ft_load_model(self.lang_detect_model_path)
self.cat_detect_model = ft_load_model(self.cat_detect_model_path)
self.tokenizer = Tokenizer("conservative", joiner_annotate=False)
def preprocess(self, text):
text = str(text).strip().replace("\n", " ").replace("\xa0", " ")
if self.is_lower:
text = text.lower()
tokens, _ = self.tokenizer.tokenize(text)
if self.max_tokens:
tokens = tokens[:self.max_tokens]
return " ".join(tokens)
def __call__(self, document) -> Optional[Document]:
title = document.get("title")
description = document.get("description", "")
text = document.get("text")
lang_text = text[:100]
lang_text_sample = " ".join((title, description, lang_text))
lang_text_sample = lang_text_sample.replace(" ",
" ").replace("\n", " ")
(lang_label, ), (lang_prob, ) = self.lang_detect_model.predict(
lang_text_sample, k=1)
lang_label = lang_label[FASTTEXT_LABEL_OFFSET:]
document["language"] = lang_label
if lang_label not in self.languages and (lang_label != "ru"
or lang_prob < 0.6):
return None
document["patched_title"] = self.preprocess(title)
document["patched_text"] = self.preprocess(text)
cat_text_sample = document["patched_title"] + " " + document[
"patched_text"]
cat_text_sample = self.preprocess(cat_text_sample)
(cat_label, ), (cat_prob, ) = self.cat_detect_model.predict(
cat_text_sample, k=1)
cat_label = cat_label[FASTTEXT_LABEL_OFFSET:]
document["category"] = cat_label
if self.is_news_only and cat_label == "not_news":
return None
return document
def __init__(
self,
model_paths={
"nn": "./models/classifier_3k_v3_4000.pth",
"svm": "./models/svm.joblib",
"knn": "./models/knn.joblib",
"dt": "./models/dt.joblib"
}):
self.tokenizer = Tokenizer('conservative')
self.smoothing = SmoothingFunction()
self.classifiers = {
key: self.load_sklearn_classifier(val)
for key, val in model_paths.items() if key != "nn"
}
self.classifiers["nn"] = self.load_nn_classifier(model_paths["nn"])
self.len_norm = self.load_sklearn_classifier(
"./models/len_norm.joblib")
self.src_norm = self.load_sklearn_classifier(
"./models/src_norm.joblib")
def process(self):
"""
The best tokenizer for error messages is TreebankWordTokenizer (nltk).
It's good at tokenizing file paths.
Alternative tokenizer. It performs much faster, but worse in tokenizing of paths.
It splits all paths by "/".
TODO: This method should be optimized to the same tokenization quality as TreebankWordTokenizer
:return:
"""
tokenized = []
if self.type == 'nltk':
for line in self.messages:
tokenized.append(TreebankWordTokenizer().tokenize(line))
elif self.tokenizer == 'pyonmttok':
tokenizer = Tokenizer("space", joiner_annotate=False, segment_numbers=False)
for line in self.messages:
tokens, features = tokenizer.tokenize(line)
tokenized.append(tokens)
self.tokenized = self.clean_tokens(tokenized)
return self.tokenized
def extend_file(file, size):
tokenizer = Tokenizer('conservative')
sentences = file.read().split('\n')
new_sentence = ""
new_set = []
while sentences:
tok_sent, _ = tokenizer.tokenize(new_sentence)
if len(tok_sent) < size:
sent = sentences[0]
sentences.remove(sent)
if new_sentence != "":
new_sentence += " "
new_sentence += sent
else:
new_set.append(new_sentence)
new_sentence = ""
if new_sentence != "":
new_set.append(new_sentence)
return '\n'.join(new_set)
def tokenize_list(list):
tokenizer = Tokenizer('conservative')
return [tokenizer.tokenize(line)[0] for line in list]
def __init__(self):
self.patterns = None
self.tokenizer = Tokenizer("conservative", spacer_annotate=True)
class Detector:
def __init__(
self,
model_paths={
"nn": "./models/classifier_3k_v3_4000.pth",
"svm": "./models/svm.joblib",
"knn": "./models/knn.joblib",
"dt": "./models/dt.joblib"
}):
self.tokenizer = Tokenizer('conservative')
self.smoothing = SmoothingFunction()
self.classifiers = {
key: self.load_sklearn_classifier(val)
for key, val in model_paths.items() if key != "nn"
}
self.classifiers["nn"] = self.load_nn_classifier(model_paths["nn"])
self.len_norm = self.load_sklearn_classifier(
"./models/len_norm.joblib")
self.src_norm = self.load_sklearn_classifier(
"./models/src_norm.joblib")
def load_sklearn_classifier(self, model_path):
try:
return load(model_path)
except FileNotFoundError:
return None
def load_nn_classifier(self, model_path):
classifier = Classifier()
classifier.load_state_dict(torch.load(model_path))
classifier.float()
return classifier
def predict(self, text, src, algorithm):
other_langs = [l for l in LANG_POOL if l != src]
results = {lang: [text] for lang in other_langs}
data = []
last_back = text
for lang in other_langs:
for _ in range(0, 2):
translation = translate(last_back, src, lang)
last_back = translate(translation, lang, src)
results[lang].append(last_back)
for lang in LANG_POOL:
if lang != src:
for i in range(0, 2):
ref_sent, _ = self.tokenizer.tokenize(results[lang][i])
hypothesis, _ = self.tokenizer.tokenize(results[lang][i +
1])
bleu = sentence_bleu(
[ref_sent],
hypothesis,
smoothing_function=self.smoothing.method4,
weights=(0.25, 0.25, 0.25, 0.25))
data.append(bleu)
else:
data += [0.0, 0.0]
data.append(
self.src_norm.transform(
np.array([LANG_POOL.index(src)]).reshape(-1, 1))[0][0])
data.append(
self.len_norm.transform(
np.array([len(self.tokenizer.tokenize(text)[0])
]).reshape(-1, 1))[0][0])
data = np.array(data).reshape(1, -1)
prediction = None
if algorithm != "nn":
prediction = self.classifiers[algorithm].predict_proba(data)
else:
tensor = torch.from_numpy(data)
prediction = self.classifiers["nn"](tensor.float()).tolist()
return list(prediction[0])
def __init__(self):
self.word_map = load_dakshina_map()
self.tokenizer = Tokenizer('aggressive')
class Transliterator:
def __init__(self):
self.word_map = load_dakshina_map()
self.tokenizer = Tokenizer('aggressive')
def process_line(self, line, rule_based_only=False):
processed_words = []
line = line.strip()
for word in line.split():
processed_tokens = []
for token in self.tokenizer.tokenize(word)[0]:
if BANGLA_CHARS.search(token):
current_transliteration = transliterate.process(
'Bengali',
'Custom',
token,
pre_options=[
'AnuChandraEqDeva', 'RemoveSchwaHindi',
'SchwaFinalBengali'
],
post_options=['RemoveDiacritics'])
if not rule_based_only:
transliteration_map = self.word_map.get(token, {})
if transliteration_map:
# approach 1: use only edit distance
# selected_transliteration = min(transliteration_map, key=lambda k: distance(current_transliteration, k))
# approach 2: use attestation scores with edit distance as tiebreaker
highest_scored_transliteration = max(
transliteration_map,
key=lambda k: int(transliteration_map[k]))
highest_score = transliteration_map[
highest_scored_transliteration]
candidate_transliterations = [
k for k, v in transliteration_map.items()
if v == highest_score
]
if len(candidate_transliterations) > 1:
selected_transliteration = min(
candidate_transliterations,
key=lambda k: distance(
current_transliteration, k))
else:
selected_transliteration = highest_scored_transliteration
processed_tokens.append(selected_transliteration)
else:
processed_tokens.append(current_transliteration)
else:
processed_tokens.append(current_transliteration)
else:
processed_tokens.append(token)
processed_words.append(''.join(processed_tokens))
return ' '.join(processed_words)
import string
from pyonmttok import Tokenizer
from pymorphy2 import MorphAnalyzer
tokenizer = Tokenizer("conservative", joiner_annotate=False)
morph = MorphAnalyzer()
def tokenize(text, lower=True):
text = str(text).strip().replace("\n", " ").replace("\xa0", " ")
if lower:
text = text.lower()
tokens, _ = tokenizer.tokenize(text)
return tokens
def tokenize_to_lemmas(text):
tokens = tokenize(text)
tokens = filter(lambda x: x not in string.punctuation, tokens)
tokens = filter(lambda x: not x.isnumeric(), tokens)
tokens = filter(lambda x: len(x) >= 2, tokens)
tokens = [morph.parse(token)[0].normal_form for token in tokens]
return tokens