class KneserNeyInterpolatedTrigramTests(unittest.TestCase):
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = KneserNeyInterpolated(3, vocabulary=vocab)
self.model.fit(training_text)
score_tests = [
# For unigram scores revert to uniform
# Vocab size: 8
# count('c'): 1
("c", None, 1.0 / 8),
# in vocabulary but unseen, still uses uniform
("z", None, 1 / 8),
# out of vocabulary should use "UNK" score, i.e. again uniform
("y", None, 1.0 / 8),
# alpha = count('bc') - discount = 1 - 0.1 = 0.9
# gamma(['b']) = discount * number of unique words that follow ['b'] = 0.1 * 2
# normalizer = total number of bigrams with this context = 2
# the final should be: (alpha + gamma * unigram_score("c"))
("c", ["b"], (0.9 + 0.2 * (1 / 8)) / 2),
# building on that, let's try 'a b c' as the trigram
# alpha = count('abc') - discount = 1 - 0.1 = 0.9
# gamma(['a', 'b']) = 0.1 * 1
# normalizer = total number of trigrams with prefix "ab" = 1 => we can ignore it!
("c", ["a", "b"], 0.9 + 0.1 * ((0.9 + 0.2 * (1 / 8)) / 2)),
]
class KneserNeyInterpolatedTrigramTests(unittest.TestCase):
def setUp(self):
vocab, training_text = _prepare_test_data(3)
self.model = KneserNeyInterpolated(3, vocabulary=vocab)
self.model.fit(training_text)
score_tests = [
# For unigram scores revert to uniform
# Vocab size: 8
# count('c'): 1
("c", None, 1.0 / 8),
# in vocabulary but unseen, still uses uniform
("z", None, 1 / 8),
# out of vocabulary should use "UNK" score, i.e. again uniform
("y", None, 1.0 / 8),
# alpha = count('bc') - discount = 1 - 0.1 = 0.9
# gamma(['b']) = discount * number of unique words that follow ['b'] = 0.1 * 2
# normalizer = total number of bigrams with this context = 2
# the final should be: (alpha + gamma * unigram_score("c"))
("c", ["b"], (0.9 + 0.2 * (1 / 8)) / 2),
# building on that, let's try 'a b c' as the trigram
# alpha = count('abc') - discount = 1 - 0.1 = 0.9
# gamma(['a', 'b']) = 0.1 * 1
# normalizer = total number of trigrams with prefix "ab" = 1 => we can ignore it!
("c", ["a", "b"], 0.9 + 0.1 * ((0.9 + 0.2 * (1 / 8)) / 2)),
]
class NGramLM:
def __init__(self, n: int = 3):
self.n = n
self.model = KneserNeyInterpolated(n)
self.model_path = LM_PATH / f"{n}_gram.model"
def train(self, dataset: NGramDataset, save=True):
self.model.fit(dataset.train, dataset.vocab)
if save:
pickle.dump(self.model, self.model_path.open("wb"))
logging.info(f"ngram model saved at: {self.model_path}")
def load_model(self, path: Path = None):
if path:
if path.is_file():
self.model_path = path
else:
logging.exception(f"ngram model doesn't exits at: {path}")
self.model = pickle.load(self.model_path.open("rb"))
# TODO: set self.n from self.model
self.n = self.model.order
def evaluate_sentence(self, sentence: str):
"""check the prob. of `sentence`"""
tokens = NGramDataset.preprocess_sentence(sentence, self.n)
ngrams = list(get_ngrams(tokens, self.n))
total_logscore = 0
for ngram in ngrams:
cur_logscore = self.model.logscore(ngram[-1], ngram[:-1])
total_logscore += cur_logscore
return math.exp(total_logscore)
def predict_next(self):
pass
def build_ngram_lm(self, train):
if not train: return None
n = 5 # up to 5 gram language model
train, vocab = padded_everygram_pipeline(n, train)
model = KneserNeyInterpolated(n)
model.fit(train, vocab)
return model
class TestKneserNeyInterpolatedTrigram(metaclass=ParametrizedTests):
@classmethod
def setup_method(self):
vocab, training_text = _prepare_test_data(3)
self.model = KneserNeyInterpolated(3, discount=0.75, vocabulary=vocab)
self.model.fit(training_text)
score_tests = [
# P(c) = count('*c') / unique('**')
# = 1 / 14
("c", None, 1.0 / 14),
# P(z) = count('*z') / unique('**')
# = 0 / 14
# 'z' is in the vocabulary, but it was not seen during training.
("z", None, 0.0 / 14),
# P(y)
# Out of vocabulary should use "UNK" score.
# P(y) = P(UNK) = count('*UNK') / unique('**')
("y", None, 3 / 14),
# We start with P(c|b)
# P(c|b) = alpha('bc') + gamma('b') * P(c)
# alpha('bc') = max(unique('*bc') - discount, 0) / unique('*b*')
# = max(1 - 0.75, 0) / 2
# = 0.125
# gamma('b') = discount * unique('b*') / unique('*b*')
# = (0.75 * 2) / 2
# = 0.75
("c", ["b"], (0.125 + 0.75 * (1 / 14))),
# Building on that, let's try P(c|ab).
# P(c|ab) = alpha('abc') + gamma('ab') * P(c|b)
# alpha('abc') = max(count('abc') - discount, 0) / count('ab*')
# = max(1 - 0.75, 0) / 1
# = 0.25
# gamma('ab') = (discount * unique('ab*')) / count('ab*')
# = 0.75 * 1 / 1
("c", ["a", "b"], 0.25 + 0.75 * (0.125 + 0.75 * (1 / 14))),
# P(c|zb)
# The ngram 'zbc' was not seen, so we use P(c|b). See issue #2332.
("c", ["z", "b"], (0.125 + 0.75 * (1 / 14))),
]
class TestKneserNeyInterpolatedTrigram(metaclass=ParametrizedTests):
@classmethod
def setup_method(self):
vocab, training_text = _prepare_test_data(3)
self.model = KneserNeyInterpolated(3, vocabulary=vocab)
self.model.fit(training_text)
score_tests = [
# # of bigrams ending with c = 1
# total # of unique bigrams = 14
("c", None, 1.0 / 14),
# in vocabulary but unseen
# # of bigrams ending with z = 0
("z", None, 0.0 / 14),
# out of vocabulary should use "UNK" score
# # of bigrams ending with <UNK> = 3
("y", None, 3 / 14),
# alpha = max(count('bc') - discount,0)/# of bigrams starting 'b'
# = (1 - 0.75)/2 = 0.125
# gamma(['b']) = (discount * number of unique continuations after ['b'])/ # of bigrams starting 'b'
# = (0.75 * 2)/2 = 0.75
# the final should be: (alpha + gamma * unigram_score("c"))
("c", ["b"], (0.125 + 0.75 * (1 / 14))),
# building on that, let's try 'a b c' as the trigram
# alpha = max(count('abc') - discount,0)/# of trigrams starting "ab"
# = max(1 - 0.1, 0)/1 = 0.25
# gamma(['a', 'b']) = (discount * number of unique continuations after ['ab'])/ # of bigrams starting 'ab'
# = 0.75 * 1/1
# final: alpha + gamma*(P(c|b))
# alpha of P(c|b) = max(# of trigrams ending in "bc" - discount,0)/# unique trigram continuations with 'b' in the middle
# = (1-0.75)/2 =0.125
# gamma of P(c|b) = (discount * # of unique continuations after 'b')/ # of unique bigram continuations with 'b' in the middle
# = 0.75 * 2/2
("c", ["a", "b"], 0.25 + 0.75 * (0.125 + 0.75 * (1 / 14))),
# The ngram 'z b c' was not seen, so we should simply revert to
# the score of the ngram 'b c'. See issue #2332.
("c", ["z", "b"], (0.125 + 0.75 * (1 / 14))),
]
def trigram_model(tokenized_text, test_sentences, sentence_count):
n = 3
average_perplexity = 0.0
train_data, padded_vocab = padded_everygram_pipeline(n, tokenized_text)
model = KneserNeyInterpolated(n)
model.fit(train_data, padded_vocab)
tokenized_text = [
list(map(str.lower, nltk.tokenize.word_tokenize(sent)))
for sent in test_sentences
]
test_data, _ = padded_everygram_pipeline(n, tokenized_text)
for test in list(test_data):
ngrams = list(test)
if model.perplexity(ngrams) != float('inf'):
average_perplexity += model.perplexity(ngrams)
average_perplexity /= sentence_count
print(
f"Average Perplexity for Trigram model on Test tweets: {round(average_perplexity, 4)}"
)
temp = word_tokenize(sent)
for idx, word in enumerate(temp):
if word not in vnword:
temp[idx] = 'unknown'
result.append(temp)
print('tokenize done')
return result
if __name__ == '__main__':
arg = get_arg()
# get train data and tokenize
with open(arg.doc_dir, 'r', encoding='utf-8') as fin:
doc = fin.readlines()
corpus = tokenize(doc)
del doc
vi_model = KneserNeyInterpolated(arg.ngram)
train_data, padded_sent = padded_everygram_pipeline(arg.ngram, corpus)
del corpus
start_time = time.time()
vi_model.fit(train_data, padded_sent)
print('train %s-gram model in %d s' %
(arg.ngram, time.time() - start_time))
print('length of vocab = %s' % (len(vi_model.vocab)))
with open(arg.model_dir, 'wb') as fout:
pickle.dump(vi_model, fout)
print('save model successfully!')
class SynonymParaphraser:
def __init__(self, model=None, ngram=3):
if True:
stanza.download('sv') # download Swedish model
self.nlp = stanza.Pipeline(
'sv') # initialize Swedish neural pipeline
self.base_url = 'https://www.synonymer.se/sv-syn/'
# Build Language Model from corpus
if model is None:
with open('kneyser_lm.pkl', 'rb') as f:
self.model = pickle.load(f)
else:
self.model = KneserNeyInterpolated(ngram)
sentences = np.loadtxt(corpus_file, dtype='U', delimiter='\n')
text = [
list(map(str.lower, word_tokenize(sent))) for sent in sentences
]
train_data, padded_sents = padded_everygram_pipeline(ngram, text)
self.model.fit(train_data, padded_sents)
def generate_paraphrases(self, source_file):
# Read data and make a copy to store edited paraphrases
source_data = pd.read_csv(source_file)['question1']
paraphrases = source_data.copy()
for i in range(1688, len(source_data)):
# Clean source sentences and generate dependency parse treee
source_data[i] = clean_str(source_data[i])
doc = self.nlp(source_data[i])
print(doc)
# Iterate all words to find potential words to replace with synonyms
candidate_words = []
for word in doc.sentences[0].words:
if word.upos in ["ADJ", "ADV", "NOUN", "VERB"] and word.feats:
candidate_word = {
'word': word.text,
'index': word.id - 1,
'POS': word.upos
}
valid_candidate = True
features = [
feature.split('=') for feature in word.feats.split('|')
]
for feature in features:
if feature[0] == 'VerbForm' and feature[1] == 'Part':
valid_candidate = False
break
candidate_word[feature[0]] = feature[1]
if valid_candidate:
candidate_words.append(candidate_word)
replacements = 0
best_candidate = {'word': '', 'index': 0, 'diff': -np.inf}
for j, candidate in enumerate(candidate_words):
candidate_synonyms = self.get_synonyms(candidate['word'])
if candidate_synonyms == None:
continue
original = (candidate['word'],
self.get_score(candidate['word'],
candidate['index'], source_data[i]))
best_synonym = original
synonym_count = 0
for synonym in candidate_synonyms:
synonym = self.get_inflection(candidate, synonym)
if synonym is None:
continue
synonym_count += 1
# Calculate score for the synonym and compare to the current best
score = self.get_score(synonym, candidate['index'],
source_data[i])
if score > best_synonym[1]:
best_synonym = (synonym, score)
diff = score - original[1]
if best_candidate['diff'] < diff:
best_candidate['word'] = synonym
best_candidate['index'] = candidate['index']
best_candidate['diff'] = diff
print(
f'New best candidate: {synonym} with score {diff}')
# Build paraphrase sentence
if best_synonym[0] != candidate['word']:
new_sentence = ''
for (k, w) in enumerate(source_data[i].split()):
if k == candidate['index'] and best_synonym[0] != w:
new_sentence += best_synonym[0]
replacements += 1
print(f'Replaced word {w} with {best_synonym[0]}')
else:
new_sentence += w
if k < len(doc.sentences[0].words) - 1:
new_sentence += ' '
source_data[i] = new_sentence
# Assure at least one word is replaced with a synonym
if replacements == 0 and best_candidate['word'] != '':
print(best_candidate.items())
new_sentence = ''
for (k, w) in enumerate(source_data[i].split()):
if k == best_candidate['index']:
new_sentence += best_candidate['word']
else:
new_sentence += w
if k < len(doc.sentences[0].words) - 1:
new_sentence += ' '
source_data[i] = new_sentence
print(f'{i} sentences done')
print(source_data[i])
print(paraphrases[i])
print('\n')
with open('synonym_samples_final.txt', 'a') as f:
f.write(source_data[i] + '\n')
return source_data
def get_inflection(self, word, synonym):
pos = POS[word['POS']]
url = f"https://ws.spraakbanken.gu.se/ws/karp/v4/query?q=extended||and|pos|equals|{POS[word['POS']]}||and|wf|equals|{synonym}&resource=saldom"
response = requests.get(url).json()['hits']
if response['total'] == 0:
return None
msd = self.word_grammar(word)
for i in range(len(response['hits'])):
if response['hits'][i]['_source']['FormRepresentations'][0][
'baseform'] in synonym:
word_forms = response['hits'][i]['_source']['WordForms']
for j in range(len(word_forms)):
if word_forms[j]['msd'] == msd:
if word['POS'] == 'NOUN' and 'Gender' in word.keys():
inherent = 'n' if word['Gender'] == 'Neut' else 'u'
if inherent != response['hits'][i]['_source'][
'FormRepresentations'][0]['inherent']:
return None
return word_forms[j]['writtenForm']
def get_synonyms(self, word):
synonyms = set()
url = self.base_url + word
html_doc = requests.get(url).text
soup = BeautifulSoup(html_doc, 'html.parser')
soup = soup.find("div", {"id": "dict-default"})
if soup == None:
return None
else:
soup = soup.find("div", {"body"}).ul
for synset in soup.find_all('li'):
for syns in synset.find_all('ol', class_=lambda x: not x):
for synonym in syns.find_all('a'):
if len(synonym.text.split()) > 1:
continue
synonyms.add(synonym.text)
return synonyms
def get_score(self, word, j, source_sentence):
scores = []
sentence_len = len(source_sentence.split())
if sentence_len >= 3:
if j >= 2:
scores.append(
self.model.logscore(
word,
source_sentence.split()[(j - 2):(j - 1)]))
if j < sentence_len - 2:
scores.append(
self.model.logscore(
source_sentence.split()[j + 2],
[word, source_sentence.split()[j + 1]]))
if j >= 1 and j < sentence_len - 1:
scores.append(
self.model.logscore(
source_sentence.split()[j - 1],
[source_sentence.split()[j + 1], word]))
else:
if j == 0:
scores.append(
self.model.logscore(source_sentence.split()[1], [word]))
else:
scores.append(
self.model.logscore(word, [source_sentence.split()[0]]))
score = sum(scores) / len(scores)
return score
def word_grammar(self, word):
grammar = None
if word['POS'] == 'ADJ':
if 'Degree' not in word:
return None
if word['Degree'] == 'Pos':
grammar = 'pos'
elif word['Degree'] == 'Cmp':
grammar = 'komp'
if 'Case' in word.keys() and word['Case'] == 'Nom':
grammar = grammar + ' nom'
else:
grammar = grammar + ' gen'
return grammar
elif word['Degree'] == 'Sup':
grammar = 'super'
if 'Case' in word.keys() and word['Case'] == 'Nom':
grammar = grammar + ' nom'
else:
grammar = grammar + ' gen'
return grammar
if 'Definite' not in word:
return None
if word['Definite'] == 'Ind':
grammar = grammar + ' indef'
elif word['Definite'] == 'Def':
grammar = grammar + ' def'
if 'Number' in word.keys():
if word['Number'] == 'Sing':
grammar = grammar + ' sg'
elif word['Number'] == 'Plur':
grammar = grammar + ' pl'
if 'Gender' in word.keys() and word['Gender'] == 'Neut':
grammar = grammar + ' n nom'
else:
grammar = grammar + ' u nom'
elif word['POS'] == 'ADV':
if 'Degree' not in word:
return None
else:
if word['Degree'] == 'Pos':
grammar = 'pos'
elif word['Degree'] == 'Cmp':
grammar = 'komp'
elif word['Degree'] == 'Sup':
grammar = 'super'
elif word['POS'] == 'VERB':
if word['VerbForm'] == 'Inf':
grammar = 'inf'
elif word['VerbForm'] == 'Sup':
grammar = 'sup'
elif 'Tense' in word.keys() and word['Tense'] == 'Past':
grammar = 'pret ind'
elif word['Mood'] == 'Ind':
grammar = 'pres ind'
elif word['Mood'] == 'Imp':
grammar = 'imper'
return grammar
if 'Voice' in word.keys() and word['Voice'] == 'Act':
grammar = grammar + ' aktiv'
else:
grammar = grammar + ' s-form'
# if
elif word['POS'] == 'NOUN':
if 'Number' not in word.keys():
return None
if word['Number'] == 'Sing':
grammar = 'sg'
elif word['Number'] == 'Plur':
grammar = 'pl'
if 'Definite' not in word.keys():
return None
elif word['Definite'] == 'Ind':
grammar = grammar + ' indef'
elif word['Definite'] == 'Def':
grammar = grammar + ' def'
if word['Case'] == 'Gen':
grammar = grammar + ' gen'
else:
grammar = grammar + ' nom'
return grammar
def kneserney_bigram_model(bigram_training_data, vocabulary):
model = KneserNeyInterpolated(order=2, vocabulary=vocabulary)
model.fit(bigram_training_data)
return model
def kneserney_trigram_model(trigram_training_data, vocabulary):
model = KneserNeyInterpolated(order=3, discount=0.75, vocabulary=vocabulary)
model.fit(trigram_training_data)
return model
print("corpus read")
tokens = nltk.word_tokenize(raw)
sents = [nltk.word_tokenize(s) for s in nltk.sent_tokenize(raw)]
voc = Voc(corpus)
print(voc)
for s in sents:
for w in s:
voc.addWord(w)
print(voc)
sents = [[str(SOS_token)] + [str(voc.word2index[w]) for w in s] + [str(EOS_token)] for s in sents]
print(sents[0])
# train, vocab = my_padded_everygram_pipeline(N, sents, left_pad_symbol="SOS", right_pad_symbol="EOS")
train, vocab = my_padded_everygram_pipeline(N, sents, left_pad_symbol="0", right_pad_symbol="0")
print("preprocessing ready")
LM.fit(train, vocab)
print("LM ready")
# LM.generate()
# print("how")
# print(LM.score("How"))
# print("are you")
# print(LM.score("you", ["are"]))
out_file = f"token_KneserNey_{N}_lm.pkl"
with open(out_file, "wb") as f:
pickle.dump(LM, f)
print(LM.score("6"))
print(LM.score("4", ["8"]))
sentences_strings_ted.extend(sent for sent in re.split('[。？！]', m.groupdict()['postcolon']) if sent)
del input_text_noparens, input_text
sentences_strings_ted = [re.sub(r'[^\w\s]', '', sent) for sent in sentences_strings_ted]
sentences_strings_ted = [re.sub(r'[a-zA-Z0-9]', '', sent) for sent in sentences_strings_ted]
sentences_strings_ted = filter(None, sentences_strings_ted)
data = ' '.join([re.sub(r'\s', '', sent) for sent in sentences_strings_ted]).split(' ')
datax = [' '.join(sent).split(' ') for sent in data]
del sentences_strings_ted, data
# 训练 5-gram
lm = KneserNeyInterpolated(5)
train, vocab = padded_everygram_pipeline(5, datax)
lm.fit(train, vocab)
del train, vocab, datax
# 困惑度测试
test = '我想带你们体验一下，我们所要实现的“信任”的感觉。'
sent_list = re.sub(r'[^\w\s]', '', test)
sent_list = ','.join(sent_list).split(',')
text = list(ngrams(pad_both_ends(sent_list, 5), 5))
entropy = lm.entropy(text) # 交叉熵
perplexity = lm.perplexity(text) # 困惑度
print('交叉熵:%f' % entropy, '困惑度:%f' % perplexity)
# 储存模型 ... 以下内容 内存不足跑不起来 去 Colaboratory 或者 kaggle 跑蹭谷歌服务器
joblib.dump(lm, 'panti_gram.pkl')
# In[]
# 测试储存的模型