/
oov.py
82 lines (65 loc) · 2.62 KB
/
oov.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import xxhash
import neologdn
import numpy as np
from simstring.searcher import Searcher
from simstring.database.dict import DictDatabase
from simstring.measure.cosine import CosineMeasure
from simstring.feature_extractor.character_ngram import CharacterNgramFeatureExtractor
def seed(s):
return xxhash.xxh32(s.encode("utf-8")).intdigest()
def ngram(words, n):
return ["".join(t) for t in list(zip(*(words[i:] for i in range(n))))]
def character_ngram(word, n_begin=3, n_end=5):
output = []
n = n_begin
while n <= n_end:
output += ngram(word, n)
n += 1
return output
class MagnitudeOOV():
def __init__(self, word2vec):
self.w2v = word2vec
self.embedding_dim = self.w2v.vector_size
self.vocab = set(self.w2v.vocab.keys())
self.db = DictDatabase(CharacterNgramFeatureExtractor(2))
for vocab_word in self.vocab:
self.db.add(vocab_word)
def generate_pseudorandom_vector(self, word):
"""calculate PRVG form CGRAM"""
vectors = []
ngram_list = character_ngram(word)
for ngram in ngram_list:
np.random.seed(seed(ngram))
vectors.append(np.random.uniform(-1, 1, self.embedding_dim))
return np.mean(vectors, axis=0)
def similar_words_top_k(self, query, measure=CosineMeasure(), initial_threshold=0.99, dec_step=0.01, k=3):
"""search similar words by using edit distance"""
searcher = Searcher(self.db, measure)
t = initial_threshold
similar_words = []
while True:
similar_words = searcher.search(query, t)
if len(similar_words) >= k or t <= 0.1:
break
t -= dec_step
if len(similar_words) > 3:
np.random.choice(42)
return np.random.choice(similar_words, k, replace=False).tolist()
else:
return similar_words
def generate_similar_words_vector(self, word):
"""calculate MATCH from similar words"""
vectors = np.mean([self.w2v[w] for w in self.similar_words_top_k(word)], axis=0)
return vectors
def out_of_vocab_vector(self, word):
vector = self.generate_pseudorandom_vector(word) * 0.3 + self.generate_similar_words_vector(word) * 0.7
final_vector = vector / np.linalg.norm(vector)
return final_vector
def query(self, word):
normalized_word = neologdn.normalize(word)
if word in self.vocab:
return self.w2v[word]
elif normalized_word in self.vocab:
return self.w2v[normalized_word]
else:
return self.out_of_vocab_vector(normalized_word)