def __init__(self, path_to_embeddings):
if path_to_embeddings.endswith("fifu"):
self._embeds = finalfusion.Embeddings(path_to_embeddings,
mmap=True)
elif path_to_embeddings.endswith("bin"):
self._embeds = finalfusion.Embeddings.read_fasttext(
path_to_embeddings, mmap=True)
elif path_to_embeddings.endswith("w2v"):
self._embeds = finalfusion.Embeddings.read_word2vec(
path_to_embeddings, mmap=True)
else:
print("attempt to read invalid embeddings")
self._embedding_dim = self._embeds.embedding(
self._embeds.vocab()[0]).shape[0]
def analogy_fifu(tests_root):
yield finalfusion.Embeddings(os.path.join(tests_root, "analogy.fifu"))
def subword_fifu(tests_root):
yield finalfusion.Embeddings(os.path.join(tests_root, "subword.fifu"))
def similarity_pq_mmap(tests_root):
yield finalfusion.Embeddings(os.path.join(tests_root,
"similarity-pq.fifu"),
mmap=True)
def similarity_fifu(tests_root):
yield finalfusion.Embeddings(os.path.join(tests_root, "similarity.fifu"))
def embeddings_fifu(tests_root):
yield finalfusion.Embeddings(os.path.join(tests_root, "embeddings.fifu"))
