def __init__(self, path_to_embeddings): if path_to_embeddings.endswith("fifu"): self._embeds = finalfusion.Embeddings(path_to_embeddings, mmap=True) elif path_to_embeddings.endswith("bin"): self._embeds = finalfusion.Embeddings.read_fasttext( path_to_embeddings, mmap=True) elif path_to_embeddings.endswith("w2v"): self._embeds = finalfusion.Embeddings.read_word2vec( path_to_embeddings, mmap=True) else: print("attempt to read invalid embeddings") self._embedding_dim = self._embeds.embedding( self._embeds.vocab()[0]).shape[0]
def analogy_fifu(tests_root): yield finalfusion.Embeddings(os.path.join(tests_root, "analogy.fifu"))
def subword_fifu(tests_root): yield finalfusion.Embeddings(os.path.join(tests_root, "subword.fifu"))
def similarity_pq_mmap(tests_root): yield finalfusion.Embeddings(os.path.join(tests_root, "similarity-pq.fifu"), mmap=True)
def similarity_fifu(tests_root): yield finalfusion.Embeddings(os.path.join(tests_root, "similarity.fifu"))
def embeddings_fifu(tests_root): yield finalfusion.Embeddings(os.path.join(tests_root, "embeddings.fifu"))