def test_read_embeddings(tests_root):
ffp.load_finalfusion(os.path.join(tests_root, "data", "simple_vocab.fifu"))
e = ffp.load_finalfusion(
os.path.join(tests_root, "data", "ff_buckets.fifu"))
e2 = ffp.load_finalfusion(os.path.join(tests_root, "data",
"ff_buckets.fifu"),
mmap=True)
assert np.allclose(e.storage, e2.storage)
with pytest.raises(TypeError):
ffp.load_finalfusion(None)
with pytest.raises(ffp.io.FinalfusionFormatError):
ffp.load_finalfusion(1)
with pytest.raises(IOError):
ffp.load_finalfusion("foo")
def test_ff_embeddings_roundtrip_ff_buckets(bucket_vocab_embeddings_fifu):
tmp_dir = tempfile.gettempdir()
filename = os.path.join(tmp_dir, "write_embeddings.fifu")
bucket_vocab_embeddings_fifu.write(filename)
e2 = ffp.load_finalfusion(filename)
assert bucket_vocab_embeddings_fifu.vocab == e2.vocab
assert bucket_vocab_embeddings_fifu.metadata == e2.metadata
assert np.allclose(bucket_vocab_embeddings_fifu.storage, e2.storage)
assert np.allclose(bucket_vocab_embeddings_fifu.norms, e2.norms)
def test_ff_embeddings_roundtrip(embeddings_fifu, vocab_array_tuple):
tmp_dir = tempfile.gettempdir()
filename = os.path.join(tmp_dir, "write_embeddings.fifu")
v = embeddings_fifu.vocab
s = embeddings_fifu.storage
embeddings_fifu.write(filename)
assert v.words == vocab_array_tuple[0]
matrix = vocab_array_tuple[1]
matrix = matrix.squeeze() / np.linalg.norm(matrix, axis=1, keepdims=True)
assert np.allclose(matrix, s)
assert np.allclose(s, ffp.load_finalfusion(filename).storage)
def pq_check(tests_root):
yield ffp.load_finalfusion(os.path.join(tests_root, "data/pq-check.fifu"),
False)
def simple_vocab_embeddings_fifu(tests_root):
yield ffp.load_finalfusion(os.path.join(tests_root,
"data/simple_vocab.fifu"),
mmap=False)
def embeddings_pq_read(tests_root):
yield ffp.load_finalfusion(os.path.join(tests_root, "data/pq.fifu"), False)
def embeddings_pq_memmap(tests_root):
yield ffp.load_finalfusion(os.path.join(tests_root, "data/pq.fifu"), True)
def embeddings_fifu(tests_root):
yield ffp.load_finalfusion(os.path.join(tests_root,
"data/embeddings.fifu"))
def bucket_vocab_embeddings_fifu(tests_root):
yield ffp.load_finalfusion(os.path.join(tests_root,
"data/ff_buckets.fifu"))