def test_get_vector_dimension():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
vect.load_all()
assert vect.get_vector_dim() == vect["."].shape[0]
assert vect.get_vector_dim() == 3
def test_basic_not_initialized():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
with pytest.raises(RuntimeError):
vect["."]
with pytest.raises(RuntimeError):
vect.token_to_vector(".")
def test_basic_max_vectors_bigger_than_num_lines():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path, max_vectors=20)
vect.load_all()
assert len(vect._vectors) == 4
contained_elements = [".", "'", ":", ","]
assert all(elem in vect._vectors for elem in contained_elements)
def test_basic_path_none_cache_doesnt_exist(tmpdir):
base = tmpdir
assert os.path.exists(base)
cache_path = os.path.join(base, "cache.t")
assert not os.path.exists(cache_path)
vect = vectorizer.WordVectors(path=None, cache_path=cache_path)
with pytest.raises(ValueError):
vect.load_all()
def test_basic_token_none():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path,
default_vector_function=None)
vect.load_all()
with pytest.raises(ValueError):
vect[None]
with pytest.raises(ValueError):
vect.token_to_vector(None)
def test_basic_max_vectors_vocab():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path, max_vectors=2)
vocab = [".", ":", ","]
vect.load_vocab(vocab)
assert len(vect._vectors) == 2
contained_elements = [".", ":"]
assert all(elem in vect._vectors for elem in contained_elements)
uncontained_elements = ["'", ","]
assert all(elem not in vect._vectors for elem in uncontained_elements)
def test_get_embedding_matrix(tokens, expected_matrix, expected_shape):
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
vect.load_all()
embedding_matrix = vect.get_embedding_matrix(vocab=tokens)
assert embedding_matrix.shape == expected_shape
assert np.allclose(a=embedding_matrix,
b=expected_matrix,
rtol=0,
atol=1e-6)
def test_basic_token_default():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(
path=vect_file_path,
default_vector_function=vectorizer.zeros_default_vector)
vect.load_all()
assert "a" not in vect._vectors
assert vect["a"].shape == (3, )
assert np.allclose(a=vect.token_to_vector("a"),
b=np.zeros(3),
rtol=0,
atol=1.0e-6)
def test_load_plain_text():
filename = "test.txt"
with tempfile.TemporaryDirectory() as tmpdir:
file_path = os.path.join(tmpdir, filename)
with open(file_path, mode="w") as file:
assert os.path.exists(file_path)
file.writelines(BASIC_VECT_DATA_PLAIN)
vec_storage = vectorizer.WordVectors(file_path, binary=False)
vec_storage.load_all()
assert len(vec_storage) == 4
for token, vec in BASIC_VECT_DATA_DICT.items():
assert np.all(vec == vec_storage[token])
def test_basic_load_all_vectors():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
vect.load_all()
assert len(vect._vectors) == 4
assert vect["."].shape == (3, )
assert vect.token_to_vector(",").shape == (3, )
assert np.allclose(a=vect["."],
b=BASIC_VECT_DATA_DICT["."],
rtol=0,
atol=1.0e-6)
assert np.allclose(a=vect.token_to_vector(","),
b=BASIC_VECT_DATA_DICT[","],
rtol=0,
atol=1.0e-6)
def test_basic_cache_vocab():
with tempfile.TemporaryDirectory() as base:
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA,
base_dir=base) as vect_file_path:
assert os.path.exists(vect_file_path)
cache_path = os.path.join(base, "cache.t")
assert not os.path.exists(cache_path)
vect = vectorizer.WordVectors(path=vect_file_path,
cache_path=cache_path)
vocab = [".", ":", ","]
vect.load_vocab(vocab)
assert os.path.exists(cache_path)
with open(cache_path, "rb") as cache_file:
content = cache_file.readlines()
assert len(content) == 3
def test_basic_cache_max_vectors(tmpdir):
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA,
base_dir=tmpdir) as vect_file_path:
assert os.path.exists(vect_file_path)
cache_path = os.path.join(tmpdir, "cache.t")
assert not os.path.exists(cache_path)
vect = vectorizer.WordVectors(path=vect_file_path,
max_vectors=2,
cache_path=cache_path)
vect.load_all()
assert os.path.exists(cache_path)
with open(cache_path, "rb") as cache_file:
content = cache_file.readlines()
assert len(content) == 2
first_line_parts = content[0].split(b" ")
word, values = first_line_parts[0], first_line_parts[1:]
assert word == b"."
assert len(values) == 3
def test_basic_load_vocab():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path,
default_vector_function=None)
vocab = [".", ":"]
vect.load_vocab(vocab=vocab)
assert len(vect._vectors) == 2
assert vect["."].shape == (3, )
assert vect.token_to_vector(":").shape == (3, )
assert np.allclose(a=vect[":"],
b=BASIC_VECT_DATA_DICT[":"],
rtol=0,
atol=1.0e-6)
assert np.allclose(a=vect.token_to_vector("."),
b=BASIC_VECT_DATA_DICT["."],
rtol=0,
atol=1.0e-6)
with pytest.raises(KeyError):
vect[","]
with pytest.raises(KeyError):
vect.token_to_vector(",")
def test_basic_diff_dimensions():
with create_temp_vect_file(vect_file_name="vect1",
file_data=DIFF_DIM_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
with pytest.raises(RuntimeError):
vect.load_all()
def test_basic_load_vocab_none():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
with pytest.raises(ValueError):
vect.load_vocab(vocab=None)
def test_get_vector_dim_not_initialized_vector_storage():
with create_temp_vect_file(vect_file_name="vect1",
file_data=BASIC_VECT_DATA) as vect_file_path:
vect = vectorizer.WordVectors(path=vect_file_path)
with pytest.raises(RuntimeError):
vect.get_vector_dim()
def test_basic_both_paths_none():
vect = vectorizer.WordVectors(path=None, cache_path=None)
with pytest.raises(ValueError):
vect.load_all()