def update_vectors(self, sv: SentenceVectors, total_sentences: int): """Given existing sentence vectors, append new ones""" logger.info( f"appending sentence vectors for {total_sentences} sentences") sentences_before = len(sv.vectors) sentences_after = len(sv.vectors) + total_sentences if sv.mapfile_path: sv.vectors = np_memmap( str(sv.mapfile_path) + ".vectors", dtype=REAL, mode="r+", shape=(sentences_after, sv.vector_size), ) for i in range(sentences_before, sentences_after): sv.vectors[i] = full(shape=sv.vector_size, fill_value=EPS, dtype=REAL) else: newvectors = empty((total_sentences, sv.vector_size), dtype=REAL) for i in range(total_sentences): newvectors[i] = full(shape=sv.vector_size, fill_value=EPS, dtype=REAL) sv.vectors = vstack([sv.vectors, newvectors]) sv.vectors_norm = None
def reset_vectors(self, sv: SentenceVectors, total_sentences: int): """Initialize all sentence vectors to zero and overwrite existing files""" logger.info( f"initializing sentence vectors for {total_sentences} sentences") if sv.mapfile_path: sv.vectors = np_memmap(str(sv.mapfile_path) + '.vectors', dtype=REAL, mode='w+', shape=(total_sentences, sv.vector_size)) else: sv.vectors = empty((total_sentences, sv.vector_size), dtype=REAL) for i in range(total_sentences): sv.vectors[i] = zeros(sv.vector_size, dtype=REAL) sv.vectors_norm = None
def test_save_load(self): p = Path("fse/test/test_data/test_vectors.vectors") self.sv.save(str(p.absolute())) self.assertTrue(p.exists()) sv2 = SentenceVectors.load(str(p.absolute())) self.assertTrue((self.sv.vectors == sv2.vectors).all()) p.unlink()
def test_save_load_with_memmap(self): p = Path("fse/test/test_data/test_vectors") p_target = Path("fse/test/test_data/test_vectors.vectors") p_not_exists = Path("fse/test/test_data/test_vectors.vectors.npy") sv = SentenceVectors(2, mapfile_path=str(p)) shape = (1000, 1000) sv.vectors = np.ones(shape, dtype=np.float32) memvecs = np.memmap( p_target, dtype=np.float32, mode='w+', shape=shape) memvecs[:] = sv.vectors[:] del memvecs self.assertTrue(p_target.exists()) sv.save(str(p.absolute())) self.assertTrue(p.exists()) self.assertFalse(p_not_exists.exists()) sv = SentenceVectors.load(str(p.absolute())) self.assertEqual(shape, sv.vectors.shape) for t in [p, p_target]: t.unlink()
def setUp(self): self.sv = SentenceVectors(2) self.sv.vectors = np.arange(10).reshape(5, 2)
class TestSentenceVectorsFunctions(unittest.TestCase): def setUp(self): self.sv = SentenceVectors(2) self.sv.vectors = np.arange(10).reshape(5, 2) def test_getitem(self): self.assertTrue(([0, 1] == self.sv[0]).all()) self.assertTrue(([[0, 1], [4, 5]] == self.sv[[0, 2]]).all()) def test_isin(self): self.assertTrue(0 in self.sv) self.assertFalse(5 in self.sv) def test_init_sims_wo_replace(self): self.sv.init_sims() self.assertIsNotNone(self.sv.vectors_norm) self.assertFalse((self.sv.vectors == self.sv.vectors_norm).all()) v1 = self.sv.vectors[0] v1 = v1 / np.sqrt(np.sum(v1**2)) v2 = self.sv.vectors[1] v2 = v2 / np.sqrt(np.sum(v2**2)) self.assertTrue(np.allclose(v1, self.sv.vectors_norm[0])) self.assertTrue(np.allclose(v2, self.sv.vectors_norm[1])) self.assertTrue(np.allclose(v2, self.sv.get_vector(1, True))) def test_get_vector(self): self.assertTrue(([0, 1] == self.sv.get_vector(0)).all()) self.assertTrue(([2, 3] == self.sv.get_vector(1)).all()) def test_init_sims_w_replace(self): self.sv.init_sims(True) self.assertTrue( np.allclose(self.sv.vectors[0], self.sv.vectors_norm[0])) def test_init_sims_w_mapfile(self): p = Path("fse/test/test_data/test_vectors") self.sv.mapfile_path = str(p.absolute()) self.sv.init_sims() p = Path("fse/test/test_data/test_vectors.vectors_norm") self.assertTrue(p.exists()) p.unlink() def test_save_load(self): p = Path("fse/test/test_data/test_vectors.vectors") self.sv.save(str(p.absolute())) self.assertTrue(p.exists()) sv2 = SentenceVectors.load(str(p.absolute())) self.assertTrue((self.sv.vectors == sv2.vectors).all()) p.unlink() def test_save_load_with_memmap(self): p = Path("fse/test/test_data/test_vectors") p_target = Path("fse/test/test_data/test_vectors.vectors") p_not_exists = Path("fse/test/test_data/test_vectors.vectors.npy") sv = SentenceVectors(2, mapfile_path=str(p)) shape = (1000, 1000) sv.vectors = np.ones(shape, dtype=np.float32) memvecs = np.memmap(p_target, dtype=np.float32, mode='w+', shape=shape) memvecs[:] = sv.vectors[:] del memvecs self.assertTrue(p_target.exists()) sv.save(str(p.absolute())) self.assertTrue(p.exists()) self.assertFalse(p_not_exists.exists()) sv = SentenceVectors.load(str(p.absolute())) self.assertEqual(shape, sv.vectors.shape) for t in [p, p_target]: t.unlink() def test_len(self): self.assertEqual(5, len(self.sv)) def test_similarity(self): v1 = self.sv.vectors[0] v1 = v1 / np.sqrt(np.sum(v1**2)) v2 = self.sv.vectors[1] v2 = v2 / np.sqrt(np.sum(v2**2)) self.assertTrue(np.allclose(v1.dot(v2), self.sv.similarity(0, 1))) self.assertTrue(np.allclose(1 - v1.dot(v2), self.sv.distance(0, 1))) def test_most_similar(self): sent_ind = IndexedList(SENTENCES) sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.most_similar(positive=0) self.assertEqual(45, o[0][0]) self.assertEqual(35, o[1][0]) o = m.sv.most_similar(positive=0, indexable=sentences) self.assertEqual("Looks good and fits snug", o[0][0]) o = m.sv.most_similar(positive=0, indexable=sent_ind) self.assertEqual("Looks good and fits snug".split(), o[0][0][0]) def test_most_similar_vec(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) m.sv.init_sims() v = m.sv.get_vector(0, use_norm=True) o = m.sv.most_similar(positive=v) # Includes 0 obviously self.assertEqual(45, o[1][0]) self.assertEqual(35, o[2][0]) def test_most_similar_vecs(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) m.sv.init_sims() v = m.sv[[0, 1]] o = m.sv.most_similar(positive=v) self.assertEqual(1, o[0][0]) self.assertEqual(0, o[1][0]) def test_most_similar_wrong_indexable(self): def indexable(self): pass sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) with self.assertRaises(RuntimeError): m.sv.most_similar(positive=0, indexable=indexable) def test_most_similar_topn(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.most_similar(positive=0, topn=20) self.assertEqual(20, len(o)) def test_most_similar_restrict_size(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.most_similar(positive=20, topn=20, restrict_size=5) self.assertEqual(5, len(o)) def test_most_similar_restrict_size_tuple(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.most_similar(positive=20, topn=20, restrict_size=(5, 25)) self.assertEqual(19, len(o)) self.assertEqual(22, o[0][0]) o = m.sv.most_similar(positive=1, topn=20, restrict_size=(5, 25)) self.assertEqual(20, len(o)) self.assertEqual(9, o[0][0]) o = m.sv.most_similar(positive=1, topn=20, restrict_size=(5, 25), indexable=sentences) self.assertEqual(20, len(o)) self.assertEqual(9, o[0][1]) def test_similar_by_word(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.similar_by_word(word="the", wv=m.wv) self.assertEqual(96, o[0][0]) o = m.sv.similar_by_word(word="the", wv=m.wv, indexable=sentences) self.assertEqual(96, o[0][1]) def test_similar_by_vector(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.similar_by_vector(m.wv["the"]) self.assertEqual(96, o[0][0]) def test_similar_by_sentence(self): sentences = IndexedLineDocument(CORPUS) m = Average(W2V) m.train(sentences) o = m.sv.similar_by_sentence(sentence=["the", "product", "is", "good"], model=m) self.assertEqual(4, o[0][0]) def test_l2_norm(self): out = np.random.normal(size=(200, 50)).astype(np.float32) result = _l2_norm(out, False) lens = np.sqrt(np.sum((result**2), axis=-1)) self.assertTrue(np.allclose(1, lens, atol=1e-6)) out = np.random.normal(size=(200, 50)).astype(np.float32) out = _l2_norm(out, True) lens = np.sqrt(np.sum((out**2), axis=-1)) self.assertTrue(np.allclose(1, lens, atol=1e-6))
def __init__( self, model: BaseKeyedVectors, sv_mapfile_path: str = None, wv_mapfile_path: str = None, workers: int = 1, lang_freq: str = None, fast_version: int = 0, batch_words: int = 10000, batch_ngrams: int = 40, **kwargs, ): """ Base class for all Sentence2Vec Models. Provides core functionality, such as save, load, sanity checking, frequency induction, data checking, scanning, etc. Parameters ---------- model : :class:`~gensim.models.keyedvectors.BaseKeyedVectors` or :class:`~gensim.models.base_any2vec.BaseWordEmbeddingsModel` This object essentially contains the mapping between words and embeddings. To compute the sentence embeddings the wv.vocab and wv.vector elements are required. sv_mapfile_path : str, optional Optional path to store the sentence-vectors in for very large datasets. Used for memmap. wv_mapfile_path : str, optional Optional path to store the word-vectors in for very large datasets. Used for memmap. Use sv_mapfile_path and wv_mapfile_path to train disk-to-disk without needing much ram. workers : int, optional Number of working threads, used for multithreading. For most tasks (few words in a sentence) a value of 1 should be more than enough. lang_freq : str, optional Some pre-trained embeddings, i.e. "GoogleNews-vectors-negative300.bin", do not contain information about the frequency of a word. As the frequency is required for estimating the word weights, we induce frequencies into the wv.vocab.count based on :class:`~wordfreq` If no frequency information is available, you can choose the language to estimate the frequency. See https://github.com/LuminosoInsight/wordfreq fast_version : {-1, 1}, optional Whether or not the fast cython implementation of the internal training methods is available. 1 means it is. batch_words : int, optional Number of words to be processed by a single job. batch_ngrams : int, optional Number of maxium ngrams for oov words. **kwargs : object Key word arguments needed to allow children classes to accept more arguments. """ """ TODO: [ ] global: [ ] windows support [ ] documentation [ ] more benchmarks [ ] remove wv_mapfile_path? [ ] modifiable sv_mapfile_path? [ ] models: [ ] check feasibility first [ ] max-pooling -> easy [ ] hierarchical pooling -> easy [ ] discrete cosine transform -> somewhat easy, questionable [ ] valve -> unclear, not cited enough [ ] power-means embedding -> very large dimensionalty [ ] z-score transformation is quite nice [ ] sentencevectors: [X] similar_by_sentence model type check [ ] approximate NN search for large files [ ] compare ANN libraries [ ] ease-of-use [ ] dependencies [ ] compatibility [ ] memory-usage """ set_madvise_for_mmap() self.workers = int(workers) self.batch_words = batch_words self.batch_ngrams = batch_ngrams self.wv = None self.is_ft = False self.wv_mapfile_path = (Path(wv_mapfile_path) if wv_mapfile_path is not None else None) self.wv_mapfile_shapes = {} if fast_version < 0: warnings.warn( "C extension not loaded, training/inferring will be slow. " "Install a C compiler and reinstall fse.") self._check_and_include_model(model) if self.wv_mapfile_path is not None: self._map_all_vectors_to_disk(self.wv_mapfile_path) if lang_freq is not None: self._check_language_settings(lang_freq) self._induce_frequencies() self.sv = SentenceVectors(vector_size=self.wv.vector_size, mapfile_path=sv_mapfile_path) self.prep = BaseSentence2VecPreparer() self.word_weights = ones(len(self.wv.vocab), REAL)