def test_persistence(self):
model = FT_gensim(sentences, min_count=1)
model.save(testfile())
self.models_equal(model, FT_gensim.load(testfile()))
# test persistence of the KeyedVectors of a model
wv = model.wv
wv.save(testfile())
loaded_wv = FastTextKeyedVectors.load(testfile())
self.assertTrue(np.allclose(wv.syn0_ngrams, loaded_wv.syn0_ngrams))
self.assertEqual(len(wv.vocab), len(loaded_wv.vocab))
self.assertEqual(len(wv.ngrams), len(loaded_wv.ngrams))
def test_persistence(self):
tmpf = get_tmpfile('gensim_fasttext.tst')
model = FT_gensim(sentences, min_count=1)
model.save(tmpf)
self.models_equal(model, FT_gensim.load(tmpf))
# test persistence of the KeyedVectors of a model
wv = model.wv
wv.save(tmpf)
loaded_wv = FastTextKeyedVectors.load(tmpf)
self.assertTrue(np.allclose(wv.syn0_ngrams, loaded_wv.syn0_ngrams))
self.assertEqual(len(wv.vocab), len(loaded_wv.vocab))
def test_norm_vectors_not_saved(self):
model = FT_gensim(sentences, min_count=1)
model.init_sims()
model.save(testfile())
loaded_model = FT_gensim.load(testfile())
self.assertTrue(loaded_model.wv.syn0norm is None)
self.assertTrue(loaded_model.wv.syn0_ngrams_norm is None)
wv = model.wv
wv.save(testfile())
loaded_kv = FastTextKeyedVectors.load(testfile())
self.assertTrue(loaded_kv.syn0norm is None)
self.assertTrue(loaded_kv.syn0_ngrams_norm is None)
def test_norm_vectors_not_saved(self):
model = FT_gensim(sentences, min_count=1)
model.init_sims()
model.save(testfile())
loaded_model = FT_gensim.load(testfile())
self.assertTrue(loaded_model.wv.syn0norm is None)
self.assertTrue(loaded_model.wv.syn0_ngrams_norm is None)
wv = model.wv
wv.save(testfile())
loaded_kv = FastTextKeyedVectors.load(testfile())
self.assertTrue(loaded_kv.syn0norm is None)
self.assertTrue(loaded_kv.syn0_ngrams_norm is None)
def test_norm_vectors_not_saved(self):
tmpf = get_tmpfile('gensim_fasttext.tst')
model = FT_gensim(sentences, min_count=1)
model.init_sims()
model.save(tmpf)
loaded_model = FT_gensim.load(tmpf)
self.assertTrue(loaded_model.wv.vectors_norm is None)
self.assertTrue(loaded_model.wv.vectors_ngrams_norm is None)
wv = model.wv
wv.save(tmpf)
loaded_kv = FastTextKeyedVectors.load(tmpf)
self.assertTrue(loaded_kv.vectors_norm is None)
self.assertTrue(loaded_kv.vectors_ngrams_norm is None)
def test_persistence_fromfile(self):
with temporary_file(get_tmpfile('gensim_fasttext1.tst')) as corpus_file:
utils.save_as_line_sentence(sentences, corpus_file)
tmpf = get_tmpfile('gensim_fasttext.tst')
model = FT_gensim(corpus_file=corpus_file, min_count=1)
model.save(tmpf)
self.models_equal(model, FT_gensim.load(tmpf))
# test persistence of the KeyedVectors of a model
wv = model.wv
wv.save(tmpf)
loaded_wv = FastTextKeyedVectors.load(tmpf)
self.assertTrue(np.allclose(wv.syn0_ngrams, loaded_wv.syn0_ngrams))
self.assertEqual(len(wv.vocab), len(loaded_wv.vocab))
from sklearn.feature_extraction.text import TfidfVectorizer
from gensim.models.wrappers.fasttext import FastTextKeyedVectors
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
import pickle as pkl
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
import datetime
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
# morph analyzer for text lemmatization
morph = pymorphy2.MorphAnalyzer()
fasttext = FastTextKeyedVectors.load('187/model.model')
pos_log_reg = pkl.load(open('pickles/pos_log_reg.pkl', 'rb'))
neg_log_reg = pkl.load(open('pickles/neg_log_reg.pkl', 'rb'))
pos_log_reg_dost = pkl.load(open('pickles/pos_log_reg_dost.pkl', 'rb'))
neg_log_reg_dost = pkl.load(open('pickles/neg_log_reg_dost.pkl', 'rb'))
old_data = pd.read_pickle('data/new_data.pkl')
old_data['index'] = old_data.index
training_data = pd.read_csv('data/training_data_with_razmetka_final.csv')
data_new = training_data.merge(old_data, on=['index', 'message'])
cut_date = lambda x: datetime.date(x.year, x.month, x.day)
data_new['local_datetime'] = pd.to_datetime(
data_new.local_datetime).apply(cut_date)