def use_pyfasttext_model():
# OK
# 训练模型可以使用fasttext命令行工具进行(../doc/fastText_train.png)，也可以使用本文件使用的pyfasttext包训练。
"""
# OK: 1. pyfasttext包训练的模型的导入
model = FastText("../data/lxw_model_sg_pyfasttext.bin")
print(model["先生"]) # type(model["先生"]): <class 'array.array'>
print(model.get_numpy_vector("先生")) # type: <class 'numpy.ndarray'>
print(model["刘晓伟"]) # OOV
print(model.get_numpy_vector("刘晓伟"))
print(model["陈贺"]) # OOV
print(model.get_numpy_vector("陈贺"))
model = FastText("../data/lxw_model_cbow_pyfasttext.bin")
print(model["先生"])
print(model.get_numpy_vector("先生")) # type: <class 'numpy.ndarray'>
print(model["刘晓伟"]) # OOV
print(model.get_numpy_vector("刘晓伟"))
print(model["陈贺"]) # OOV
print(model.get_numpy_vector("陈贺"))
# NOTE: 简单的测试发现, 两个不同的模型针对同一个OOV计算得到的向量是一样的(与fasttext包的情况相同，详情可参见NO_2_use_fasttext_model), 非OOV的向量是不一样的。
"""
# OK: 2. fasttext命令行工具训练出来的模型的导入
model = FastText("../data/880w_fasttext_skip_gram.bin")
print(model["先生"]) # type(model["先生"]): <class 'array.array'>
print(model.get_numpy_vector("先生"))
# print(model["刘晓伟"]) # OK. OOV
# print(model["陈贺"]) # OK. OOV
# Sentence and text vectors.
sentence_vec = model.get_numpy_sentence_vector("刘晓伟 是 个 好人")
print(sentence_vec)
"""
class NB_Implement():
def __init__(self):
start_time = time.time()
# self.model = FastText("../data/input/models/sg_pyfasttext.bin") # DEBUG
self.model = FastText(
"../data/input/models/880w_fasttext_skip_gram.bin")
end_time = time.time()
print(f"Loading word vector model cost: {end_time - start_time:.2f}s")
# self.vocab_size, self.vector_size = self.model.numpy_normalized_vectors.shape # OK
self.vocab_size = self.model.nwords
self.vector_size = self.model.args.get("dim")
print(
f"self.vector_size:{self.vector_size}, self.vocab_size: {self.vocab_size}"
) # self.vector_size:200, self.vocab_size: 925242
# 句子的表示形式: {"avg": 向量和的平均, "fasttext": get_numpy_sentence_vector, "matrix": matrix}
self.sentence_vec_type = "avg"
def set_sent_vec_type(self, sentence_vec_type):
assert self.sentence_vec_type in [
"avg", "matrix", "fasttext"
], "self.sentence_vec_type must be in ['avg', 'fasttext', 'matrix']"
self.sentence_vec_type = sentence_vec_type
def gen_sentence_vec(self, sentence):
"""
:param sentence:
:return:
"""
sentence = sentence.strip()
if self.sentence_vec_type == "fasttext":
return self.model.get_numpy_sentence_vector(sentence)
word_list = [word for word in sentence.split(" ")]
word_len = len(word_list)
if self.sentence_vec_type == "matrix":
sentence_matrix = np.empty(word_len, dtype=list)
for idx, word in enumerate(word_list):
sentence_matrix[idx] = self.model.get_numpy_vector(word)
return sentence_matrix
else: # self.sentence_vec_type == "avg":
sentence_vector = np.zeros(self.vector_size) # <ndarray>
# print(f"type(sentence_vector): {type(sentence_vector)}")
for idx, word in enumerate(word_list):
# print(f"type(self.model.get_numpy_vector(word)): {type(self.model.get_numpy_vector(word))}") # <ndarry>
sentence_vector += self.model.get_numpy_vector(word)
return sentence_vector / len(word_list)
def gen_train_val_data(self):
"""
构造训练, 验证数据
"""
X_train = list()
y_train = list()
for line in open("../data/input/training_set.txt"):
line = line.strip().split("\t")
sent_vector = self.gen_sentence_vec(line[-1])
X_train.append(sent_vector)
y_train.append(int(line[0]))
X_val = list()
y_val = list()
for line in open("../data/input/validation_set.txt"):
line = line.strip().split("\t")
sent_vector = self.gen_sentence_vec(line[-1])
X_val.append(sent_vector)
y_val.append(int(line[0]))
return np.array(X_train), np.array(y_train), np.array(X_val), np.array(
y_val),
def train_bayes(self, X_train, y_train):
"""
基于Naive Bayes的分类算法
"""
from sklearn.naive_bayes import GaussianNB
model = GaussianNB()
model.fit(X_train, y_train)
joblib.dump(model, "../data/output/models/bayes_model")
def evaluate_bayes(self, model_path, X_val, y_val):
"""
基于Naive Bayes分类器的预测
"""
model = joblib.load(model_path)
y_val = list(y_val)
correct = 0
"""
y_predict = list()
for sent_vec in X_val: # sent_vec.shape: (self.vector_size,)
predicted = model.predict(sent_vec.reshape(1, -1)) # sent_vec.reshape(1, -1).shape: (1, self.vector_size)
y_predict.append(predicted[0])
"""
y_predict = model.predict(X_val)
print(f"len(y_predict): {len(y_predict)}, len(y_val): {len(y_val)}")
assert len(y_predict) == len(
y_val
), "Unexpected Error: len(y_predict) != len(y_val), but it should be"
for idx in range(len(y_predict)):
if int(y_predict[idx]) == int(y_val[idx]):
correct += 1
score = correct / len(y_predict)
print(f"Bayes Classification Accuray:{score}")
return score
def predict_bayes(self, model_path):
"""
实际应用测试
"""
model = joblib.load(model_path)
sentence = "这件 衣服 真的 太 好看 了 ！ 好想 买 啊 "
sent_vec = np.array(self.gen_sentence_vec(sentence)).reshape(1, -1)
print(f"'{sentence}': {model.predict(sent_vec)}") # 1: 负向
sentence = "这个 电视 真 尼玛 垃圾 ， 老子 再也 不买 了"
sent_vec = np.array(self.gen_sentence_vec(sentence)).reshape(1, -1)
print(f"'{sentence}': {model.predict(sent_vec)}") # 1: 负向
print('[{}] Start fasttext training'.format(time.time() - start_time))
model = fasttext.cbow('ftext_name.txt',
'model',
dim=24,
ws=4,
lr=.05,
min_count=1,
thread=8,
epoch=4,
silent=0)
modelcb = FastText('model.bin')
print('[{}] Start fasttext mat creation'.format(time.time() - start_time))
ftmat = np.zeros((merge.shape[0], 24))
for c, vals in tqdm(enumerate(merge[['category_name', 'name']].values)):
ftmat[c] = modelcb.get_numpy_sentence_vector(
'%s %s' % (vals[0].replace('/', ' '), vals[1]))
ftmat = pd.DataFrame(ftmat)
print('[{}] Finished fasttext mat creation'.format(time.time() - start_time))
ftmat.head()
'''
def make_weights(dict_):
embmat = np.zeros((len(dict_.keys())+1, 24))
for k, v in dict_.items():
embmat[v] = modelcb[k]
return embmat
embmatcat = make_weights(tok_raw_cat)
embmatnam = make_weights(tok_raw_nam)
embmatdsc = make_weights(tok_raw_dsc)
embmatntk = make_weights(tok_raw_ntk)
'''
class Title2Rec(AbstractRecommender):
def __init__(self, dataset=False, dry=True, w2r_model_file=None, pl_model_file=None, ft_model_file=None,
ft_vec_file=None, cluster_file=None, num_clusters=100, fallback=MostPopular, rnn=False):
super().__init__(dataset, dry=dry)
if rnn:
self.init_light(ft_model_file)
return
print('Import playlists')
self.playlists = self.dataset.reader(self.train_playlists, self.train_items)
self.playlists = np.array(list(filter(lambda p: p['title'] and len(p['items']) > 0, self.playlists)))
self.fallback = fallback(dataset, dry=dry)
if os.path.isfile(ft_model_file):
self.ft_model = FastText(ft_model_file)
else:
print('***Full init started***')
self.num_clusters = num_clusters
print('- Import w2r models')
self.w2r_model = self.get_w2r(dataset, dry, w2r_model_file)
print('- Compute playlists embeddings')
self.pl_embs = self.compute_pl_embs(pl_model_file)
print('- Cluster the playlist')
self.clusters = self.compute_clusters(cluster_file, self.pl_embs, num_clusters)
print('- Fast_text on the clusters')
self.ft_model = self.compute_fasttext(ft_model_file)
print('***Full init end***')
if not os.path.isfile(ft_vec_file):
self.title_vecs = [self.get_title_vector_from_playlist(pl) for pl in self.playlists]
with open(ft_vec_file, 'w') as file_handler:
file_handler.write('%d %d\n' % (len(self.title_vecs), len(self.title_vecs[0])))
for idx, vec in enumerate(self.title_vecs):
file_handler.write('%d %s\n' % (idx, ' '.join(vec.astype(np.str))))
self.pl_vec = KeyedVectors.load_word2vec_format(ft_vec_file, binary=False)
# nltk.download('stopwords')
def init_light(self, ft_model_file):
self.ft_model = FastText(ft_model_file)
def get_w2r(self, dataset, dry, model_file):
if os.path.isfile(model_file):
# Load the model
model = Word2Vec.load(model_file)
return model.wv
else:
# Train the model
w2r = Word2Rec(dataset, dry=dry, model_file=model_file, mode=sentence.Mode.ITEM)
return w2r.model
def compute_fasttext(self, ft_model_file):
ft_model_file = ft_model_file.replace('.bin', '')
descr_keywords = self.compute_tfidf_descr()
documents = []
for i in np.arange(self.num_clusters):
involved_pl = np.array(self.playlists)[self.clusters == i]
# take only collaboratives
involved_pl = list(filter(lambda pl: pl['collaborative'], involved_pl))
titles = [process_title(pl['title']).strip() for pl in involved_pl]
descr = [' '.join(descr_keywords[pl['pid']]) if pl['pid'] in descr_keywords else ''
for pl in involved_pl]
documents.append(titles + descr)
doc_file = 'models/documents.txt'
np.savetxt(doc_file, [' '.join(d) for d in documents], fmt='%s')
model = FastText()
model.skipgram(input=doc_file, output=ft_model_file, epoch=100, lr=0.1)
os.remove(doc_file)
return model
def compute_tfidf_descr(self):
description = dict()
with open('data/playlists_descr.csv', 'r', newline='') as descr_file:
description_reader = csv.reader(descr_file)
for d in description_reader:
text = d[1]
if "soundiiz" not in text:
description[d[0]] = text
tf = TfidfVectorizer(analyzer='word', ngram_range=(1, 1), min_df=0, stop_words='english')
corpus = [process_description(v) for k, v in description.items()]
tfidf_matrix = tf.fit_transform(corpus)
feature_names = tf.get_feature_names()
dense = tfidf_matrix.todense()
keywords = dict()
for i, k in enumerate(list(description.keys())):
keywords[k] = get_descr_keywords(i, 3, dense, feature_names)
return keywords
def compute_pl_embs(self, pl_model_file):
if os.path.isfile(pl_model_file):
# Load the model
return np.loadtxt(pl_model_file)
else:
# Train the model
_embs = [self.get_vector_from_w2r(playlist) for playlist in self.playlists]
np.savetxt(pl_model_file, _embs)
return _embs
def get_vector_from_w2r(self, playlist):
_item_embs = list(map(lambda track_id: self.w2r_model[str(track_id)], playlist['items']))
return np.array(_item_embs).mean(axis=0)
def compute_clusters(self, cluster_file, pl_embs, num_clusters=100):
if os.path.isfile(cluster_file):
return np.loadtxt(cluster_file)
else:
# from scipy.cluster.hierarchy import linkage, fcluster
# Z = linkage(pl_embs, 'ward')
# clusters = fcluster(Z, num_clusters, criterion='maxclust')
kmeans = KMeans(n_clusters=num_clusters, random_state=0).fit(pl_embs)
clusters = kmeans.predict(pl_embs)
np.savetxt(cluster_file, clusters, fmt="%u")
return clusters
def get_vector_from_title(self, title):
return self.ft_model.get_numpy_sentence_vector(process_title(title).strip())
def get_title_vector_from_playlist(self, playlist):
return self.get_vector_from_title(playlist['title'])
def recommend(self, playlist, n=500, n_pl=300):
if not playlist['title']:
return self.fallback.recommend(playlist)
this_vec = self.get_title_vector_from_playlist(playlist)
seeds = playlist['items']
# get more popular tracks among the 100 most similar playlists
most_similar_vec = self.pl_vec.most_similar(positive=[this_vec], topn=n_pl)
most_similar_pl = self.playlists[[int(v[0]) for v in most_similar_vec]]
weights = [v[1] for v in most_similar_vec]
# prioritize collaborative playlists
# weights = np.multiply(weights, [1.3 if pl['collaborative'] else 1.0 for pl in most_similar_pl])
# prioritize edited playlists
# weights = np.multiply(weights, [0.9 + pl['num_edits'] / 10 for pl in most_similar_pl])
predictions_and_seeds = [pl['items'] for pl in most_similar_pl]
playlist['items'] = count_and_weights(predictions_and_seeds, seeds, weights)[0:n]
def getFMFTRL():
#os.chdir('/Users/dhanley2/Documents/mercari/data')
os.chdir('/home/darragh/mercari/data')
train = pd.read_csv('../data/train.tsv', sep='\t', encoding='utf-8')
test = pd.read_csv('../data/test.tsv', sep='\t', encoding='utf-8')
glove_file = '../feat/glove.6B.50d.txt'
threads = 4
save_dir = '../feat'
print('[{}] Finished to load data'.format(time.time() - start_time))
print('Train shape: ', train.shape)
print('Test shape: ', test.shape)
nrow_test = train.shape[0] # -dftt.shape[0]
dftt = train[(train.price < 1.0)]
train = train.drop(train[(train.price < 1.0)].index)
del dftt['price']
nrow_train = train.shape[0]
# print(nrow_train, nrow_test)
y = np.log1p(train["price"])
merge = pd.concat([train, dftt, test])
merge['target'] = np.log1p(merge["price"])
submission = test[['test_id']]
ix = (merge['brand_name'] == merge['brand_name']) & (
~merge['brand_name'].str.lower().fillna('ZZZZZZ').isin(
merge['name'].str.lower()))
merge['name'][ix] = merge['brand_name'][ix] + ' ' + merge['name'][ix]
#EXTRACT DEVELOPTMENT TEST
trnidx, validx = train_test_split(range(train.shape[0]),
random_state=233,
train_size=0.90)
del train
del test
gc.collect()
merge['general_cat'], merge['subcat_1'], merge['subcat_2'] = \
zip(*merge['category_name'].apply(lambda x: split_cat(x)))
#merge.drop('category_name', axis=1, inplace=True)
print('[{}] Split categories completed.'.format(time.time() - start_time))
handle_missing_inplace(merge)
print('[{}] Handle missing completed.'.format(time.time() - start_time))
cutting(merge)
print('[{}] Cut completed.'.format(time.time() - start_time))
to_categorical(merge)
print('[{}] Convert categorical completed'.format(time.time() -
start_time))
'''
fasttext mats
'''
from pyfasttext import FastText
import fasttext
from tqdm import tqdm
fonm = open('ftext_name.txt', 'w')
for nm, ct, ds in zip(merge.name_token.str.lower(),
merge.category_name_split.str.lower(),
merge.description_token.str.lower()):
fonm.write('%s %s %s\n' % (ct.encode(
'ascii', 'ignore').lower(), ds.encode('ascii', 'ignore').lower(),
nm.encode('ascii', 'ignore').lower()))
fonm.close()
print('[{}] Start fasttext training'.format(time.time() - start_time))
model = fasttext.cbow('ftext_name.txt',
'model',
dim=24,
ws=4,
lr=.05,
min_count=1,
thread=8,
epoch=4,
silent=0)
modelcb = FastText('model.bin')
print('[{}] Start fasttext mat creation'.format(time.time() - start_time))
ftmat = np.zeros((merge.shape[0], 24))
for c, vals in tqdm(enumerate(merge[['category_name', 'name']].values)):
ftmat[c] = modelcb.get_numpy_sentence_vector(
'%s %s' % (vals[0].replace('/', ' '), vals[1]))
ftmat = coo_matrix(ftmat)
#ftmat = pd.DataFrame(ftmat)
#print('[{}] Finished fasttext mat creation'.format(time.time() - start_time))
#ftmat.head()
'''
Crossed columns
'''
# my understanding on how to replicate what layers.crossed_column does. One
# can read here: https://www.tensorflow.org/tutorials/linear.
def cross_columns(x_cols):
"""simple helper to build the crossed columns in a pandas dataframe
"""
crossed_columns = dict()
colnames = ['_'.join(x_c) for x_c in x_cols]
for cname, x_c in zip(colnames, x_cols):
crossed_columns[cname] = x_c
return crossed_columns
merge['item_condition_id_str'] = merge['item_condition_id'].astype(str)
merge['shipping_str'] = merge['shipping'].astype(str)
x_cols = (
['brand_name', 'item_condition_id_str'],
['brand_name', 'subcat_1'],
['brand_name', 'subcat_2'],
['brand_name', 'general_cat'],
#['brand_name', 'subcat_1', 'item_condition_id_str'],
#['brand_name', 'subcat_2', 'item_condition_id_str'],
#['brand_name', 'general_cat', 'item_condition_id_str'],
['brand_name', 'shipping_str'],
['shipping_str', 'item_condition_id_str'],
['shipping_str', 'subcat_2'],
['item_condition_id_str', 'subcat_2'])
crossed_columns_d = cross_columns(x_cols)
categorical_columns = list(merge.select_dtypes(include=['object']).columns)
D = 2**30
for k, v in crossed_columns_d.items():
print('Crossed column ', k)
outls_ = []
indicator = 0
for col in v:
outls_.append((np.array(merge[col].apply(hash))) % D + indicator)
indicator += 10**6
merge[k] = sum(outls_).tolist()
'''
Count crossed cols
'''
cross_nm = [k for k in crossed_columns_d.keys()]
lb = LabelBinarizer(sparse_output=True)
x_col = lb.fit_transform(merge[cross_nm[0]])
for i in range(1, len(cross_nm)):
x_col = hstack((x_col, lb.fit_transform(merge[cross_nm[i]])))
del (lb)
'''
Encode Original Strings
'''
for col in ['item_description', 'name']:
lb = LabelBinarizer(sparse_output=True)
if 'X_orig' not in locals():
X_orig = lb.fit_transform(merge[col].apply(hash))
else:
X_orig = hstack((X_orig, lb.fit_transform(merge[col].apply(hash))))
X_orig = hstack(
(X_orig,
lb.fit_transform(
(merge['item_description'] + merge['name']).apply(hash))))
X_orig = hstack((X_orig,
lb.fit_transform(
(merge['brand_name'] + merge['name']).apply(hash))))
X_orig = hstack((X_orig,
lb.fit_transform(
(merge['subcat_2'] + merge['name']).apply(hash))))
X_orig = hstack((X_orig,
lb.fit_transform(
(merge['brand_name'] + merge['name'] +
merge['item_description']).apply(hash))))
X_orig = X_orig.tocsr()
X_orig = X_orig[:,
np.
array(np.clip(X_orig.getnnz(axis=0) -
2, 0, 1), dtype=bool)]
X_orig = X_orig[:,
np.array(np.clip(X_orig.getnnz(axis=0) - 5000, 1, 0),
dtype=bool)]
print('Shape of original hash', X_orig.shape)
X_orig = X_orig.tocoo()
gc.collect()
cpuStats()
'''
Hash name
'''
wb = wordbatch.WordBatch(normalize_text,
extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2**29,
"norm": None,
"tf": 'binary',
"idf": None,
}),
procs=8)
wb.dictionary_freeze = True
X_name = wb.fit_transform(merge['name'])
del (wb)
X_name = X_name[:,
np.
array(np.clip(X_name.getnnz(axis=0) -
1, 0, 1), dtype=bool)]
print('[{}] Vectorize `name` completed.'.format(time.time() - start_time))
'''
Hash category
'''
wb = wordbatch.WordBatch(normalize_text,
extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2**20,
"norm": None,
"tf": 'binary',
"idf": None,
}),
procs=8)
wb.dictionary_freeze = True
cat = merge["category_name"].str.replace('/', ' ')
X_cat = wb.fit_transform(cat)
del (wb)
X_cat = X_cat[:,
np.array(np.clip(X_cat.getnnz(axis=0) -
1, 0, 1), dtype=bool)]
print('[{}] Vectorize `category` completed.'.format(time.time() -
start_time))
'''
Count category
'''
wb = CountVectorizer()
X_category1 = wb.fit_transform(merge['general_cat'])
X_category2 = wb.fit_transform(merge['subcat_1'])
X_category3 = wb.fit_transform(merge['subcat_2'])
print('[{}] Count vectorize `categories` completed.'.format(time.time() -
start_time))
# wb= wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 3, "hash_ngrams_weights": [1.0, 1.0, 0.5],
wb = wordbatch.WordBatch(normalize_text,
extractor=(WordBag, {
"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2**28,
"norm": "l2",
"tf": 1.0,
"idf": None
}),
procs=8)
wb.dictionary_freeze = True
X_description = wb.fit_transform(merge['item_description'])
del (wb)
X_description = X_description[:,
np.array(np.clip(
X_description.getnnz(axis=0) - 1, 0, 1),
dtype=bool)]
print('[{}] Vectorize `item_description` completed.'.format(time.time() -
start_time))
lb = LabelBinarizer(sparse_output=True)
X_brand = lb.fit_transform(merge['brand_name'])
print('[{}] Label binarize `brand_name` completed.'.format(time.time() -
start_time))
X_dummies = csr_matrix(
pd.get_dummies(merge[['item_condition_id', 'shipping']],
sparse=True).values)
print('[{}] Get dummies on `item_condition_id` and `shipping` completed.'.
format(time.time() - start_time))
print(X_dummies.shape, X_description.shape, X_brand.shape,
X_category1.shape, X_category2.shape, X_category3.shape,
X_name.shape, X_cat.shape, x_col.shape, X_orig.shape)
sparse_merge = hstack(
(X_dummies, X_description, X_brand, X_category1, X_category2,
X_category3, X_name, X_cat, x_col, X_orig, ftmat)).tocsr()
print('[{}] Create sparse merge completed'.format(time.time() -
start_time))
# Remove features with document frequency <=1 in the training set
print(sparse_merge.shape)
mask = np.array(np.clip(
sparse_merge[:nrow_train][trnidx].getnnz(axis=0) - 1, 0, 1),
dtype=bool)
sparse_merge = sparse_merge[:, mask]
X = sparse_merge[:nrow_train]
X_test = sparse_merge[nrow_test:]
print(sparse_merge.shape)
gc.collect()
if develop:
#train_X1, valid_X1, train_y1, valid_y1 = train_test_split(X, y, train_size=0.90, random_state=233)
train_X, valid_X, train_y, valid_y = X[trnidx], X[validx], y.values[
trnidx], y.values[validx]
model = FM_FTRL(alpha=0.01,
beta=0.01,
L1=0.00001,
L2=0.1,
D=sparse_merge.shape[1],
alpha_fm=0.01,
L2_fm=0.0,
init_fm=0.01,
D_fm=200,
e_noise=0.0001,
iters=1,
inv_link="identity",
threads=threads) #iters=15
baseline = 1.
for i in range(15):
model.fit(train_X, train_y, verbose=1)
predsfm = model.predict(X=valid_X)
score_ = rmsle(np.expm1(valid_y), np.expm1(predsfm))
print("FM_FTRL dev RMSLE:", score_)
if score_ < baseline:
baseline = score_
else:
break
print('[{}] Train ridge v2 completed'.format(time.time() - start_time))
if develop:
predsfm = model.predict(X=valid_X)
print("FM_FTRL dev RMSLE:", rmsle(np.expm1(valid_y),
np.expm1(predsfm)))
# 0.44532
# Full data 0.424681
predsFM = model.predict(X_test)
print('[{}] Predict FM_FTRL completed'.format(time.time() - start_time))
return merge, trnidx, validx, nrow_train, nrow_test, glove_file, predsFM, predsfm
class Preprocessing:
def __init__(self):
start_time = time.time()
# self.model = FastText("../data/input/models/sg_pyfasttext.bin") # DEBUG
self.model = FastText(
"../data/input/models/880w_fasttext_skip_gram.bin")
end_time = time.time()
print(f"Loading word vector model cost: {end_time - start_time:.2f}s")
# self.vocab_size, self.vector_size = self.model.numpy_normalized_vectors.shape # OK
self.vocab_size = self.model.nwords
self.vector_size = self.model.args.get("dim")
# self.vector_size:200, self.vocab_size: 925242
print(
f"self.vector_size:{self.vector_size}, self.vocab_size: {self.vocab_size}"
)
# 句子的表示形式:
# {"avg": 向量和的平均, "fasttext": get_numpy_sentence_vector, "concatenate": 向量拼接和补齐, "matrix": 矩阵}
self.sentence_vec_type = "matrix"
self.MAX_SENT_LEN = 70 # DEBUG: 超参数. self.get_sent_max_length()
# 对于"concatenate": self.MAX_SENT_LEN = 30, 取其他不同值的结果: 100: 50.22%, 80: 50.23%, 70: 50.33%, 60: 55.92%, 50: 69.11%, 40: 68.91%, 36: 69.34%, 30: 69.22%, 20: 69.17%, 10: 67.07%
# 对于"matrix": self.MAX_SENT_LEN = 70, 取其他不同值的结果: TODO:
@classmethod
def data_analysis(cls):
train_df = pd.read_csv("../data/input/training_set.txt",
sep="\t",
header=None,
names=["label", "sentence"])
val_df = pd.read_csv("../data/input/validation_set.txt",
sep="\t",
header=None,
names=["label", "sentence"])
y_train = train_df["label"]
y_val = val_df["label"]
sns.set(style="white", context="notebook", palette="deep")
# 查看样本数据分布情况(各个label数据是否均匀分布)
sns.countplot(y_train)
plt.show()
sns.countplot(y_val)
plt.show()
print(y_train.value_counts())
print(y_val.value_counts())
def set_sent_vec_type(self, sentence_vec_type):
assert sentence_vec_type in ["avg", "concatenate", "fasttext", "matrix"], \
"sentence_vec_type must be in ['avg', 'fasttext', 'concatenate', 'matrix']"
self.sentence_vec_type = sentence_vec_type
def get_sent_max_length(self): # NOT_USED
sent_len_counter = Counter()
max_length = 0
with open("../data/input/training_set.txt") as f:
for line in f:
content = line.strip().split("\t")[1]
content_list = content.split()
length = len(content_list)
sent_len_counter[length] += 1
if max_length <= length:
max_length = length
sent_len_counter = sorted(list(sent_len_counter.items()),
key=lambda x: x[0])
print(sent_len_counter)
# [(31, 1145), (32, 1105), (33, 1017), (34, 938), (35, 839), (36, 830), (37, 775), (38, 737), (39, 720), (40, 643), (41, 575), (42, 584), (43, 517), (44, 547), (45, 514), (46, 514), (47, 480), (48, 460), (49, 470), (50, 444), (51, 484), (52, 432), (53, 462), (54, 495), (55, 487), (56, 500), (57, 496), (58, 489), (59, 419), (60, 387), (61, 348), (62, 265), (63, 222), (64, 153), (65, 127), (66, 103), (67, 67), (68, 34), (69, 21), (70, 22), (71, 8), (72, 6), (73, 4), (74, 10), (75, 2), (76, 4), (77, 2), (78, 1), (79, 2), (80, 4), (81, 2), (82, 3), (83, 1), (84, 5), (86, 4), (87, 3), (88, 3), (89, 2), (90, 2), (91, 3), (92, 5), (93, 2), (94, 4), (96, 1), (97, 5), (98, 1), (99, 2), (100, 2), (101, 2), (102, 1), (103, 2), (104, 2), (105, 2), (106, 5), (107, 3), (108, 2), (109, 3), (110, 4), (111, 1), (112, 2), (113, 3), (114, 1), (116, 1), (119, 3), (679, 1)]
return max_length
def gen_sentence_vec(self, sentence):
"""
:param sentence:
:return:
"""
sentence = sentence.strip()
if self.sentence_vec_type == "fasttext":
return self.model.get_numpy_sentence_vector(sentence)
word_list = sentence.split(" ")
if self.sentence_vec_type == "concatenate":
sentence_vector = self.model.get_numpy_vector(word_list[0])
for word in word_list[1:]:
sentence_vector = np.hstack(
(sentence_vector, self.model.get_numpy_vector(word)))
return sentence_vector # NOTE: 对于concatenate情况, 每个句子的sentence_vector是不一样长的
if self.sentence_vec_type == "matrix": # for Deep Learning.
sentence_matrix = []
for word in word_list[
-self.
MAX_SENT_LEN:]: # NOTE: 截取后面的应该是要好些(参考https://github.com/lxw0109/SentimentClassification_UMICH_SI650/blob/master/src/LSTM_wo_pretrained_vector.py#L86)
sentence_matrix.append(self.model.get_numpy_vector(word))
length = len(sentence_matrix)
# 一定成立，因为上面做了切片截取
assert length <= self.MAX_SENT_LEN, "CRITICAL ERROR: len(sentence_matrix) > self.MAX_SENT_LEN."
# 参数中的matrix类型为list of ndarray, 返回值的matrix是ndarray of ndarray
sentence_matrix = np.pad(sentence_matrix,
pad_width=((0,
self.MAX_SENT_LEN - length),
(0, 0)),
mode="constant",
constant_values=-1)
return sentence_matrix
else: # self.sentence_vec_type == "avg":
sentence_vector = np.zeros(self.vector_size) # <ndarray>
# print(f"type(sentence_vector): {type(sentence_vector)}")
for idx, word in enumerate(word_list):
# print(f"type(self.model.get_numpy_vector(word)): {type(self.model.get_numpy_vector(word))}") # <ndarray>
sentence_vector += self.model.get_numpy_vector(word)
return sentence_vector / len(word_list)
def gen_train_val_data(self):
# 构造训练数据 & 验证数据
train_df = pd.read_csv("../data/input/training_set.txt",
sep="\t",
header=None,
names=["label", "sentence"])
val_df = pd.read_csv("../data/input/validation_set.txt",
sep="\t",
header=None,
names=["label", "sentence"])
# 打乱训练集的顺序. TODO: 不打乱感觉训练出来的模型是有问题的?(好看那句总是预测结果是1？)
train_df = train_df.sample(frac=1, random_state=1)
# val_df = val_df.sample(frac=1, random_state=1) # 验证集不用打乱
X_train = train_df["sentence"]
X_train_vec = list()
for sentence in X_train:
sent_vector = self.gen_sentence_vec(sentence)
X_train_vec.append(sent_vector)
y_train = train_df["label"] # <Series>
X_val = val_df["sentence"]
X_val_vec = list()
for sentence in X_val:
sent_vector = self.gen_sentence_vec(sentence)
X_val_vec.append(sent_vector)
y_val = val_df["label"] # <Series>
if self.sentence_vec_type == "concatenate":
# NOTE: 注意，这里的dtype是必须的，否则dtype默认值是"int32", 词向量所有的数值会被全部转换为0
X_train_vec = sequence.pad_sequences(X_train_vec,
maxlen=self.MAX_SENT_LEN *
self.vector_size,
value=0,
dtype=np.float)
X_val_vec = sequence.pad_sequences(X_val_vec,
maxlen=self.MAX_SENT_LEN *
self.vector_size,
value=0,
dtype=np.float)
return np.array(X_train_vec), np.array(X_val_vec), np.array(
y_train), np.array(y_val)