def compute_scaled_average_sim(early, later):
M_early = WordEmbeddingsKeyedVectors(300)
M_early.add(words, early.to_numpy())
M_later = WordEmbeddingsKeyedVectors(300)
M_later.add(words, later.to_numpy())
scores = list()
for word in words:
score = 0
early_values = M_early.most_similar(word, topn=20)
later_values = M_later.most_similar(word, topn=20)
early_dict = {v[0]:v[1] for v in early_values}
later_dict = {v[0]:v[1] for v in later_values}
overlap = set([w for w in early_dict.keys() if w in later_dict])
early_avg = 0
later_avg = 0
for entry in overlap:
early_avg += early_dict[entry]
later_avg += later_dict[entry]
early_avg = early_avg / len(overlap) if len(overlap) else 0
later_avg = later_avg / len(overlap) if len(overlap) else 0
scores.append(len(overlap) + (1 - abs(later_avg - early_avg)))
return scores
def getRecos(liked, disliked, watched, threshold=1000):
liked = [str(l) for l in liked]
disliked = [str(l) for l in disliked]
watched = [str(w) for w in watched if str(
w) not in liked and str(w) not in disliked]
df_restr = df_movies[~df_movies["movieId"].isin(watched)].sort_values(
by="count", ascending=False)
kv = WordEmbeddingsKeyedVectors(movie_embedding_size)
kv.add(
df_restr['movieId'].apply(str).values,
w[df_restr.movieId]
)
idlist = [int(x[0])
for x in kv.most_similar(positive=liked, negative=disliked, restrict_vocab=4000, topn=12)]
return getAll(idlist=idlist)
def createKVs(DFFile, COFIle, type):
#createWordandVectorList() # to create word and vector list for wiki 50
# wordList - list of words
# vectorList - list of the vector corresponding to the words
wordListW2V, vectorListW2V = loadWordANdVectorsW2V()
wordListPCA, vectorListPCA = loadWordANdVectorsPCA(DFFile, COFIle)
w2v_len = 50
PCA_len = 10
kv1 = WordEmbeddingsKeyedVectors(w2v_len)
kv2 = WordEmbeddingsKeyedVectors(PCA_len)
kv1.add(wordListW2V, list(vectorListW2V))
kv2.add(wordListPCA, vectorListPCA)
filename = 'KV' + type + '.obj'
with open(filename, "wb") as f:
pickle.dump(kv1, f)
pickle.dump(kv2, f)
print(kv1.most_similar('love')) # gives the list of words similar to word1
return filename
def make_prediction(favofite_movie):
"""
Input:
feature_dict: a dictionary of the form {"feature_name": "value"}
Function makes sure the features are fed to the model in the same order the
model expects them.
Output:
Returns (x_inputs, probs) where
x_inputs: a list of feature values in the order they appear in the model
probs: a list of dictionaries with keys 'name', 'prob'
"""
movie = favofite_movie
threshold = 100
mainstream_movies = movies_df[
movies_df.n_ratings >= threshold].reset_index(drop=True)
movie_embedding_size = w.shape[1]
kv = WordEmbeddingsKeyedVectors(movie_embedding_size)
kv.add(mainstream_movies['key'].values, w[mainstream_movies.movieId])
results = kv.most_similar(movie)
return [result[0] for result in results]
def run(song_meta_data, train_data, test_data):
train_data['updt_year'] = train_data['updt_date'].str.slice(start=0,
stop=4)
test_data['updt_year'] = test_data['updt_date'].str.slice(start=0, stop=4)
song_meta_data['issue_year'] = song_meta_data['issue_date'].str.slice(
start=0, stop=4)
song_meta_data['id'] = song_meta_data['id'].astype(str)
print("Tokenize...")
tokenize(train_data, test_data)
train_data = train_data.sort_values(by='updt_date').reset_index(drop=True)
test_data = test_data.sort_values(by='updt_date').reset_index(drop=True)
print("Total Dict Loading")
if os.path.exists(
BASE_DIR + 'model/total_data_final.pickle') and os.path.exists(
BASE_DIR + 'model/song_dict_final.pickle') and os.path.exists(
BASE_DIR +
'model/tag_dict_final.pickle') and os.path.exists(
BASE_DIR + 'model/title_dict_final.pickle'):
with open(BASE_DIR + 'model/total_data_final.pickle', 'rb') as handle:
total_data = pickle.load(handle)
with open(BASE_DIR + 'model/song_dict_final.pickle', 'rb') as handle:
song_dict = pickle.load(handle)
with open(BASE_DIR + 'model/tag_dict_final.pickle', 'rb') as handle:
tag_dict = pickle.load(handle)
with open(BASE_DIR + 'model/title_dict_final.pickle', 'rb') as handle:
title_dict = pickle.load(handle)
else:
print("Total Dict Not Existing... Calculating")
total_data, song_dict, tag_dict, title_dict = getTotalDict(
train_data, test_data)
with open(BASE_DIR + 'model/total_data_final.pickle', 'wb') as handle:
pickle.dump(total_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(BASE_DIR + 'model/song_dict_final.pickle', 'wb') as handle:
pickle.dump(song_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(BASE_DIR + 'model/tag_dict_final.pickle', 'wb') as handle:
pickle.dump(tag_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(BASE_DIR + 'model/title_dict_final.pickle', 'wb') as handle:
pickle.dump(title_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
print("Frequency Loading...")
if os.path.exists(BASE_DIR +
'model/tag_freq_by_song.pickle') and os.path.exists(
BASE_DIR + 'model/song_freq_by_tag.pickle'):
with open(BASE_DIR + 'model/tag_freq_by_song.pickle', 'rb') as handle:
tag_freq_by_song = pickle.load(handle)
with open(BASE_DIR + 'model/song_freq_by_tag.pickle', 'rb') as handle:
song_freq_by_tag = pickle.load(handle)
else:
print("Frequency Not Existing... Calculating")
tag_freq_by_song, song_freq_by_tag = getFreqDict(train_data)
with open(BASE_DIR + 'model/tag_freq_by_song.pickle', 'wb') as handle:
pickle.dump(tag_freq_by_song,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
with open(BASE_DIR + 'model/song_freq_by_tag.pickle', 'wb') as handle:
pickle.dump(song_freq_by_tag,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
print("Update Date Loading...")
if os.path.exists(BASE_DIR + 'model/updt_dict.pickle'):
with open(BASE_DIR + 'model/updt_dict.pickle', 'rb') as handle:
updt_dict = pickle.load(handle)
else:
print("Update Date Not Existing... Calculating")
updt_dict = getUpdtDict(song_meta_data)
with open(BASE_DIR + 'model/updt_dict.pickle', 'wb') as handle:
pickle.dump(updt_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
print("Song Popularity Loading...")
if os.path.exists(BASE_DIR +
'model/popular_song_by_year.pickle') and os.path.exists(
BASE_DIR + 'model/popular_tag_by_year.pickle'):
with open(BASE_DIR + 'model/popular_tag_by_year.pickle',
'rb') as handle:
popular_tag_by_year = pickle.load(handle)
with open(BASE_DIR + 'model/popular_song_by_year.pickle',
'rb') as handle:
popular_song_by_year = pickle.load(handle)
else:
print("Song Popularity Not Existing... Calculating")
popular_song_by_year, popular_tag_by_year = getPopularDict(train_data)
with open(BASE_DIR + 'model/popular_tag_by_year.pickle',
'wb') as handle:
pickle.dump(popular_tag_by_year,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
with open(BASE_DIR + 'model/popular_song_by_year.pickle',
'wb') as handle:
pickle.dump(popular_song_by_year,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
print("Word2Vec Model Loading...")
if os.path.exists(BASE_DIR + 'model/w2v_model_sg_title.model'):
w2v_model = Word2Vec.load(BASE_DIR + 'model/w2v_model_sg_title.model')
else:
print("Word2Vec Model Not Found !")
print("Training...")
w2v_model = Word2Vec(total_data,
min_count=3,
size=100,
window=210,
sg=1)
w2v_model.save(BASE_DIR + 'model/w2v_model_sg_title.model')
print("Training...")
p2v_model = WordEmbeddingsKeyedVectors(100)
updateP2V(train_data, test_data, w2v_model, p2v_model, song_dict, tag_dict,
title_dict)
print("Word2Vec Second Model Loading...")
if os.path.exists(BASE_DIR +
'model/w2v_tag_final.model') and os.path.exists(
BASE_DIR + 'model/w2v_song_final.model'):
tag_model = Word2Vec.load(BASE_DIR + 'model/w2v_tag_final.model')
song_model = Word2Vec.load(BASE_DIR + 'model/w2v_song_final.model')
mt = W2VModel(tag_model, "tags")
ms = W2VModel(song_model, "songs")
else:
print("Word2Vec Second Model Not Found !")
print("Tag Training...")
mt = W2VModel(pd.concat([train_data, test_data]), "tags")
mt.model.save(BASE_DIR + 'model/w2v_tag_final.model')
print("Song Training...")
ms = W2VModel(pd.concat([train_data, test_data]), "songs")
ms.model.save(BASE_DIR + 'model/w2v_song_final.model')
print("start")
answer = []
for i, row in tqdm(test_data.iterrows()):
year = str(row['updt_year'])
id = str(row['id'])
songs = []
tags = []
try:
most_id_list = [x[0] for x in p2v_model.most_similar(id, topn=200)]
fillAnswer(getItemById(most_id_list, song_dict, 200), songs, 100,
song_dict, id, updt_dict, year)
fillAnswer(getItemById(most_id_list, tag_dict, 20), tags, 10,
tag_dict, id)
except:
pass
if len(songs) < 100:
fillAnswer(ms.recommand(test_data, int(row['id']), 200), songs,
100, song_dict, id, updt_dict, year)
if len(tags) < 10:
fillAnswer(mt.recommand(test_data, int(row['id']), 20), tags, 10,
tag_dict, id)
if len(songs) < 100:
fillAnswer(getSongByTagFreq(song_freq_by_tag, row['tags'], 200),
songs, 100, song_dict, id, updt_dict, year)
if len(tags) < 10:
fillAnswer(getTagBySongFreq(tag_freq_by_song, row['songs'], 20),
tags, 10, tag_dict, id)
if len(songs) < 100:
fillAnswer(getSongByYear(popular_song_by_year, year, 200), songs,
100, song_dict, id, updt_dict, year)
if len(tags) < 10:
fillAnswer(getTagByYear(popular_tag_by_year, year, 20), tags, 10,
tag_dict, id)
if len(songs) < 100:
try:
fillAnswer(
getSongByYear(popular_song_by_year, str(int(year) - 1),
20), songs, 100, song_dict, id, updt_dict,
year)
except:
fillAnswer(
getSongByYear(popular_song_by_year, str(int(year) + 1),
200), songs, 100, song_dict, id, updt_dict,
year)
if len(tags) < 10:
try:
fillAnswer(
getTagByYear(popular_tag_by_year, str(int(year) - 1), 20),
tags, 10, tag_dict, id)
except:
fillAnswer(
getTagByYear(popular_tag_by_year, str(int(year) + 1), 200),
tags, 10, tag_dict, id)
if len(songs) < 100:
print("song 의 개수가 적습니다. id : ", str(row['id']), str(year))
if len(tags) < 10:
print("tag 의 개수가 적습니다. id : ", str(row['id']), str(year))
answer.append({"id": row["id"], "songs": songs, "tags": tags})
write_json(answer, "results.json")
def train():
MODE = "Test"
if MODE == "Valid":
train = load_json("arena_data/orig/train.json") + load_json(
"arena_data/questions/val.json")
dev = load_json("res/val.json")
test = load_json("res/test.json")
else:
train = load_json("res/train.json")
dev = load_json("res/val.json")
test = load_json("res/test.json")
pred_tag = load_json("arena_data/model/pred_tag.json")
dic_pred_tag = {}
for p_t in pred_tag:
dic_pred_tag[p_t['id']] = p_t['predict_tag']
for doc in train:
doc['tags_org'] = doc['tags'].copy()
doc['tags'] += dic_pred_tag[doc['id']]
for doc in dev:
doc['tags_org'] = doc['tags'].copy()
doc['tags'] += dic_pred_tag[doc['id']]
for doc in test:
doc['tags_org'] = doc['tags'].copy()
doc['tags'] += dic_pred_tag[doc['id']]
item_list = []
len_item = []
for doc in train + dev + test:
song_list = []
for i in doc['songs']:
song_list.append(str(i))
item_list.append(song_list + doc['tags'])
len_item.append(len(song_list + doc['tags']))
print("Max length of item list :", max(len_item), ", Min :", min(len_item))
item_list = [x for x in item_list if len(x) > 1]
print("Train set :", len(item_list))
print("Training Item2Vec model")
SIZE = 100
model = Word2Vec(sentences=item_list,
size=SIZE,
window=240,
min_count=2,
sg=1,
workers=8,
iter=10,
negative=7,
compute_loss=True,
callbacks=[LossPrinter()])
model.save("arena_data/model/word2vec.model")
print("Vocab : ", len(model.wv.vocab))
print("Building and saving playlist embeddings")
song_dic = {}
tag_dic = {}
for q in tqdm(train + test + dev):
song_dic[str(q['id'])] = q['songs']
tag_dic[str(q['id'])] = q['tags_org']
p2v_song = WordEmbeddingsKeyedVectors(SIZE)
ID = []
vec = []
for q in tqdm(train + test + dev):
tmp_vec = 0
cnt_vocab = 0
if len(q['songs']) >= 1:
for item in q['songs']:
try:
tmp_vec += model.wv.get_vector(str(item)) * 2
cnt_vocab += 1
except KeyError:
pass
if len(q['tags']) >= 1:
for item in q['tags']:
try:
tmp_vec += model.wv.get_vector(str(item))
cnt_vocab += 1
except KeyError:
pass
if type(tmp_vec) != int:
ID.append(str(q['id']))
vec.append(tmp_vec)
p2v_song.add(ID, vec)
p2v_song.save("arena_data/model/p2v_song.model")
p2v_tag = WordEmbeddingsKeyedVectors(SIZE)
ID = []
vec = []
for q in tqdm(train + test + dev):
tmp_vec = 0
cnt_vocab = 0
if len(q['songs']) >= 1:
for item in q['songs']:
try:
tmp_vec += model.wv.get_vector(str(item))
cnt_vocab += 1
except KeyError:
pass
if len(q['tags']) >= 1:
for item in q['tags']:
try:
tmp_vec += model.wv.get_vector(str(item)) * 2
cnt_vocab += 1
except KeyError:
pass
if type(tmp_vec) != int:
ID.append(str(q['id']))
vec.append(tmp_vec)
p2v_tag.add(ID, vec)
p2v_tag.save("arena_data/model/p2v_tag.model")
if MODE == "Valid":
print("Testing")
questions = load_json("arena_data/questions/val.json")
cnt_wv_song = 0
cnt_wv_tag = 0
res = []
for q in tqdm(questions):
dic_song_score = {}
dic_tag_score = {}
song_result = []
tag_result = []
if str(q['id']) in p2v_song.wv.vocab:
most_id = [
x for x in p2v_song.most_similar(str(q['id']), topn=50)
]
for ID in most_id:
for s in song_dic[ID[0]]:
if s in dic_song_score:
dic_song_score[s] += ID[1]
else:
dic_song_score[s] = ID[1]
if str(q['id']) in p2v_tag.wv.vocab:
most_id = [
x for x in p2v_tag.most_similar(str(q['id']), topn=50)
]
for t in tag_dic[ID[0]]:
if t in dic_tag_score:
dic_tag_score[t] += ID[1]
else:
dic_tag_score[t] = ID[1]
if len(dic_song_score) > 0:
sort_song_score = sorted(dic_song_score.items(),
key=lambda x: x[1],
reverse=True)
for s in sort_song_score:
song_result.append(s[0])
cnt_wv_song += 1
if len(dic_tag_score) > 0:
sort_tag_score = sorted(dic_tag_score.items(),
key=lambda x: x[1],
reverse=True)
for s in sort_tag_score:
tag_result.append(s[0])
cnt_wv_tag += 1
res.append({
"id": q["id"],
"songs": remove_seen(q["songs"], song_result)[:100],
"tags": remove_seen(q["tags"], tag_result)[:10],
})
print(len(questions), cnt_wv_song, cnt_wv_tag)
ans = load_json("arena_data/answers/val.json")
evaluator = CustomEvaluator()
evaluator._evaluate(ans, res)
def run(total_concat, apply_data):
total_concat['id'] = total_concat['id'].astype(str)
c = Counter()
for i in total_concat['tags']:
c.update(i)
tag_list = list(
map(lambda y: y[0], (filter(lambda x: x[1] > 5, c.items()))))
p = re.compile('|'.join(tag_list))
total_concat['tag_in_title'] = total_concat['plylst_title'].apply(
lambda x: p.findall(x))
data = []
for i in total_concat.index:
temp = total_concat.loc[i]
data.append({
'id': temp['id'],
'songs': temp['songs'],
'tags': temp['tags'],
'tag_in_title': temp['tag_in_title']
})
song_dic = {}
tag_dic = {}
for q in data:
song_dic[q['id']] = q['songs']
tag_dic[q['id']] = q['tags']
total = list(
map(
lambda x: list(map(str, x['songs'])) + x['tags'] + x['tag_in_title'
], data))
total = [x for x in total if len(x) > 1]
print("start training item2Vec")
size = 300
if 'item2vec.model' in os.listdir():
w2v_model = Word2Vec.load('item2vec.model')
else:
w2v_model = train.item2vec(total, size=size)
print("done. \n")
p2v_model = WordEmbeddingsKeyedVectors(size)
ID = []
vec = []
for q in data:
tmp_vec = 0
for song in list(map(str, q['songs'])) + q['tags'] + q['tag_in_title']:
try:
tmp_vec += w2v_model.wv.get_vector(song)
except KeyError:
pass
if type(tmp_vec) != int:
ID.append(str(q['id']))
vec.append(tmp_vec)
p2v_model.add(ID, vec)
with open("./arena_data/pre_tag.json", encoding="utf-8") as f:
our_best = json.load(f)
not_in = 0
answers = []
for i, q in enumerate(apply_data.index):
q = apply_data.loc[q]
try:
most_id = [
x[0] for x in p2v_model.most_similar(str(q['id']), topn=200)
]
get_song = []
get_tag = []
for ID in most_id:
get_song += song_dic[ID]
get_tag += tag_dic[ID]
get_song = list(pd.value_counts(get_song)[:300].index)
get_tag = list(pd.value_counts(get_tag)[:30].index)
output_song = remove_seen(q["songs"], get_song)[:100]
output_tag = remove_seen(q["tags"], get_tag)[:10]
answers.append({
"id": q["id"],
"songs": output_song,
"tags": output_tag,
})
except KeyError:
not_in += 1
answers.append({
"id": our_best[i]["id"],
"songs": our_best[i]['songs'],
"tags": our_best[i]["tags"],
})
for n, q in enumerate(answers):
if len(q['songs']) != 100:
answers[n]['songs'] += remove_seen(
q['songs'], our_best[n]['songs'])[:100 - len(q['songs'])]
if len(q['tags']) != 10:
answers[n]['tags'] += remove_seen(
q['tags'], our_best[n]['tags'])[:10 - len(q['tags'])]
write_json(answers, 'final_tags.json')
return answers