def main(): word_model = word2vec_model.word2vec_model() wv, vocabulary = load_embeddings(word_model) tsne = TSNE(n_components=2, random_state=0) np.set_printoptions(suppress=True) Y = tsne.fit_transform(wv[:1000,:]) # area = np.pi * similiarity plt.figure(figsize=(18, 18)) plt.scatter(Y[:, 0], Y[:, 1]) for label, x, y in zip(vocabulary, Y[:, 0], Y[:, 1]): plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points') plt.show()
query_wordcount[q] = ProcDoc.word_count(q_content, {}) query_unigram = ProcDoc.unigram(dict(query_wordcount)) query_model = query_unigram Pickle.dump(query_model, open("model/query_model.pkl", "wb"), True) # remove template file for rm_file in remove_list: if os.path.isfile("model/" + rm_file): os.remove("model/" + rm_file) # Embedded Query Expansion m_list = np.linspace(4, 4, num=1) m = 1 interpolated_aplpha_list = np.linspace(0, 1.0, num=11) word2vec = word2vec_model.word2vec_model(word_emb_path) embd = EmbeddedBased(query_wordcount, collection, word2vec) evaluate_model = EvaluateModel(relevance_path) EQE1 = [] EQE2 = [] print "Embedded..." # Embedding-based system (hyperparameter) tmp_eqe1 = embd.embedded_query_expansion_ci(0.4, 4) tmp_eqe2 = embd.embedded_query_expansion_qi(0.4, 4) tmp_eqe1 = ProcDoc.modeling(tmp_eqe1, background_model, query_lambda) tmp_eqe2 = ProcDoc.modeling(tmp_eqe2, background_model, query_lambda) EQE1.append([ProcDoc.dict2np(tmp_eqe1), tmp_eqe1]) EQE2.append([ProcDoc.dict2np(tmp_eqe2), tmp_eqe2]) Pickle.dump(EQE1, open("model/eqe1_10.pkl", "wb"), True)
newEmbeddingList = [] for idx, name in enumerate(objList): cur_idx = nameList.index(name) print EmbeddingList[cur_idx].shape newEmbeddingList.append(EmbeddingList[cur_idx]) return newEmbeddingList with open(model_path + "doc_list.pkl", "rb") as f: doc_list = Pickle.load(f) with open(model_path + "query_list.pkl", "rb") as f: qry_list = Pickle.load(f) with open(model_path + "test_query_list.pkl", "rb") as f: tstQry_list = Pickle.load(f) wordModel = word2vec_model.word2vec_model() wordVec = wordModel.getWord2Vec() vocab_length = wordModel.vocabulary_length print vocab_length # document doc = ProcDoc.read_file(document_path) doc = ProcDoc.doc_preprocess(doc) #[docTmpList, docEmbList] = content2Emb(doc, wordVec, 100) #doc_emb = rePermute(docTmpList, docEmbList, doc_list) #doc_emb = content2List(doc, doc_list) #doc_emb = np.asarray(doc_emb) #print doc_emb.shape #np.save(model_path + "doc_id_fix_pad.npy", doc_emb) # train query
query_wordcount[q] = ProcDoc.word_count(q_content, {}) query_unigram = ProcDoc.unigram(dict(query_wordcount)) query_model = query_unigram Pickle.dump(query_model, open("model/query_model.pkl", "wb"), True) ''' # remove template file for rm_file in remove_list: if os.path.isfile("model/" + rm_file): os.remove("model/" + rm_file) ''' # Embedded Query Expansion m_list = np.linspace(1, 80, num=1) m = 1 interpolated_aplpha_list = np.linspace(0, 1.0, num=11) word2vec = word2vec_model.word2vec_model() evaluate_model = EvaluateModel( "../Corpus/TDT2/AssessmentTrainSet/AssessmentTrainSet.txt") EQE1 = [] EQE2 = [] for m in m_list: [tmp_eqe1, tmp_eqe2] = Embedded_based.EmbeddedQuery(query_wordcount, collection, word2vec, 1, int(m)) tmp_eqe1 = ProcDoc.modeling(tmp_eqe1, background_model, query_lambda) tmp_eqe2 = ProcDoc.modeling(tmp_eqe2, background_model, query_lambda) EQE1.append(ProcDoc.dict2np(tmp_eqe1)) EQE2.append(ProcDoc.dict2np(tmp_eqe2)) Pickle.dump(EQE1, open("model/eqe1_10.pkl", "wb"), True) Pickle.dump(EQE2, open("model/eqe2_10.pkl", "wb"), True)