def get_item_embeddings(self): term_embed = DataSet.load_term_embed(self.data_space) embed_size = term_embed.shape[1] pad_embed = np.array([0]*embed_size).reshape(1,-1) all_embeding = np.vstack([pad_embed,term_embed]) all_index = [_PAD_]+term_embed.index.values.tolist() all_embeding_df = pd.DataFrame(data=all_embeding,index=all_index) sort_word = [self.idx2item[i] for i in range(len(self.idx2item))] return all_embeding_df.loc[sort_word].values
def __init__(self,data_df,space,bucket_num,batch_size,is_prefix_pad,is_shuffle,is_test): assert space in ["words","chars"] self.data_df = data_df self.space = space self.bucket_num = bucket_num self.batch_size = batch_size self.is_prefix_pad = is_prefix_pad self.is_shuffle = is_shuffle self.is_test = is_test if os.path.exists(self._temp_file): print("detect cached intermediate files...loading...") if DataGenerator.item2idx is None: all_cached = pickle.load(open(self._temp_file,"rb")) DataGenerator.item2idx = all_cached["item2idx"] DataGenerator.idx2item = all_cached["idx2item"] DataGenerator.item_embed = all_cached["item_embed"] DataGenerator.q2idvec = all_cached["q2idvec"] print("finish") else: print("Generating intermediate files...") DataGenerator.item2idx = {} DataGenerator.idx2item = {} DataGenerator.item_embed = {} DataGenerator.q2idvec = {} spaces = ["words","chars"] question_df = DataSet.load_all_questions() all_qids = DataSet.load_all_unique_ids_train_test() for space in spaces: print("for",space) corpus = question_df[space] w2i,i2w = self._get_item2id_id2item(corpus) DataGenerator.item2idx[space] = w2i DataGenerator.idx2item[space] = i2w ##Finish mapping table term_embed = DataSet.load_term_embed(space) embed_size = term_embed.shape[1] pad_embed = np.array([0] * embed_size).reshape(1, -1) all_embeding = np.vstack([pad_embed, term_embed]) all_index = [_PAD_] + term_embed.index.values.tolist() all_embeding_df = pd.DataFrame(data=all_embeding, index=all_index) sort_word = [i2w[i] for i in range(len(i2w))] DataGenerator.item_embed[space] = all_embeding_df.loc[sort_word].values ##Finish item embedding tmp_q2idvec = {} for qid in all_qids: items = question_df.loc[qid][space].split() idvec = np.array([w2i[w] for w in items]) tmp_q2idvec[qid] = idvec DataGenerator.q2idvec[space]=tmp_q2idvec ##Finish map from question to id vector print("finish generating inter files.") print("begin caching..") all_cached = {} all_cached["item2idx"] = DataGenerator.item2idx all_cached["idx2item"] = DataGenerator.idx2item all_cached["item_embed"] = DataGenerator.item_embed all_cached["q2idvec"] = DataGenerator.q2idvec try: os.makedirs("./temp") except: pass pickle.dump(all_cached,open(self._temp_file,"wb")) print("finish caching")