def form_clean_corpora(raw, company): """ :param company_list: 将原始预料raw_data中抽取company_list的新语料 :return: """ indexs = [] for i in range(len(raw.index)): if raw.ix[i, "Company"] not in company: indexs.append(i) clean_data = raw.drop(indexs, axis=0) datapreprocess.write_file(path="../Data_2/raw/clean_company_corpora.csv", item_data=clean_data) return clean_data
def seed_trigger(self, seed_words): """ :return: 每一类事件触发对应的words,且根据频数对其排序,选取前top个数据 """ top_seed_trigger = {} for event, value in seed_words.items(): top_seed_trigger[event] = self.sort_words(items=value) datapreprocess.write_file(path=self.trigger_path + "top_event_trigger.csv", item_data=pd.DataFrame(top_seed_trigger).T) datapreprocess.write_json_file(path=self.trigger_path + "top_event_trigger.json", item_data=top_seed_trigger) return top_seed_trigger
def seed_vec(self, e_trigger, filename, weight=False): """ :return: 得到每类事件的向量 """ logger.info("计算种子数据中的事件向量...") model = Word2Vec.load(self.model_path) seed_vec = {} for e, value in e_trigger.items(): seed_vec[e] = self.compute_vec(item=value, model=model, weight=weight) datapreprocess.write_file(path=self.trigger_path + filename, item_data=pd.DataFrame(seed_vec).T) return seed_vec
def form_target_data(item, label_map, company): indexs = [] for i in range(len(item.index)): if item.ix[i, "Company"] not in company: indexs.append(i) clean_data = item.drop(indexs, axis=0) clean_data = clean_data.dropna() clean_data["Label"] = clean_data["Label"].apply(lambda x: label_map[x]) for i in range(len(company)): print(company[i], len(clean_data[clean_data["Company"] == company[i]].index)) for i in range(1, 7): print(i, len(clean_data[clean_data["Label"] == i].index)) print(len(clean_data.index)) print(clean_data["Company"].unique()) datapreprocess.write_file(path="../Data_2/raw/target.csv", item_data=clean_data) return clean_data
def add_new_word(self, threshold=0.8, top=2000, enrich=True, source_from=1, new=False): """ 从测试集中提取top=1000的新词,加入到event :return: """ if enrich: logger.info("丰富种子动词形成触发器词...") # 加载语料库中高频新词 if source_from == 1: corpora_split_content = self.corpora_split_content.copy() else: # 加载测试集中高频新词 corpora_split_content = \ datapreprocess.read_file(path="../Data/pre_process/target_test_x.csv")["Split_content"] model = Word2Vec.load(self.model_path) new_trigger_words, seed_event_vec, all_word = self.top_seed_trigger.copy( ), self.seed_event_vec, [] final_trigger_words = {} # sorted the corpora high frequency words for i in range(len(corpora_split_content.index)): all_word.extend( literal_eval(corpora_split_content.ix[i, "Split_content"])) high_frequency_words = self.sort_words(all_word, top=top) new_trigger_words_list = [] for value in new_trigger_words.values(): for wd in value: new_trigger_words_list.append(wd[0]) # add new word to trigger. if not new: for item in high_frequency_words: if item[0] not in new_trigger_words_list: cosine_value = [] for i in sorted(seed_event_vec.keys()): if item[0] in model: vec_pair = [ seed_event_vec[i], model.wv[item[0]] ] cosine_value.append( cosine_similarity(vec_pair)[1, 0]) if cosine_value and max(cosine_value) >= threshold: event = cosine_value.index(max(cosine_value)) + 1 logger.info( "增加词:{word},频率:{frequency},相似度:{simi},对应事件{event}" .format(word=item[0], frequency=item[1], simi=max(cosine_value), event=event)) new_trigger_words[event].append(item) else: new_trigger_words = {1: [], 2: [], 3: [], 4: [], 5: [], 6: []} for item in high_frequency_words: cosine_value = [] for i in sorted(seed_event_vec.keys()): if item[0] in model: vec_pair = [seed_event_vec[i], model.wv[item[0]]] cosine_value.append( cosine_similarity(vec_pair)[1, 0]) if cosine_value and max(cosine_value) >= threshold: event = cosine_value.index(max(cosine_value)) + 1 logger.info( "增加词:{word},频率:{frequency},相似度:{simi},对应事件{event}". format(word=item[0], frequency=item[1], simi=max(cosine_value), event=event)) new_trigger_words[event].append(item) datapreprocess.write_json_file(path=self.trigger_path + "event_words_add_new.json", item_data=new_trigger_words) high_top = min( list(map(lambda x: len(x), new_trigger_words.values()))) logger.info("每类标签的词数{num}".format(num=high_top)) for key, value in new_trigger_words.items(): final_trigger_words[key] = value[:high_top] datapreprocess.write_json_file(path=self.trigger_path + "final_event_words.json", item_data=final_trigger_words) datapreprocess.write_file( path=self.trigger_path + "final_event_word.csv", item_data=pd.DataFrame(final_trigger_words).T) return final_trigger_words else: return self.top_seed_trigger.copy()
def form_test_data(item): test_data = item.drop("Sentence_vec", axis=1) datapreprocess.write_file(path="../Data_2/pre_process/test_data_label.csv", item_data=test_data)