def __init__(self, target_train_path, test_x_path, test_y_path):
self.svm_model = NuSVC()
self.bayes_model = MultinomialNB()
self.nn_model = MLPClassifier()
self.target_data = datapreprocess.read_file(target_train_path)
self.test_x = datapreprocess.read_file(test_x_path)
self.test_y = datapreprocess.read_file(test_y_path)
def __init__(self, train_path, corpora_split_path, model_path):
self.trigger_path = "../Data/trigger/"
self.model_path = model_path
self.target_train_data = datapreprocess.read_file(train_path)
self.corpora_split_content = datapreprocess.read_file(
corpora_split_path)
self.seed_words, self.top_seed_trigger, self.seed_event_vec, self.event_words, self.event_vec = \
None, None, None, None, None
def __init__(self, train_path, corpora_split_path, model_path, tain_x_path,
train_y_path, test_x_path, test_y_path, weight_1, weight_2,
method, threshold, top, enrich, source_from, new):
Trigger.__init__(self, train_path, corpora_split_path, model_path)
self.event_vec = Trigger.run(self,
weight_1=weight_1,
weight_2=weight_2,
method=method,
threshold=threshold,
top=top,
enrich=enrich,
source_from=source_from,
new=new)
self.model_path = "../Data/model/"
self.train_x = datapreprocess.read_file(tain_x_path)
self.train_y = datapreprocess.read_file(train_y_path)
self.test_x = datapreprocess.read_file(test_x_path)
self.test_y = datapreprocess.read_file(test_y_path)
def __init__(self, train_path, corpora_split_path, test_vec_path,
test_label_path, model_path, weight_1, weight_2, threshold,
top_1, top_2, enrich, source_from, new):
Trigger.__init__(self, train_path, corpora_split_path, model_path,
test_vec_path)
self.event_vec = Trigger.run(self,
weight_1=weight_1,
weight_2=weight_2,
threshold=threshold,
top_1=top_1,
top_2=top_2,
enrich=enrich,
source_from=source_from,
new=new)
self.model_path = "../Data/model/"
self.train_x = self.target_train_data
self.train_y = self.target_train_data
self.test_x = self.test_split_data
self.test_y = datapreprocess.read_file(test_label_path)
def convert_one_hot(self, item_data):
target_words = datapreprocess.read_file(
path="../Data/pre_process/target_represent.csv")["Split_content"]
all_words = set()
for i in range(len(target_words.index)):
all_words = all_words | set(
literal_eval(target_words.ix[i, "Split_content"]))
all_words = list(all_words)
onehot_result = pd.DataFrame(np.zeros(
(len(item_data.index), len(all_words))),
columns=all_words,
index=item_data.index)
for i in range(len(item_data.index)):
words = literal_eval(item_data.ix[i, "Split_content"])
for word in words:
if word in all_words:
onehot_result.ix[i, word] = onehot_result.ix[i, word] + 1
else:
pass
return onehot_result
def add_new_word(self,
threshold=0.8,
top=2000,
enrich=True,
source_from=1,
new=False):
"""
从测试集中提取top=1000的新词，加入到event
:return:
"""
if enrich:
logger.info("丰富种子动词形成触发器词...")
# 加载语料库中高频新词
if source_from == 1:
corpora_split_content = self.corpora_split_content.copy()
else:
# 加载测试集中高频新词
corpora_split_content = \
datapreprocess.read_file(path="../Data/pre_process/target_test_x.csv")["Split_content"]
model = Word2Vec.load(self.model_path)
new_trigger_words, seed_event_vec, all_word = self.top_seed_trigger.copy(
), self.seed_event_vec, []
final_trigger_words = {}
# sorted the corpora high frequency words
for i in range(len(corpora_split_content.index)):
all_word.extend(
literal_eval(corpora_split_content.ix[i, "Split_content"]))
high_frequency_words = self.sort_words(all_word, top=top)
new_trigger_words_list = []
for value in new_trigger_words.values():
for wd in value:
new_trigger_words_list.append(wd[0])
# add new word to trigger.
if not new:
for item in high_frequency_words:
if item[0] not in new_trigger_words_list:
cosine_value = []
for i in sorted(seed_event_vec.keys()):
if item[0] in model:
vec_pair = [
seed_event_vec[i], model.wv[item[0]]
]
cosine_value.append(
cosine_similarity(vec_pair)[1, 0])
if cosine_value and max(cosine_value) >= threshold:
event = cosine_value.index(max(cosine_value)) + 1
logger.info(
"增加词：{word},频率:{frequency},相似度:{simi},对应事件{event}"
.format(word=item[0],
frequency=item[1],
simi=max(cosine_value),
event=event))
new_trigger_words[event].append(item)
else:
new_trigger_words = {1: [], 2: [], 3: [], 4: [], 5: [], 6: []}
for item in high_frequency_words:
cosine_value = []
for i in sorted(seed_event_vec.keys()):
if item[0] in model:
vec_pair = [seed_event_vec[i], model.wv[item[0]]]
cosine_value.append(
cosine_similarity(vec_pair)[1, 0])
if cosine_value and max(cosine_value) >= threshold:
event = cosine_value.index(max(cosine_value)) + 1
logger.info(
"增加词：{word},频率:{frequency},相似度:{simi},对应事件{event}".
format(word=item[0],
frequency=item[1],
simi=max(cosine_value),
event=event))
new_trigger_words[event].append(item)
datapreprocess.write_json_file(path=self.trigger_path +
"event_words_add_new.json",
item_data=new_trigger_words)
high_top = min(
list(map(lambda x: len(x), new_trigger_words.values())))
logger.info("每类标签的词数{num}".format(num=high_top))
for key, value in new_trigger_words.items():
final_trigger_words[key] = value[:high_top]
datapreprocess.write_json_file(path=self.trigger_path +
"final_event_words.json",
item_data=final_trigger_words)
datapreprocess.write_file(
path=self.trigger_path + "final_event_word.csv",
item_data=pd.DataFrame(final_trigger_words).T)
return final_trigger_words
else:
return self.top_seed_trigger.copy()
print(company[i],
len(clean_data[clean_data["Company"] == company[i]].index))
for i in range(1, 7):
print(i, len(clean_data[clean_data["Label"] == i].index))
print(len(clean_data.index))
print(clean_data["Company"].unique())
datapreprocess.write_file(path="../Data_2/raw/target.csv",
item_data=clean_data)
return clean_data
def form_test_data(item):
test_data = item.drop("Sentence_vec", axis=1)
datapreprocess.write_file(path="../Data_2/pre_process/test_data_label.csv",
item_data=test_data)
if __name__ == "__main__":
company_list = [
'中兴通讯', '华为中国区', 'vivo智能手机', 'coolpad官方微博', 'OPPO', '联想', '小米公司',
'金立智能手机', "小辣椒", "360手机", 'HTC官方微博', '魅族科技', '天语手机', '超级手机官微',
'朵唯女性手机', '锤子科技', 'TCL通讯中国', 'nubia智能手机'
]
label_num = {"招募": 1, "合作": 2, "研发": 3, "推广": 4, "销售": 5, "无": 6}
# raw_data = datapreprocess.read_file(path="../Data/raw/Corpora.csv")
# form_clean_corpora(raw_data, company_list)
# item = datapreprocess.read_file(path="../Data_2/raw/Corpora_label.csv")
# form_target_data(item=item, label_map=label_num, company=company_list)
data = datapreprocess.read_file(path="../Data_2/pre_process/test_data.csv")
form_test_data(data)
def write_file(self, item_data):
with open("../Data/raw/Corpora.csv", "w", encoding="utf-8") as f:
f.write("Weibo_id" + '\t' + "Company" + '\t' + "Content" + '\t' +
"Time"
'\t' + "Comment" + '\t' + 'Like' + '\t' + "Transfer" +
'\n')
i = 1
for item in item_data:
f.write("weibo_" + str(i) + '\t' + item["nickname"] + '\t' +
item["Post"] + '\t' + item["Pubtime"] + '\t' +
str(item["Comment_num"]) + '\t' +
str(item["Like_num"]) + '\t' +
str(item["Transfer_num"]) + '\n')
i += 1
return i
if __name__ == "__main__":
conn = MongoDB()
items = conn.find_item()
conn.write_file(item_data=items)
corpora = datapreprocess.read_file(
"../Data/pre_process/corpora_clean_data.csv")
print(corpora["Company"].unique())
company_list = [
'中兴通讯', '华为中国区', 'vivo智能手机', 'coolpad官方微博', 'OPPO', '联想', '小米公司',
'金立智能手机', "小辣椒", "360手机", 'HTC官方微博', '魅族科技', '天语手机', '超级手机官微', '一加手机',
'朵唯女性手机', '锤子科技', 'TCL通讯中国', 'nubia智能手机', '天语手机'
]