def load_data(datasets):
train_data = [[],[]]
test_data = [[],[]]
valid_data = [[],[]]
train = 1000
test = 500
vectors = readVectors('bc3_vector_with_subject')
corpus = 'bc3/bc3corpus.1.0/corpus.xml'
annotation = 'bc3/bc3corpus.1.0/annotation.xml'
mails = parse_bc3(corpus,annotation)
assignVectors(mails,vectors)
count = 0
trainSet,validSet,testSet = divide_data(len(mails),0.6,0.2,mails)
def assignBinaryScore(dataset,output):
for i in dataset:
for j in i.text:
if j.score>=0.32 and j.score<0.65:
score = 1
elif j.score >= 0.65 and j.score<0.99:
score = 2
elif j.score == 0:
score = 0
else:
score = 3
subject = i.subject_feature[int(j.index.split('.')[0]) - 1]
thread = i.thread_feature[int(j.index.split('.')[0]) - 1]
output[0].append(j.vector + subject + thread)
output[1].append(score)
assignBinaryScore(trainSet,train_data)
assignBinaryScore(validSet,valid_data)
assignBinaryScore(testSet,test_data)
'''
for i in mails:
for j in i.text:
if j.score > 0:
score = 1
else:
score = 0
if count<train:
train_data[0].append(j.vector + i.vector)
train_data[1].append(score)
elif count>train and count<train+test:
test_data[0].append(j.vector + i.vector)
test_data[1].append(score)
else:
valid_data[0].append(j.vector + i.vector)
valid_data[1].append(score)
count += 1
'''
return (process_dataset(train_data,test_data,valid_data),len(train_data[0][0]),testSet)
def calculate_sentence_features(mails):
token_dict = {}
stemmer = PorterStemmer()
tfidf = tfidf_generize(mails,token_dict)
for i in mails:
for j in i.thread:
for k in j.sentences:
k.sentence_features = []
max_sum = 0
min_sum = 100
max_avg = 0
min_avg = 100
max_sub_sim = 0
min_sub_sim = 100
max_clue_score = 0
min_clue_score = 100
#clue_score_calculation(mails,repeat_words)
speech_act_score(mails)
for i in mails:
relative_position(i)
tmp_max,tmp_min = subject_sim(i)
if tmp_max > max_sub_sim:
max_sub_sim = tmp_max
if tmp_min < min_sub_sim:
min_sub_sim = tmp_min
message_number(i)
t_index = 0
for j in i.thread:
tmp_subject = i.subject[t_index]
subject_tfidf = numpy.array(tfidf.transform([tmp_subject]).todense())[0]
t_index += 1
m_rel_pos(j)
fol_quote(j)
for k in j.sentences:
length(k)
is_question(k)
k.tfidf = numpy.array(tfidf.transform([k.sentence]).todense())[0]
tfidf_sum(k)
'''
if k.tfidf_sum_score > max_sum:
max_sum = k.tfidf_sum_score
elif k.tfidf_sum_score <min_sum:
min_sum = k.tfidf_sum_score
'''
tfidf_avg(k)
#print tmp_subject
k.tfidf_subject_similarity_score = cosine_similarity(k.tfidf,subject_tfidf)
'''
if k.tfidf_avg_score > max_avg:
max_avg = k.tfidf_avg_score
elif k.tfidf_avg_score < min_avg:
min_avg = k.tfidf_avg_score
if k.clue_score > max_clue_score:
max_clue_score = k.clue_score
elif k.clue_score < min_clue_score:
min_clue_score = k.clue_score
'''
vectors = readVectors('bc3_vector_with_subject')
assignVectors(mails,vectors)
for i in mails:
#print len(i.thread)
#print len(i.thread_feature)
subjectivity_score(i)
t_index = 0
for j in i.thread:
#print len(i.thread)
question_similarity(j,paragraph_sim)
j.vector = i.thread_feature[t_index]
j.subject_vector = i.subject_feature[t_index]
t_index += 1
pre_sim(j,paragraph_sim)
#print t_index
for k in j.sentences:
for l in i.text:
if k.index == l.index:
k.vector = l.vector
k.subject_similarity_score = cosine_similarity(k.vector,j.subject_vector)
k.topic_similarity_score = cosine_similarity(k.vector,j.vector)
break
#input_x = []
#input_y = []
clue_score_calculation(mails,repeat_words,paragraph_sim)
sentiment_score(mails,'bc3_sentiment_vectors.txt')
for i in mails:
for j in i.thread:
for k in j.sentences:
#k.subject_sim_score = (k.subject_sim_score - min_sub_sim)/float(max_sub_sim-min_sub_sim)
#k.tfidf_sum_score = (k.tfidf_sum_score - min_sum) / float(max_sum - min_sum)
#k.tfidf_avg_score = (k.tfidf_avg_score - min_avg) / float(max_avg - min_avg)
k.unnor_sentence_features = [k.req,k.dlv,k.cmt,k.prop,k.meet,k.ddata,k.subjectivity_score,k.tfidf_subject_similarity_score,k.clue_score,k.subject_sim_score,k.message_number_score,k.fol_quote_score,k.m_rel_pos_score,k.is_question_score,k.tfidf_sum_score,k.tfidf_avg_score,k.length_score,k.relative_position_score]
#k.clue_score = (k.clue_score - min_clue_score) / float(max_clue_score - min_clue_score)
k.para_features = [k.subjectivity_score,k.qa_score,k.question_similarity_score,k.seq_sim_score,k.para_clue_score,k.fol_quote_score,k.is_question_score,k.tfidf_sum_score,k.tfidf_avg_score,k.subject_similarity_score,k.topic_similarity_score]
#print k.subject_sim_score
#k.sentence_features = [k.clue_score,k.subject_sim_score,k.message_number_score,k.fol_quote_score,k.m_rel_pos_score,k.is_question_score,k.tfidf_sum_score,k.tfidf_avg_score,k.length_score,k.relative_position_score]
#print k.sentence_features
#input_x.append(k.sentence_features)
#input_y.append(k.score)
unnor_label = {'subj':'real','tfidf_sim':'real','clue':'real','sub_sim':'real','m_num':'real','fol':'real','rel':'real','is_q':'real','tfidf_sum':'real','tfidf_avg':'real','leng':'real','rel_pos':'real','score':'real'}
para_label = {'sen':'real','subj':'real','qa':'real','qs':'real','seq':'real','clue':'real','fol':'real','is_q':'real','tfidf_sum':'real','tfidf_avg':'real','subj_sim':'real','topic_sim':'real','score':'real'}
'''
clue_score_calculation(mails,paragraph_sim)
for i in mails:
for j in i.thread:
for j in j.sentences:
k.para_features = [k.clue_score,k.message_number_score,k.fol_quote_score,k.m_rel_pos_score,k.is_question_score,k.tfidf_sum_score,k.tfidf_avg_score,k.length_score,k.relative_position_score,k.subject_similarity_score,k.topic_similarity_score]
'''
#input_x = numpy.asarray(input_x)
#input_y = numpy.asarray(input_y)
# basic feature extraction end
#clue_score
#clue_score end
'''
vectors = readVectors('bc3_vector_with_subject')
assignVectors(mails,vectors)
for i in mails:
#print len(i.thread)
#print len(i.thread_feature)
t_index = 0
for j in i.thread:
#print len(i.thread)
j.vector = i.thread_feature[t_index]
j.subject_vector = i.subject_feature[t_index]
t_index += 1
#print t_index
for k in j.sentences:
for l in i.text:
if k.index == l.index:
k.vector = l.vector
k.subject_similarity_score = cosine_similarity(k.vector,j.subject_vector)
k.topic_similarity_score = cosine_similarity(k.vector,j.vector)
k.para_features = k.sentence_features
k.para_features.append(k.subject_similarity_score)
k.para_features.append(k.topic_similarity_score)
break
'''
ori = 0.0
para = 0.0
o_ro = 0.0
p_ro = 0.0
rnn_rouge = 0.0
rnn_score = 0.0
for p in range(5):
train,valid,test = divide_data(len(mails),0.8,0,mails)
if p == 0:
tmp_rnn = rnn_summ(train,test,"rnn_hidden")
else:
tmp_rnn = rnn_summ(train,test)
tmp_rnn.init_rnn(0.01,0)
rate = 0.33
for j in range(6000):
if j % 100 == 0:
rate = rate * 0.9
error = tmp_rnn.rnn_train(rate)
if j % 100 == 0:
print error
if p==0:
tmp_rnn.close_file()
unnor_input_x = []
unnor_test_x = []
para_input_x = []
para_test_x = []
input_x = []
input_y = []
for i in train:
for j in i.thread:
for k in j.sentences:
input_x.append(k.sentence_features)
input_y.append(k.score)
unnor_input_x.append(k.unnor_sentence_features)
para_input_x.append(k.para_features)
test_x = []
test_y = []
for i in test:
for j in i.thread:
for k in j.sentences:
test_x.append(k.sentence_features)
unnor_test_x.append(k.unnor_sentence_features)
para_test_x.append(k.para_features)
test_y.append(k.score)
tmp_input = unnor_input_x + unnor_test_x
tmp_input = preprocessing.scale(tmp_input)
unnor_input_x = tmp_input[0:len(unnor_input_x)]
unnor_test_x = tmp_input[len(unnor_input_x):len(tmp_input)]
tmp_input = para_input_x + para_test_x
tmp_input = preprocessing.scale(tmp_input)
para_input_x = tmp_input[0:len(para_input_x)]
para_test_x = tmp_input[len(para_input_x):len(tmp_input)]
def eval(input_x,input_y,test_x,test,label,write_folder = None):
tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],
'C': [1, 10, 100, 1000]},
{'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]
grid_clf = GridSearchCV(SVR(C=1,epsilon=0.2), tuned_parameters)
grid_clf.fit(input_x,input_y)
print "params : \t"
print grid_clf.get_params()
result = grid_clf.predict(test_x)
#py_weka = python_weka(input_x,input_y,label)
#py_weka.train()
#result = py_weka.predict(test_x)
#py_weka.close()
#clf = SVR(C=1.0, epsilon=0.2)
#clf.fit(input_x,input_y)
#result = clf.predict(test_x)
score_index = 0
produce_set = []
for i in test:
produce_set.append([])
score_list = []
index_list = []
for j in i.thread:
for k in j.sentences:
k.predict_score = result[score_index]
score_index += 1
score_list.append(k.predict_score)
index_list.append(k.index)
sorted_index_array = sorted_index(score_list)
sen_length = 0
for j in range(len(index_list)):
if sen_length < float(len(index_list))*0.3:
produce_set[-1].append(index_list[sorted_index_array[len(index_list)-j-1]])
sen_length += 1
else:
break
score = weightRecall(test,produce_set,write_folder)
print score
rouge_eval = rouge(test,produce_set)
rouge_score = rouge_eval.eval()['rouge_l_f_score']
print rouge_score
return score,rouge_score
print "rnn:"
rnn_tmp_score,rnn_tmp_rouge = tmp_rnn.rnn_test()
'''
if p != 0:
rnn_tmp_score,rnn_tmp_rouge = tmp_rnn.rnn_test()
else:
rnn_tmp_score,rnn_tmp_rouge = tmp_rnn.rnn_test("rnn_folder")
'''
rnn_score += rnn_tmp_score
rnn_rouge += rnn_tmp_rouge
print "avg rnn:"
print rnn_score
print "avg rnn rouge:"
print rnn_rouge
print "\n"
print "ori:"
if p == 0:
ori_score,ori_rouge = eval(unnor_input_x,input_y,unnor_test_x,test,unnor_label,"ori_folder")
else:
ori_score,ori_rouge = eval(unnor_input_x,input_y,unnor_test_x,test,unnor_label)
ori += ori_score
o_ro += ori_rouge
print "avg_ori:"
print ori
print "avg_ori_rouge:"
print o_ro
print "\n"
print "para"
if p != 4:
para_score,para_rouge = eval(para_input_x,input_y,para_test_x,test,para_label)
else:
para_score,para_rouge = eval(para_input_x,input_y,para_test_x,test,para_label,"para_folder")
para += para_score
p_ro += para_rouge
print "avg_para:"
print para
print "avg_para_rouge:"
print p_ro
print "\n"