"""

convert data as indexes using word2index dicts.

:param traning_data_path:

:param vocab_word2index:

:param vocab_label2index:

:return:

"""

cache_data_dir = 'cache' + "_" + name_scope # path to save cache

cache_file =cache_data_dir+"/"+'train_valid_test.pik'

print("cache_path:",cache_file,"train_valid_test_file_exists:",os.path.exists(cache_file))

if os.path.exists(cache_file):

with open(cache_file, 'rb') as data_f:

print("going to load cache file from file system and return")

return pickle.load(data_f)

csvfile = open(traning_data_path, 'r')

spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|')

label_size=len(vocab_label2index)

X1_ = []

X2_ = []

Y_ = []

tfidf_source_file = './data/atec_nl_sim_train.txt'

tfidf_target_file = './data/atec_nl_sim_tfidf.txt'

if not os.path.exists(tfidf_target_file):

get_tfidf_score_and_save(tfidf_source_file,tfidf_target_file)

BLUE_SCORES_=[]

word_vec_fasttext_dict=load_word_vec('data/fasttext_fin_model_50.vec') #word embedding from fasttxt

word_vec_word2vec_dict = load_word_vec('data/word2vec.txt') #word embedding from word2vec

#word2vec.word2vec('/Users/test/PycharmProjects/question_answering_similarity/data/atec_additional_cropus.txt',

# '/Users/test/PycharmProjects/question_answering_similarity/data/word2vec_fin.bin', size=50, verbose=True,kind='txt')

#print("word_vec_word2vec_dict:",word_vec_word2vec_dict)

tfidf_dict=load_tfidf_dict('data/atec_nl_sim_tfidf.txt')

for i, row in enumerate(spamreader):##row:['\ufeff1', '\ufeff怎么更改花呗手机号码', '我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号', '1']

x1_list=token_string_as_list(row[1],tokenize_style=tokenize_style)

x1 = [vocab_word2index.get(x, UNK_ID) for x in x1_list]

x2_list=token_string_as_list(row[2],tokenize_style=tokenize_style)

x2 = [vocab_word2index.get(x, UNK_ID) for x in x2_list]

#add blue score features 2018-05-06

features_vector=data_mining_features(i,row[1], row[2],vocab_word2index,word_vec_fasttext_dict,word_vec_word2vec_dict,tfidf_dict, n_gram=8)

features_vector=[float(x) for x in features_vector]

BLUE_SCORES_.append(features_vector)

y_=row[3]

y=vocab_label2index[y_]

X1_.append(x1)

X2_.append(x2)

Y_.append(y)

if i==0 or i==1 or i==2:

print(i,"row[1]:",row[1],";x1:");print(row[1].decode("utf-8"))

print(i,"row[2]:", row[2], ";x2:");print(row[2].decode("utf-8"))

print(i,"row[3]:", row[3], ";y:", str(y))

print(i,"row[4].feature vectors:",features_vector)

number_examples = len(Y_)

#shuffle

X1=[]

X2=[]

Y=[]

BLUE_SCORES=[]

permutation = np.random.permutation(number_examples)

for index in permutation:

X1.append(X1_[index])

X2.append(X2_[index])

Y.append(Y_[index])

BLUE_SCORES.append(BLUE_SCORES_[index])

X1 = pad_sequences(X1, maxlen=sentence_len, value=0.) # padding to max length

X2 = pad_sequences(X2, maxlen=sentence_len, value=0.) # padding to max length

valid_number=min(3200,int((1-training_portion)*number_examples)) #1600

test_number=800

training_number=number_examples-valid_number-test_number

valid_end=training_number+valid_number

print(";training_number:",training_number,"valid_number:",valid_number,";test_number:",test_number)

#generate more training data, while still keep data distribution for valid and test.

X1_final, X2_final, BLUE_SCORE_final,Y_final,training_number_big=get_training_data(X1[0:training_number], X2[0:training_number], BLUE_SCORES[0:training_number],Y[0:training_number], training_number)

train = (X1_final,X2_final, BLUE_SCORE_final,Y_final)

valid = (X1[training_number+ 1:valid_end],X2[training_number+ 1:valid_end],BLUE_SCORES[training_number + 1:valid_end],Y[training_number + 1:valid_end])

test=(X1[valid_end+1:],X2[valid_end:],BLUE_SCORES[valid_end:],Y[valid_end:])

true_label_numbers=len([y for y in Y if y==1])

true_label_pert=float(true_label_numbers)/float(number_examples)

#save train/valid/test/true_label_pert to file system as cache

# save to file system if vocabulary of words not exists(pickle).

if not os.path.exists(cache_file):

with open(cache_file, 'ab') as data_f:

print("going to dump train/valid/test data to file sytem.")

pickle.dump((train,valid,test,true_label_pert),data_f)

"""

create vocabulary

:param training_data_path:

:param vocab_size:

:param name_scope:

:return:

"""

cache_vocabulary_label_pik='cache'+"_"+name_scope # path to save cache

if not os.path.isdir(cache_vocabulary_label_pik): # create folder if not exists.

os.makedirs(cache_vocabulary_label_pik)

# if cache exists. load it; otherwise create it.

cache_path =cache_vocabulary_label_pik+"/"+'vocab_label.pik'

print("cache_path:",cache_path,"file_exists:",os.path.exists(cache_path))

if os.path.exists(cache_path):

with open(cache_path, 'rb') as data_f:

return pickle.load(data_f)

else:

vocabulary_word2index={}

vocabulary_index2word={}

vocabulary_word2index[_PAD]=PAD_ID

vocabulary_index2word[PAD_ID]=_PAD

vocabulary_word2index[_UNK]=UNK_ID

vocabulary_index2word[UNK_ID]=_UNK

vocabulary_label2index={'0':0,'1':1}

vocabulary_index2label={0:'0',1:'1'}

#1.load raw data

csvfile = open(training_data_path, 'r')

spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|')

#2.loop each line,put to counter

c_inputs=Counter()

c_labels=Counter()

for i,row in enumerate(spamreader):#row:['\ufeff1', '\ufeff怎么更改花呗手机号码', '我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号', '1']

string_list_1=token_string_as_list(row[1],tokenize_style=tokenize_style)

string_list_2 = token_string_as_list(row[2],tokenize_style=tokenize_style)

c_inputs.update(string_list_1)

c_inputs.update(string_list_2)

#return most frequency words

vocab_list=c_inputs.most_common(vocab_size)

#put those words to dict

for i,tuplee in enumerate(vocab_list):

word,_=tuplee

vocabulary_word2index[word]=i+2

vocabulary_index2word[i+2]=word

#save to file system if vocabulary of words not exists(pickle).

if not os.path.exists(cache_path):

with open(cache_path, 'ab') as data_f:

pickle.dump((vocabulary_word2index,vocabulary_index2word,vocabulary_label2index,vocabulary_index2label), data_f)

#save to file system as file(added. for predict purpose when only few package is supported in test env)

save_vocab_as_file(vocabulary_word2index,vocabulary_index2label,vocab_list,name_scope=name_scope)

#1.1save vocabulary_word2index

cache_vocab_label_pik = 'cache' + "_" + name_scope

vocab_word2index_object=open(cache_vocab_label_pik+'/'+'vocab_word2index.txt',mode='a')

for word,index in vocab_word2index.items():

vocab_word2index_object.write(word+splitter+str(index)+"\n")

vocab_word2index_object.close()

#1.2 save word and frequent

word_freq_object=open(cache_vocab_label_pik+'/'+'word_freq.txt',mode='a')

for tuplee in vocab_list:

word,count=tuplee

word_freq_object.write(word+"|||"+str(count)+"\n")

word_freq_object.close()

#2.vocabulary_index2label

vocab_index2label_object = open(cache_vocab_label_pik + '/' + 'vocab_index2label.txt',mode='a')

for index,label in vocab_index2label.items():

vocab_index2label_object.write(str(index)+splitter+str(label)+"\n")

# 1.form more training data by swap sentence1 and sentence2

X1_big = []

X2_big = []

BLUE_SCORE_big=[]

Y_big = []

X1_final = []

X2_final = []

BLUE_SCORE_final=[]

Y_final = []

for index in range(0, training_number):

X1_big.append(X1[index])

X2_big.append(X2[index])

BLUE_SCORE_big.append(BLUE_SCORES[index])

y_temp = Y[index]

Y_big.append(y_temp)

#a.swap sentence1 and sentence2

if str(y_temp) == TRUE_LABEL:

X1_big.append(X2[index])

X2_big.append(X1[index])

BLUE_SCORE_big.append(BLUE_SCORES[index])

Y_big.append(y_temp)

#b.random change location of words

if shuffle_word_flag:

for x in range(5):

x1=X1[index]

x2=X2[index]

x1_random=[x1[i] for i in range(len(x1))]

x2_random = [x2[i] for i in range(len(x2))]

random.shuffle(x1_random)

random.shuffle(x2_random)

X1_big.append(x1_random)

X2_big.append(x2_random)

BLUE_SCORE_big.append(BLUE_SCORES[index])

Y_big.append(Y[index])

# shuffle data

training_number_big = len(X1_big)

permutation2 = np.random.permutation(training_number_big)

for index in permutation2:

X1_final.append(X1_big[index])

X2_final.append(X2_big[index])

BLUE_SCORE_final.append(BLUE_SCORE_big[index])

Y_final.append(Y_big[index])

string=string.decode("utf-8")

string=string.replace("***","*")

length=len(string)

if tokenize_style=='char':

listt=[string[i] for i in range(length)]

elif tokenize_style=='word':

listt=jieba.lcut(string) #cut_all=True

elif tokenize_style=='pinyin':

string=" ".join(jieba.lcut(string))

listt = ''.join(lazy_pinyin(string)).split() #list:['nihao', 'wo', 'de', 'pengyou']

listt=[x for x in listt if x.strip()]

"""

get data mining feature given two sentences as string.

1)n-gram similiarity(blue score);

2) get length of questions, difference of length

3) how many words are same, how many words are unique

4) question 1,2 start with how/why/when(为什么，怎么，如何，为何）

5）edit distance

6) cos similiarity using bag of words

:param input_string_x1:

:param input_string_x2:

:return:

"""

input_string_x1=input_string_x1.decode("utf-8")

input_string_x2 = input_string_x2.decode("utf-8")

#1. get blue score vector

feature_list=[]

#get blue score with n-gram

for i in range(n_gram):

x1_list=split_string_as_list_by_ngram(input_string_x1,i+1)

x2_list = split_string_as_list_by_ngram(input_string_x2, i + 1)

blue_score_i_1 = compute_blue_ngram(x1_list,x2_list)

blue_score_i_2 = compute_blue_ngram(x2_list,x1_list)

feature_list.append(blue_score_i_1)

feature_list.append(blue_score_i_2)

#2. get length of questions, difference of length

length1=float(len(input_string_x1))

length2=float(len(input_string_x2))

length_diff=(float(abs(length1-length2)))/((length1+length2)/2.0)

feature_list.append(length_diff)

#3. how many words are same, how many words are unique

sentence_diff_overlap_features_list=get_sentence_diff_overlap_pert(index,input_string_x1,input_string_x2)

feature_list.extend(sentence_diff_overlap_features_list)

#4. question 1,2 start with how/why/when(为什么，怎么，如何，为何）

#how_why_feature_list=get_special_start_token(input_string_x1,input_string_x2,special_start_token)

#print("how_why_feature_list:",how_why_feature_list)

#feature_list.extend(how_why_feature_list)

#5.edit distance

edit_distance=float(edit(input_string_x1, input_string_x2))/30.0

feature_list.append(edit_distance)

#6.cos distance from sentence embedding

x1_list=token_string_as_list(input_string_x1, tokenize_style='word')

x2_list = token_string_as_list(input_string_x2, tokenize_style='word')

distance_list_fasttext = cos_distance_bag_tfidf(x1_list, x2_list, word_vec_fasttext_dict, tfidf_dict)

distance_list_word2vec = cos_distance_bag_tfidf(x1_list, x2_list, word_vec_word2vec_dict, tfidf_dict)

#distance_list2 = cos_distance_bag_tfidf(x1_list, x2_list, word_vec_fasttext_dict, tfidf_dict,tfidf_flag=False)

#sentence_diffence=np.abs(np.subtract(sentence_vec_1,sentence_vec_2))

#sentence_multiply=np.multiply(sentence_vec_1,sentence_vec_2)

feature_list.extend(distance_list_fasttext)

feature_list.extend(distance_list_word2vec)

#feature_list.extend(list(sentence_diffence))

#feature_list.extend(list(sentence_multiply))

source_object = open(file_path, 'r')

word_vec_dict={}

for i,line in enumerate(source_object):

if i==0 and 'word2vec' in file_path:

continue

line=line.strip()

line_list=line.split()

word=line_list[0].decode("utf-8")

vec_list=[float(x) for x in line_list[1:]]

word_vec_dict[word]=np.array(vec_list)

#print("word_vec_dict:",word_vec_dict)

source_object = open(file_path, 'r')

tfidf_dict={}

for line in source_object:

word,tfidf_value=line.strip().split(splitter)

word=word.decode("utf-8")

tfidf_dict[word]=float(tfidf_value)

#print("tfidf_dict:",tfidf_dict)

feature_list1=[0.0 for i in range(len(special_token_list))]

feature_list2=[0.0 for i in range(len(special_token_list))]

for i,speical_token in enumerate(special_token_list):

if input_string_x1.find(speical_token)>0: #speical_token in input_string_x1:

feature_list1[i]=1.0

if input_string_x2.find(speical_token)>0:

feature_list2[i]=1.0

feature_list1.extend(feature_list2)

#0. get list from string

input_list1=[input_string_x1[token] for token in range(len(input_string_x1)) if input_string_x1[token].strip()]

input_list2 = [input_string_x2[token] for token in range(len(input_string_x2)) if input_string_x2[token].strip()]

length1=len(input_list1)

length2=len(input_list2)

num_same=0

same_word_list=[]

#1.compute percentage of same tokens

for word1 in input_list1:

for word2 in input_list2:

if word1==word2:

num_same=num_same+1

same_word_list.append(word1)

continue

num_same_pert_min=float(num_same)/float(max(length1,length2))

num_same_pert_max = float(num_same) / float(min(length1, length2))

num_same_pert_avg = float(num_same) / (float(length1+length2)/2.0)

#2.compute percentage of unique tokens in each string

input_list1_unique=set([x for x in input_list1 if x not in same_word_list])

input_list2_unique = set([x for x in input_list2 if x not in same_word_list])

num_diff_x1=float(len(input_list1_unique))/float(length1)

num_diff_x2= float(len(input_list2_unique)) / float(length2)

if index==0:#print debug message

print("input_string_x1:",input_string_x1)

print("input_string_x2:",input_string_x2)

print("same_word_list:",same_word_list)

print("input_list1_unique:",input_list1_unique)

print("input_list2_unique:",input_list2_unique)

print("num_same:",num_same,";length1:",length1,";length2:",length2,";num_same_pert_min:",num_same_pert_min,

";num_same_pert_max:",num_same_pert_max,";num_same_pert_avg:",num_same_pert_avg,

";num_diff_x1:",num_diff_x1,";num_diff_x2:",num_diff_x2)

diff_overlap_list=[num_same_pert_min,num_same_pert_max, num_same_pert_avg,num_diff_x1, num_diff_x2]

#print("input_string0:",input_string)

input_string="".join([string for string in input_string if string.strip()])

#print("input_string1:",input_string)

length = len(input_string)

result_string=[]

for i in range(length):

if i + ngram_value < length + 1:

result_string.append(input_string[i:i+ngram_value])

#print("ngram:",ngram_value,"result_string:",result_string)

"""

compute blue score use ngram information. x1_list as predict sentence,x2_list as target sentence

:param x1_list:

:param x2_list:

:return:

"""

count_dict={}

count_dict_clip={}

#1. count for each token at predict sentence side.

for token in x1_list:

if token not in count_dict:

count_dict[token]=1

else:

count_dict[token]=count_dict[token]+1

count=np.sum([value for key,value in count_dict.items()])

#2.count for tokens existing in predict sentence for target sentence side.

for token in x2_list:

if token in count_dict:

if token not in count_dict_clip:

count_dict_clip[token]=1

else:

count_dict_clip[token]=count_dict_clip[token]+1

#3. clip value to ceiling value for that token

count_dict_clip={key:(value if value<=count_dict[key] else count_dict[key]) for key,value in count_dict_clip.items()}

count_clip=np.sum([value for key,value in count_dict_clip.items()])

result=float(count_clip)/(float(count)+0.00000001)

matrix = [[i + j for j in range(len(str2) + 1)] for i in range(len(str1) + 1)]

#print("matrix:",matrix)

for i in xrange(1, len(str1) + 1):

for j in xrange(1, len(str2) + 1):

if str1[i - 1] == str2[j - 1]:

d = 0

else:

d = 1

matrix[i][j] = min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + d)