import pandas as pd
from sklearn.model_selection import train_test_split
from dependency_parse import get_pos_dep
from semantic_similarity import align
from features import percentage_semantic_similarity_both
data = pd.read_csv("input/train.csv")
X = data[['question1', 'question2']].values
y = data['is_duplicate'].values
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=42)
X_sample = X_train[:10]
for x in X_sample:
print x[0]
print x[1]
S = get_pos_dep(x[0])
T = get_pos_dep(x[1])
A, info_A = align(S, T)
p = percentage_semantic_similarity_both(S, T, A)
print 'Percentage Semantic Similarity: ', p
print '\n'
from semantic_similarity import align
from features import *
import gzip
len_test = pd.read_csv('../../input/test.csv').shape[0]
with gzip.open('stanford_corenlp_test.nlp', 'rb') as handle:
rows = []
for i in range(len_test):
try:
d = pickle.load(handle)
id = d['id']
# print(id)
#print(d['q1']['raw'])
#print(d['q2']['raw'])
S = get_pos_dep(d['q1']['toks'], d['q1']['deps'])
T = get_pos_dep(d['q2']['toks'], d['q2']['deps'])
#print(S)
#print(T)
A = align(S, T)
#print(A)
#print('Len(S):',len(S))
#print('Len(T):', len(T))
#print('Len(A):', len(A))
# Semantic Similarity Features
S_sem_sim = percentage_semantic_similarity_one(S, A)
T_sem_sim = percentage_semantic_similarity_one(T, A)
sem_sim = percentage_semantic_similarity_both(S, T, A)
# Noun Features
S_unmatch_n, T_unmatch_n = number_unmatched(S,
T,
def train_features():
with gzip.open('data_dump/stanfordData_train_ner.nlp', 'rb') as handle:
rows = []
while True:
try:
d = pickle.load(handle)
id = d['id']
print(id)
is_duplicate = d['is_duplicate']
#print(d['q1']['raw'])
#print(d['q2']['raw'])
S = get_pos_dep(d['q1']['toks'], d['q1']['deps'])
T = get_pos_dep(d['q2']['toks'], d['q2']['deps'])
#print(S)
#print(T)
A = align(S, T)
#print(A)
#print('Len(S):',len(S))
#print('Len(T):', len(T))
#print('Len(A):', len(A))
# Semantic Similarity Features
S_sem_sim = percentage_semantic_similarity_one(S, A)
T_sem_sim = percentage_semantic_similarity_one(T, A)
sem_sim = percentage_semantic_similarity_both(S, T, A)
# Noun Features
S_unmatch_n, T_unmatch_n = number_unmatched(S,
T,
A,
'n',
inferred_pos=True)
S_unmatch_n_p, T_unmatch_n_p = percent_unmatched(
S, T, A, 'n', inferred_pos=True)
# Adjective Features
S_unmatch_a, T_unmatch_a = number_unmatched(S,
T,
A,
'a',
inferred_pos=True)
S_unmatch_a_p, T_unmatch_a_p = percent_unmatched(
S, T, A, 'a', inferred_pos=True)
# Verb Features
S_unmatch_v, T_unmatch_v = number_unmatched(S,
T,
A,
'v',
inferred_pos=True)
S_unmatch_v_p, T_unmatch_v_p = percent_unmatched(
S, T, A, 'v', inferred_pos=True)
# Personal Pronoun Feature
S_unmatch_pp, T_unmatch_pp = number_unmatched(
S, T, A, 'PRP', inferred_pos=False)
S_unmatch_pp_p, T_unmatch_pp_p = percent_unmatched(
S, T, A, 'PRP', inferred_pos=False)
# WH-Pronoun Feature
S_unmatch_wp, T_unmatch_wp = number_unmatched(
S, T, A, 'WP', inferred_pos=False)
S_unmatch_wp_p, T_unmatch_wp_p = percent_unmatched(
S, T, A, 'WP', inferred_pos=False)
# Numbers Feature
S_unmatch_num, T_unmatch_num = number_unmatched(
S, T, A, 'CD', inferred_pos=False)
S_unmatch_num_p, T_unmatch_num_p = percent_unmatched(
S, T, A, 'CD', inferred_pos=False)
# NER Feature
S_unmatch_ner, T_unmatch_ner = ner_unmatched(S, T)
# Length Difference Feature
len_dif = len_difference(S, T)
len_dif_p = len_difference_p(S, T)
# Put all features in a row
features_row = [
id, S_sem_sim, T_sem_sim, sem_sim, S_unmatch_n,
T_unmatch_n, S_unmatch_n_p, T_unmatch_n_p, S_unmatch_a,
T_unmatch_a, S_unmatch_a_p, T_unmatch_a_p, S_unmatch_v,
T_unmatch_v, S_unmatch_v_p, T_unmatch_v_p, S_unmatch_pp,
T_unmatch_pp, S_unmatch_pp_p, T_unmatch_pp_p, S_unmatch_wp,
T_unmatch_wp, S_unmatch_wp_p, T_unmatch_wp_p,
S_unmatch_num, T_unmatch_num, S_unmatch_num_p,
T_unmatch_num_p, S_unmatch_ner, T_unmatch_ner, len_dif,
len_dif_p, is_duplicate
]
rows.append(features_row)
except EOFError:
break
columns = [
'id', 'S_sem_sim', 'T_sem_sim', 'sem_sim', 'S_unmatch_n',
'T_unmatch_n', 'S_unmatch_n_p', 'T_unmatch_n_p', 'S_unmatch_a',
'T_unmatch_a', 'S_unmatch_a_p', 'T_unmatch_a_p', 'S_unmatch_v',
'T_unmatch_v', 'S_unmatch_v_p', 'T_unmatch_v_p', 'S_unmatch_pp',
'T_unmatch_pp', 'S_unmatch_pp_p', 'T_unmatch_pp_p', 'S_unmatch_wp',
'T_unmatch_wp', 'S_unmatch_wp_p', 'T_unmatch_wp_p',
'S_unmatch_num', 'T_unmatch_num', 'S_unmatch_num_p',
'T_unmatch_num_p', 'S_unmatch_ner', 'T_unmatch_ner', 'len_dif',
'len_dif_p', 'is_duplicate'
]
df = pd.DataFrame(np.array(rows), columns=columns)
df.to_csv('Features/dean_train_features.csv', index=False)