def get_accuracy(test_data):
predicted_stances = test_data['Stance'].as_matrix()
actual_data = data_instance('./competition_test_stances.csv',
'./test_bodies.csv')
actual_stances = actual_data.all_data_df['Stance'].as_matrix()
print(f'Accuracy: {np.mean(predicted_stances==actual_stances)}')
print(f'actual stances: {string_to_int_labels(actual_stances)}')
print(f'predicted stances: {string_to_int_labels(predicted_stances)}')
LABELS = ['agree', 'disagree', 'discuss', 'unrelated']
score.report_score(
[LABELS[e] for e in string_to_int_labels(actual_stances)],
[LABELS[e] for e in string_to_int_labels(predicted_stances)])
def grid_search(X_train, X_test, y_train, y_test, estimator, param_grid):
clf = model_selection.GridSearchCV(estimator=estimator,
param_grid=param_grid,
scoring=scorer,
cv=5)
clf.fit(X_train, y_train)
#print(clf.best_score_)
#print(clf.best_params_)
#print(clf.cv_results_)
test_clf = clf.best_estimator_
test_clf.fit(X_train, y_train)
#print(test_clf)
pred = test_clf.predict(X_test)
temp = score.report_score(y_test, pred)
#temp = np.argwhere(y_test==0)
#temp1 = np.argwhere(y_test==1)
#ix1 = np.in1d(y_test.ravel(), 1).reshape(y_test.shape)
#ix0 = np.in1d(y_test.ravel(), 0).reshape(y_test.shape)
#print(np.array(X_test[ix0]['A']))
#print(X_test[np.array(temp1)])
#print(temp)
#plotDB(test_clf, X_test[ix0]['A'], X_test[ix0]['B'], X_test[ix1]['A'], X_test[ix1]['B'], "Decision Tree")
#print(estimator, clf.best_score_)
return test_clf, test_clf.score(X_train,
y_train), test_clf.score(X_test,
y_test), temp
def get_performance(predicted, truth, n_classes=None, outputStyle='dict'):
# Predicted and observed are both integer vectors of class label
# Cast both predicted and observed to numpy integer
predicted = np.asarray(predicted, dtype=np.int64)
truth = np.asarray(truth, dtype=np.int64)
assert len(predicted) == len(truth)
# Compute competition score:
competition_score = report_score(
[LABELS[e] for e in truth],
[LABELS[e] for e in predicted]) #scorer(predicted, truth)
output = []
# If n_classes is unknown, infer from the labels
if n_classes is None:
n_classes = len(np.unique(predicted.extend(truth)))
for i in range(n_classes):
# Get 2-way table
tp = sum((predicted == i) & (truth == i))
tn = sum((predicted != i) & (truth != i))
fp = sum((predicted == i) & (truth != i))
fn = sum((predicted != i) & (truth == i))
print('tp ' + str(tp))
print('tn ' + str(tn))
print('fp ' + str(fp))
print('fn ' + str(fn))
# Compute performance metrics
recall = tp / (tp + fn) # aka sensitivity
print('recall ' + str(recall))
precision = tp / (tp + fp) # aka ppv
print('precision ' + str(precision))
specificity = tn / (tn + fp)
print('specificity ' + str(specificity))
f1 = 2 * tp / (2 * tp + fp + fn)
print('f1 ' + str(f1))
accuracy = (tp + tn) / len(truth)
keys = [
'tp', 'tn', 'fp', 'fn', 'recall', 'precision', 'specificity', 'f1',
'accuracy', 'competition'
]
values = [
tp, tn, fp, fn, recall, precision, specificity, f1, accuracy,
competition_score
]
output.append(dict(zip(keys, values)))
return output
def train_and_test():
result = pd.DataFrame()
f = open(F_PKL, 'rb')
testX_all = pickle.load(f)
f.close()
lr = RandomForestClassifier(n_estimators=70,
min_samples_split=100,
min_samples_leaf=20,
max_depth=8,
max_features='sqrt',
random_state=10)
lr.fit(trainX_all, trainy_all)
y_pred_binary = lr.predict(testX_all)
y_pred_binary = list((np.array(y_pred_binary) - 1) * (-1))
result['binary'] = y_pred_binary
stage2 = result[result['binary'] == 1].index.tolist()
testX = []
for i in stage2:
textX.append(testX_all[i])
gb = GradientBoostingClassifier(n_estimators=1000,
learning_rate=0.1,
min_samples_split=300,
max_features='sqrt',
subsample=0.8,
random_state=10)
gb.fit(trainX, trainY)
y_pred = list(gb.predict(testX))
Stance = {0: 'unrelated', 1: 'discuss', 2: 'agree', 3: 'disagree'}
pred = []
for i in range(len(y_pred_binary)):
if y_pred_binary[i] == 0:
pred.append('unrelated')
else:
pred.append(Stance[y_pred.pop(0)])
dataframe = pd.read_hdf(F_H5)
actual = list(dataframe['Stance'])
report_score(actual, pred)
return pred
def on_epoch_end(self, epoch, logs={}):
test_outputs = []
test_predictions = []
labels = ['unrelated', 'agree', 'disagree', 'discuss']
for target in self.Y:
test_outputs += [labels[target.tolist().index(1)]]
aux = self.model.predict([self.X1, self.X2, self.overlapFeatures_fnc, self.refutingFeatures_fnc, self.polarityFeatures_fnc,\
self.handFeatures_fnc, self.cosFeatures, self.cosFeatures_two, self.bleu_fnc, self.rouge_fnc, self.cider_fnc_test, \
self.X2_two_sentences, self.overlapFeatures_fnc_two, self.refutingFeatures_fnc_two, self.polarityFeatures_fnc_two, self.handFeatures_fnc_two,\
self.bleu_two_sentences, self.rouge_two_sentences, self.cider_two_test, self.talos_counts_test, self.talos_tfidfsim_test, \
self.talos_svdHeadline_test, self.talos_svdBody_test , self.talos_svdsim_test, self.talos_w2vHeadline_test , self.talos_w2vBody_test, \
self.talos_w2vsim_test ,self.talos_sentiHeadline_test , self.talos_sentiBody_test], batch_size=64)
for prediction in aux:
test_predictions += [labels[prediction.argmax()]]
score = report_score(test_outputs, test_predictions, matrix=False)
def train():
data_x, data_y, body_ids, target_stance = build_data()
print(data_x, data_y, body_ids, target_stance)
# read test data
test_x, body_ids_test, true_y = build_test_data()
print(Counter(data_y))
# create the SVM model, generating pickle file
bst = svm_FNC.fit(data_x, data_y)
joblib.dump(bst, 'svm_cluster_train.pkl')
pred_y = bst.predict(test_x)
print(len(pred_y))
print("------------------------------------")
# add encoding and diplay evaluation
df_test = pd.DataFrame()
df_test['pred_y'] = pred_y
df_test['true_y'] = true_y
df_test['pred_y'] = df_test['pred_y'].replace([0, 1, 2],
['unknown', 'false', 'true'])
df_test['true_y'] = df_test['true_y'].replace([0, 1, 2],
['unknown', 'false', 'true'])
df_test = df_test.dropna()
print(df_test['pred_y'].value_counts())
print(df_test['true_y'].value_counts())
print(score.report_score(df_test['true_y'], df_test['pred_y']))
predicted = [LABELS[int(a)] for a in pred_y]
stances = target_stance
df_output = pd.DataFrame()
df_output['Headline'] = stances['Headline']
df_output['Body ID'] = stances['Body ID']
df_output['Stance'] = predicted
df_output.to_csv('tree_pred_prob_cor2.csv', index=False)
df_output[['Headline', 'Body ID', 'Stance']].to_csv('tree_pred_cor2.csv',
index=False)
print(df_output)
print(Counter((df_output['Stance'])))
def scorer(estimator, X, y):
y_pred = estimator.predict(X)
return score.report_score(y, y_pred) / 100.0
print("\t-Dev size:\t", len(dev_data))
print("\t-Test data:\t", len(test_data))
##############################################################################
# Feature extraction
print("[2] Extracting features.. ")
training_features, dev_features, test_features = extract_features(training_data, dev_data, test_data)
##############################################################################
# Fitting model
print("[3] Fitting model..")
print("\t-Logistic Regression")
#lr = LogisticRegression(C = 1.0, class_weight='balanced', solver="lbfgs", max_iter=150)
#lr = MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)
lr = RandomForestClassifier(n_estimators=10, random_state=12345)
targets_tr = [a['Stance'] for a in training_data]
targets_dev = [a['Stance'] for a in dev_data]
targets_test = [a['Stance'] for a in test_data]
y_pred = lr.fit(training_features, targets_tr).predict(test_features)
##############################################################################
# Evaluation
print("[4] Evaluating model..")
score.report_score(targets_test, y_pred)
voting_result = []
temp_count = 0
temp_res = ""
voting_result.append(final_prediction[i])
voting_result.append(y_pred_logistic[i])
voting_result.append(y_pred_random[i])
voting_result.append(y_pred_mnb[i])
if (voting_result.count("agree") > temp_count):
temp_count = voting_result.count("agree")
temp_res = "agree"
if (voting_result.count("disagree") > temp_count):
temp_count = voting_result.count("disagree")
temp_res = "disagree"
if (voting_result.count("discuss") > temp_count):
temp_count = voting_result.count("discuss")
temp_res = "discuss"
if (voting_result.count("unrelated") > temp_count):
temp_count = voting_result.count("unrelated")
temp_res = "unrelated"
final.append(temp_res)
#saving the predicted stances for submission purpose
np.savetxt("stance.csv", np.array(final), delimiter="\n", fmt='%s')
targets_test = [a['Stance'] for a in test_data]
# Evaluation
print("[4] Evaluating model..")
score.report_score(targets_test, final)
#Concatenation approach #final_pred=np.concatenate((np.matmul(test_prediction1,weight_pred_1),np.matmul(test_prediction2,weight_pred_2),np.matmul(test_prediction3,weight_pred_3)),axis=1) #Summation approach final_pred=np.matmul(test_prediction1,weight_pred_1)+np.matmul(test_prediction2,weight_pred_2)+np.matmul(test_prediction3,weight_pred_3) #=======no ensemble============ #final_pred=test_prediction1 #============================== final_pred_index = np.argmax(final_pred,1) save_predictions(base_dir,final_pred_index, file_predictions) # Calculate score golden_stance = pd.read_csv(base_dir+"/"+"test_stances_labeled.csv") prediction_stance=pd.read_csv(base_dir+"/"+"predictions_test.csv") competition_grade,agree_recall,disagree_recall,discuss_recall,unrelated_recall, agree_precision, disagree_precision, discuss_precision, unrelated_precision,all_recall, f1_agree, f1_disagree, f1_discuss, f1_unrelated, F1m=report_score(golden_stance['Stance'],prediction_stance['Prediction']) Grade.append(competition_grade) Agree_recall.append(agree_recall) Disagree_recall.append(disagree_recall) Discuss_recall.append(discuss_recall) Unrelated_recall.append(unrelated_recall) Agree_precision.append(agree_precision) Disagree_precision.append(disagree_precision) Discuss_precision.append(discuss_precision) Unrelated_precision.append(unrelated_precision) Recall.append(all_recall)
RandomForestClassifier(n_estimators=10),
MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True),
GradientBoostingClassifier(n_estimators=200,
random_state=14128,
verbose=True),
KNeighborsClassifier(4),
SVC(kernel="linear", C=0.025),
DecisionTreeClassifier(max_depth=5),
LogisticRegression(C=1e5)
]
# KNeighborsClassifier(4, algorithm='kd_tree'),
for n, clf in zip(names, classifiers):
print(n)
y_pred = clf.fit(train_features, train_labels).predict(validate_features)
print(score.report_score(test_labels, y_pred))
print('\n')
# ## Run Classifiers and Score Test Output
# This is how well we would have scored in the actual competition
names = [
"Random Forest", "Multinomial Naive Bayes", "Gradient Boosting",
"K Nearest Neighbors", "Linear SVM", "Decision Tree", "Logistic Regression"
]
classifiers = [
RandomForestClassifier(n_estimators=10),
MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True),
GradientBoostingClassifier(n_estimators=200,
random_state=14128,
def cv():
K_FOLD = 10
data = load_features().astype(float)
targets = load_targets()
# K-Fold and Score Tracking
scores = []
wscores = []
pscores = []
n_folds = 10
best_iters = [0] * n_folds
# kf = GroupKFold(n_splits=K_FOLD)
kf = StratifiedKFold(n_splits=K_FOLD)
print('Training Model...')
for fold, (train_idx,
test_idx) in enumerate(kf.split(data, targets['target'])):
print('\n[K = ' + str(fold + 1) + ']')
# Train Model
dtrain = xgb.DMatrix(data[train_idx], label=targets[train_idx])
dtest = xgb.DMatrix(data[test_idx])
watchlist = [(dtrain, 'train'), (dtest, 'eval')]
bst = xgb.train(
params_xgb,
dtrain,
num_round,
watchlist,
verbose_eval=10,
#feval = eval_metric,
#maximize = True,
early_stopping_rounds=80)
pred_prob_y = bst.predict(dtest).reshape(targets[test_idx].shape[0],
2) # predicted probabilities
pred_y = bst.predict(dtest, ntree_limit=bst.best_ntree_limit).reshape(
targets[test_idx].shape[0], 2)
print('predicted probabilities: ')
print(pred_y)
pred_y = np.argmax(pred_y, axis=1)
print('predicted label indices: ')
print(pred_y)
print('best iterations: ', bst.best_ntree_limit)
best_iters[fold] = bst.best_ntree_limit
print(pred_y)
print('pred_y.shape')
print(pred_y.shape)
print('y_valid.shape')
print(targets[test_idx].shape)
predicted = [LABELS[int(a)] for a in pred_y]
actual = [LABELS[int(a)] for a in targets[test_idx]]
s, _ = score_submission(actual, predicted)
s_perf, _ = score_submission(actual, actual)
r_score = report_score(actual, predicted)
wscore = float(s) / s_perf
print('report_score', r_score)
print('fold %s, score = %f, perfect_score %f, weighted percentage %f' %
(fold, s, s_perf, wscore))
scores.append(s)
pscores.append(s_perf)
wscores.append(wscore)
print('scores:')
print(scores)
print('mean score:')
print(np.mean(scores))
print('perfect scores:')
print(pscores)
print('mean perfect score:')
print(np.mean(pscores))
print('w scores:')
print(wscores)
print('mean w score:')
print(np.mean(wscores))
print('best iters:')
print(best_iters)
print('mean best_iter:')
m_best = np.mean(best_iters)
print(m_best)
print("training the model")
#clf = GridSearchCV(AdaBoostClassifier(), parameters, cv=10, verbose=2)
#clf.fit(train_features, y_train)
#print('Best Score: ', clf.best_score_)
#print('Best Params: ', clf.best_params_)
classifiers.fit(train_features, y_train)
print("Prediction on dev dataset Ada")
predicted = {}
predicted = classifiers.predict(val_features)
print(report_score(y_val, predicted))
print("Prediction on test dataset")
predicted = classifiers.predict(test_features)
competetion_unlabeled = pd.read_csv(
'C:\\Users\\jeswi\\competition_test_stances_unlabeled.csv')
Predicted = pd.DataFrame({'Stance': predicted})
result = pd.concat([competetion_unlabeled, Predicted], axis=1, sort=False)
result.to_csv('C:\\Users\\jeswi\\Desktop\\submission_LSTM_Glove.csv',
index=False,
encoding='utf-8')
print(report_score(test_labels, predicted))
print('ADA:')
# Test the model on the FNC Test Set #
test_outputs = []
test_predictions = []
labels = ['unrelated', 'agree', 'disagree', 'discuss']
for target in Y_test:
test_outputs += [labels[target.tolist().index(1)]]
aux = final_model.predict([X1_test, X2_test, overlapFeatures_fnc_test, refutingFeatures_fnc_test, polarityFeatures_fnc_test, handFeatures_fnc_test, \
cosFeatures_fnc_test, cosFeatures_two_test, bleu_fnc_test, rouge_fnc_test, cider_fnc_test, \
X2_test_two_sentences, overlapFeatures_fnc_two_test, refutingFeatures_fnc_two_test, polarityFeatures_fnc_two_test, handFeatures_fnc_two_test, \
bleu_two_sentences_test, rouge_two_sentences_test, cider_two_test, talos_counts_test, talos_tfidfsim_test, talos_svdHeadline_test, talos_svdBody_test, talos_svdsim_test, \
talos_w2vHeadline_test, talos_w2vBody_test, talos_w2vsim_test, talos_sentiHeadline_test, talos_sentiBody_test])
preds = []
for prediction in aux:
pred = prediction.argmax()
if pred == 0:
one_hot = [1, 0, 0, 0]
if pred == 1:
one_hot = [0, 1, 0, 0]
if pred == 2:
one_hot = [0, 0, 1, 0]
if pred == 3:
one_hot = [0, 0, 0, 1]
preds += [one_hot]
test_predictions += [labels[prediction.argmax()]]
report_score(test_outputs, test_predictions)
######################################
gc.collect()
mlp_model = load_model(os.path.join(models_dir, mlp_model_file))
if sentiment == 'yes':
test_features = np.concatenate(
(np.load('test.tfidf.npy'), np.load('test.sent.npy')), axis=1)
else:
test_features = np.load('test.tfidf.npy')
test_labels = np.load('test.labels.npy')
print(f"\n\nShape of test set (Inputs): {test_features.shape}")
print(f"Shape of test set (Labels): {test_labels.shape}\n\n")
# Prediction
test_predictions = mlp_model.predict(test_features)
test_predictions = np.argmax(test_predictions, axis=1)
test_predictions = [label_ref_rev[i] for i in test_predictions]
test_labels = [label_ref_rev[i] for i in test_labels]
print("Scores on test set")
report_score(test_labels, test_predictions)
print()
print()
print(classification_report(test_labels, test_predictions))
# Save predictions
save_predictions(test_predictions, file_predictions)
train_hand_features = np.array(train_hand_features)
test_hand_features = np.array(test_hand_features)
train_features = hstack(
[train_body_tfidf, train_headline_tfidf, train_hand_features])
test_features = hstack(
[test_body_tfidf, test_headline_tfidf, test_hand_features])
#Extract training and test labels
train_labels = list(train_df['Stance'])
test_labels = list(test_df['Stance'])
#Initialize random forest classifier (Scikit Learn)
rf_classifier = RandomForestClassifier(n_estimators=10)
y_pred = rf_classifier.fit(train_features, train_labels).predict(test_features)
score.report_score(test_labels, y_pred)
#Initialize multinomialnb classifier
nb_classifier = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
y_pred = nb_classifier.fit(train_features, train_labels).predict(test_features)
score.report_score(test_labels, y_pred)
#Add predicted labels to test dataframe
test_df['RF_Predicted_Stance'] = list(y_pred)
test_df2 = test_df[['Headline', 'Body_Text', 'RF_Predicted_Stance', 'Stance']]
test_df2[test_df2['RF_Predicted_Stance'] == 'unrelated']
#Initialize random forest classifier (Scikit Learn)
rf_classifier = RandomForestClassifier(n_estimators=10)
y_pred = rf_classifier.fit(train_features, train_labels).predict(test_features)
score.report_score(test_labels, y_pred)
else:
training_on = training_set_stance
testing_on = dev_set_stance
training_set = pd.merge(training_on, bodies, how='inner', left_on='Body ID', right_index=True, sort=True, suffixes=('_x', '_y'), copy=True, indicator=True)
test_set = pd.merge(testing_on, bodies, how='inner', left_on='Body ID', right_index=True, sort=True, suffixes=('_x', '_y'), copy=True, indicator=True)
classifier.fit(training_set.drop('Stance', axis = 1), training_set['Stance'])
#print(classifier.best_params_)
print("getting predicted labels...")
hypotheses = classifier.predict(test_set)
#hypotheses = ['unrelated'] * len(test_set)
c_r = classification_report(test_set['Stance'], hypotheses)
print(c_r)
print(accuracy_score(test_set['Stance'], hypotheses))
print(report_score(test_set['Stance'], hypotheses))
splits= c_r.split('\n')[2:6]
fscore_mean = np.mean([float(x.strip().split()[3]) for x in splits])
print('fscore mean')
print(fscore_mean)
#
#now = datetime.now()
#date_parts = [str(x) for x in [now.month,now.day,now.year]]
#time_parts = [str(x) for x in [now.hour,now.minute,now.second]]
#now_str = ''
#joblib.dump(classifier, 'classifiers/classifier' + now_str + '.pkl')
#now_str = "-".join(date_parts) + "_" + "-".join(time_parts)
y = torch.tensor(label, dtype=torch.long, device=device)
x_similarity = torch.tensor(sim,
dtype=torch.float,
device=device)
x_entailment = torch.stack(entail, dim=0).to(device)
outputs = model(x_p, x_h, x_similarity, x_entailment)
out.append(outputs)
y_label.append(y)
outputs = torch.cat(out, dim=0)
y = torch.cat(y_label)
loss = criterion(outputs, y)
accuracy = evaluation(outputs, y)
print("Dev | Epoch:{0} | Loss:{1} | Accuracy:{2}".format(
epoch + 1, loss, accuracy))
report_F1(outputs, y)
report_score(outputs, y)
# torch.save(model.state_dict(), 'runs/{0}/parameters_{1}'.format(argvs[1], epoch+1))
'''
# test
model.eval()
loss_sum = 0
accuracy_sum = 0
with torch.no_grad():
for premise, hypothesis, label in batcher(token_data["test"], test_batch):
x_p = torch.tensor(premise, dtype=torch.long, device=device)
x_h = torch.tensor(hypothesis, dtype=torch.long, device=device)
y = torch.tensor(label, dtype=torch.long, device=device)
outputs = model(x_p, x_h)
loss = criterion(outputs, y)
accuracy = evaluation(outputs, y)
loss_sum += loss
#final_pred=np.matmul(test_prediction1,weight_pred_1)+np.matmul(test_prediction2,weight_pred_2)
#=======ensemble for three=========
#final_pred=np.concatenate((np.matmul(test_prediction1,weight_pred_1),np.matmul(test_prediction2,weight_pred_2),np.matmul(test_prediction3,weight_pred_3)),axis=1)
#final_pred=np.matmul(test_prediction1,weight_pred_1)+np.matmul(test_prediction2,weight_pred_2)+np.matmul(test_prediction3,weight_pred_3)
#=======no ensemble ========
final_pred = test_prediction1
#====================================
final_pred_index = np.argmax(final_pred, 1)
save_predictions(base_dir, final_pred_index, file_predictions)
# =========Calculate score=================================================
golden_stance = pd.read_csv(base_dir + "/" +
"test_stances_labeled.csv")
prediction_stance = pd.read_csv(base_dir + "/" +
"predictions_test.csv")
competition_grade, agree_recall, disagree_recall, discuss_recall, unrelated_recall, all_recall = report_score(
golden_stance['Stance'], prediction_stance['Prediction'])
Grade.append(competition_grade)
Agree.append(agree_recall)
Disagree.append(disagree_recall)
Discuss.append(discuss_recall)
Unrelated.append(unrelated_recall)
Recall.append(all_recall)
print('Grade', Grade)
print('Agree', Agree)
print('Disagree', Disagree)
print('Discuss', Discuss)
print('Unrelated', Unrelated)
print('All Recall', Recall)
# import packages
import pandas as pd
import score
from sklearn.metrics import f1_score
# read the data
### predicted y
predict_df = pd.read_csv("competition_test_stances.csv", encoding='iso-8859-1')
### true y
origin_df = pd.read_csv("tree_pred_cor2.csv", encoding='iso-8859-1')
# confusion matrix and official score
score.report_score(origin_df['Stance'], predict_df['Stance'])
# f1_score original
macro_score = f1_score(origin_df['Stance'],
predict_df['Stance'],
average='macro')
print("Macro score: " + str(macro_score))
# f1_score weighted
weighted_score = f1_score(origin_df['Stance'],
predict_df['Stance'],
average='weighted')
print("Weighted score: " + str(weighted_score))
clf.fit(X_train, y_train)
predicted = [LABELS[int(a)] for a in clf.predict(X_test)]
actual = [LABELS[int(a)] for a in y_test]
fold_score, _ = score_submission(actual, predicted)
max_fold_score, _ = score_submission(actual, actual)
score = fold_score / max_fold_score
print("Score for fold " + str(fold) + " was - " + str(score))
if score > best_score:
best_score = score
best_fold = clf
#Run on Holdout set and report the final score on the holdout set
predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS[int(a)] for a in y_holdout]
print("Scores on the dev set")
report_score(actual, predicted)
print("")
print("")
#Run on competition dataset
predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]
actual = [LABELS[int(a)] for a in y_competition]
print("Scores on the test set")
report_score(actual, predicted)
def main():
MODEL_NUM = 3
if len(sys.argv) > 1:
MODEL_NUM = int(sys.argv[1])
print("[INFO] Processing training and test data")
headlines, articles, stances = processing_data("processed_data.csv")
headlines_test, articles_test, stances_test = processing_data(
"processed_test.csv")
print("[INFO] Fitting data into tokenizer")
headline_tokenizer = fitting_tokenizer(headlines, headlines_test,
'headline')
article_tokenizer = fitting_tokenizer(articles, articles_test, 'article')
print("[INFO] Padding data to achieve max length")
X_headline, X_article, y = padding(headlines, articles, stances,
headline_tokenizer, article_tokenizer)
X_headline_test, X_article_test, y_test = padding(headlines_test,
articles_test,
stances_test,
headline_tokenizer,
article_tokenizer)
print("[INFO] Spliting out validation data")
X_headline_train, X_headline_val, X_article_train, X_article_val, y_train, y_val = train_test_split(
X_headline, X_article, y, random_state=10, test_size=0.1)
print("[INFO] Loading GloVe embedding from file")
embeddings = dict()
with open(ROOT_PATH + 'glove.6B.300d.txt') as f:
for line in f:
tokens = line.strip().split(" ")
embeddings[tokens[0]] = np.array(tokens[1:], dtype='float32')
print("[INFO] Building embedding matrix for HEADLINE and ARTICLE")
headline_embeddings_matrix = building_embedding_matrix(
headline_tokenizer, embeddings)
article_embeddings_matrix = building_embedding_matrix(
article_tokenizer, embeddings)
del embeddings
print("[INFO] Building neural network model")
headline_input = Input(shape=(MAX_HEADLINE_LEN, ), name='headline_input')
headline_embedding = Embedding(
output_dim=EMBEDDING_DIM,
input_dim=len(headline_tokenizer.word_index) + 1,
input_length=MAX_HEADLINE_LEN,
weights=[headline_embeddings_matrix],
trainable=False,
name='headline_embedding')
article_input = Input(shape=(MAX_ARTICLE_LEN, ), name='article_input')
article_embedding = Embedding(output_dim=EMBEDDING_DIM,
input_dim=len(article_tokenizer.word_index) +
1,
input_length=MAX_ARTICLE_LEN,
weights=[article_embeddings_matrix],
trainable=False,
name='article_embedding')
if MODEL_NUM == 1:
# ======= Model 1 ======
headline_lstm = LSTM(LSTM_DIM, name='headline_lstm')(
headline_embedding(headline_input))
article_lstm = LSTM(LSTM_DIM, name='article_lstm')(
article_embedding(article_input))
merged_vector = average([headline_lstm, article_lstm],
name='merged_vector')
output = Dense(4, activation='softmax',
name='output_layer')(merged_vector)
N_EPOCHS = EPOCH_OPTIONS[0]
elif MODEL_NUM == 2:
# ======= Model 2 ======
headline_lstm = LSTM(LSTM_DIM, return_state=True, name='headline_lstm')
headline_outputs, state_h, state_c = headline_lstm(
headline_embedding(headline_input))
headline_states = [state_h, state_c]
article_lstm = LSTM(LSTM_DIM,
return_sequences=False,
return_state=False,
name='article_lstm')
article_lstm = article_lstm(article_embedding(article_input),
initial_state=headline_states)
dense = Dense(128, activation='relu')(article_lstm)
output = Dense(4, activation='softmax', name='output_layer')(dense)
N_EPOCHS = EPOCH_OPTIONS[1]
else:
# ======= Model 3 ======
headline_lstm = Bidirectional(LSTM(LSTM_DIM,
return_sequences=False,
return_state=True,
name='headline_lstm'),
merge_mode='concat')
headline_outputs, state_h_forward, state_c_forward, state_h_backward, state_c_backward = headline_lstm(
headline_embedding(headline_input))
headline_states = [
state_h_forward, state_c_forward, state_h_backward,
state_c_backward
]
article_lstm = Bidirectional(LSTM(LSTM_DIM,
return_sequences=False,
return_state=False,
name='article_lstm'),
merge_mode='concat')
article_lstm = article_lstm(article_embedding(article_input),
initial_state=headline_states)
output = Dense(4, activation='softmax',
name='output_layer')(article_lstm)
N_EPOCHS = EPOCH_OPTIONS[2]
print("[INFO] Compiling neural network model")
model = Model(inputs=[headline_input, article_input], outputs=output)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
#plot_model(model, to_file=ROOT_PATH + 'model.png', show_layer_names=True, show_shapes=True)
#Image(ROOT_PATH + 'model.png')
print("[INFO] Training neural network model")
model.fit([X_headline_train, X_article_train],
y_train,
batch_size=BATCH_SIZE,
epochs=N_EPOCHS,
validation_data=([X_headline_val, X_article_val], y_val))
print("[INFO] Predicting on test data")
prediction = model.predict([X_headline_test, X_article_test],
batch_size=BATCH_SIZE)
outputs = [np.argmax(pred, axis=-1) for pred in prediction]
predict_result = [CLASSIFICATION[output] for output in outputs]
test_data = pd.read_csv(ROOT_PATH + "competition_test_stances.csv")
real_result = test_data['Stance']
correct = 0
for i in range(len(predict_result)):
if predict_result[i] == real_result[i]:
correct += 1
print("[INFO] My accuracy = ", correct / len(predict_result))
report_score(real_result, predict_result)
test_data['Stance'] = predict_result
test_data.to_csv(path_or_buf=ROOT_PATH + "submission.csv", index=False)
