def get_accuracy(test_data): predicted_stances = test_data['Stance'].as_matrix() actual_data = data_instance('./competition_test_stances.csv', './test_bodies.csv') actual_stances = actual_data.all_data_df['Stance'].as_matrix() print(f'Accuracy: {np.mean(predicted_stances==actual_stances)}') print(f'actual stances: {string_to_int_labels(actual_stances)}') print(f'predicted stances: {string_to_int_labels(predicted_stances)}') LABELS = ['agree', 'disagree', 'discuss', 'unrelated'] score.report_score( [LABELS[e] for e in string_to_int_labels(actual_stances)], [LABELS[e] for e in string_to_int_labels(predicted_stances)])
def grid_search(X_train, X_test, y_train, y_test, estimator, param_grid): clf = model_selection.GridSearchCV(estimator=estimator, param_grid=param_grid, scoring=scorer, cv=5) clf.fit(X_train, y_train) #print(clf.best_score_) #print(clf.best_params_) #print(clf.cv_results_) test_clf = clf.best_estimator_ test_clf.fit(X_train, y_train) #print(test_clf) pred = test_clf.predict(X_test) temp = score.report_score(y_test, pred) #temp = np.argwhere(y_test==0) #temp1 = np.argwhere(y_test==1) #ix1 = np.in1d(y_test.ravel(), 1).reshape(y_test.shape) #ix0 = np.in1d(y_test.ravel(), 0).reshape(y_test.shape) #print(np.array(X_test[ix0]['A'])) #print(X_test[np.array(temp1)]) #print(temp) #plotDB(test_clf, X_test[ix0]['A'], X_test[ix0]['B'], X_test[ix1]['A'], X_test[ix1]['B'], "Decision Tree") #print(estimator, clf.best_score_) return test_clf, test_clf.score(X_train, y_train), test_clf.score(X_test, y_test), temp
def get_performance(predicted, truth, n_classes=None, outputStyle='dict'): # Predicted and observed are both integer vectors of class label # Cast both predicted and observed to numpy integer predicted = np.asarray(predicted, dtype=np.int64) truth = np.asarray(truth, dtype=np.int64) assert len(predicted) == len(truth) # Compute competition score: competition_score = report_score( [LABELS[e] for e in truth], [LABELS[e] for e in predicted]) #scorer(predicted, truth) output = [] # If n_classes is unknown, infer from the labels if n_classes is None: n_classes = len(np.unique(predicted.extend(truth))) for i in range(n_classes): # Get 2-way table tp = sum((predicted == i) & (truth == i)) tn = sum((predicted != i) & (truth != i)) fp = sum((predicted == i) & (truth != i)) fn = sum((predicted != i) & (truth == i)) print('tp ' + str(tp)) print('tn ' + str(tn)) print('fp ' + str(fp)) print('fn ' + str(fn)) # Compute performance metrics recall = tp / (tp + fn) # aka sensitivity print('recall ' + str(recall)) precision = tp / (tp + fp) # aka ppv print('precision ' + str(precision)) specificity = tn / (tn + fp) print('specificity ' + str(specificity)) f1 = 2 * tp / (2 * tp + fp + fn) print('f1 ' + str(f1)) accuracy = (tp + tn) / len(truth) keys = [ 'tp', 'tn', 'fp', 'fn', 'recall', 'precision', 'specificity', 'f1', 'accuracy', 'competition' ] values = [ tp, tn, fp, fn, recall, precision, specificity, f1, accuracy, competition_score ] output.append(dict(zip(keys, values))) return output
def train_and_test(): result = pd.DataFrame() f = open(F_PKL, 'rb') testX_all = pickle.load(f) f.close() lr = RandomForestClassifier(n_estimators=70, min_samples_split=100, min_samples_leaf=20, max_depth=8, max_features='sqrt', random_state=10) lr.fit(trainX_all, trainy_all) y_pred_binary = lr.predict(testX_all) y_pred_binary = list((np.array(y_pred_binary) - 1) * (-1)) result['binary'] = y_pred_binary stage2 = result[result['binary'] == 1].index.tolist() testX = [] for i in stage2: textX.append(testX_all[i]) gb = GradientBoostingClassifier(n_estimators=1000, learning_rate=0.1, min_samples_split=300, max_features='sqrt', subsample=0.8, random_state=10) gb.fit(trainX, trainY) y_pred = list(gb.predict(testX)) Stance = {0: 'unrelated', 1: 'discuss', 2: 'agree', 3: 'disagree'} pred = [] for i in range(len(y_pred_binary)): if y_pred_binary[i] == 0: pred.append('unrelated') else: pred.append(Stance[y_pred.pop(0)]) dataframe = pd.read_hdf(F_H5) actual = list(dataframe['Stance']) report_score(actual, pred) return pred
def on_epoch_end(self, epoch, logs={}): test_outputs = [] test_predictions = [] labels = ['unrelated', 'agree', 'disagree', 'discuss'] for target in self.Y: test_outputs += [labels[target.tolist().index(1)]] aux = self.model.predict([self.X1, self.X2, self.overlapFeatures_fnc, self.refutingFeatures_fnc, self.polarityFeatures_fnc,\ self.handFeatures_fnc, self.cosFeatures, self.cosFeatures_two, self.bleu_fnc, self.rouge_fnc, self.cider_fnc_test, \ self.X2_two_sentences, self.overlapFeatures_fnc_two, self.refutingFeatures_fnc_two, self.polarityFeatures_fnc_two, self.handFeatures_fnc_two,\ self.bleu_two_sentences, self.rouge_two_sentences, self.cider_two_test, self.talos_counts_test, self.talos_tfidfsim_test, \ self.talos_svdHeadline_test, self.talos_svdBody_test , self.talos_svdsim_test, self.talos_w2vHeadline_test , self.talos_w2vBody_test, \ self.talos_w2vsim_test ,self.talos_sentiHeadline_test , self.talos_sentiBody_test], batch_size=64) for prediction in aux: test_predictions += [labels[prediction.argmax()]] score = report_score(test_outputs, test_predictions, matrix=False)
def train(): data_x, data_y, body_ids, target_stance = build_data() print(data_x, data_y, body_ids, target_stance) # read test data test_x, body_ids_test, true_y = build_test_data() print(Counter(data_y)) # create the SVM model, generating pickle file bst = svm_FNC.fit(data_x, data_y) joblib.dump(bst, 'svm_cluster_train.pkl') pred_y = bst.predict(test_x) print(len(pred_y)) print("------------------------------------") # add encoding and diplay evaluation df_test = pd.DataFrame() df_test['pred_y'] = pred_y df_test['true_y'] = true_y df_test['pred_y'] = df_test['pred_y'].replace([0, 1, 2], ['unknown', 'false', 'true']) df_test['true_y'] = df_test['true_y'].replace([0, 1, 2], ['unknown', 'false', 'true']) df_test = df_test.dropna() print(df_test['pred_y'].value_counts()) print(df_test['true_y'].value_counts()) print(score.report_score(df_test['true_y'], df_test['pred_y'])) predicted = [LABELS[int(a)] for a in pred_y] stances = target_stance df_output = pd.DataFrame() df_output['Headline'] = stances['Headline'] df_output['Body ID'] = stances['Body ID'] df_output['Stance'] = predicted df_output.to_csv('tree_pred_prob_cor2.csv', index=False) df_output[['Headline', 'Body ID', 'Stance']].to_csv('tree_pred_cor2.csv', index=False) print(df_output) print(Counter((df_output['Stance'])))
def scorer(estimator, X, y): y_pred = estimator.predict(X) return score.report_score(y, y_pred) / 100.0
print("\t-Dev size:\t", len(dev_data)) print("\t-Test data:\t", len(test_data)) ############################################################################## # Feature extraction print("[2] Extracting features.. ") training_features, dev_features, test_features = extract_features(training_data, dev_data, test_data) ############################################################################## # Fitting model print("[3] Fitting model..") print("\t-Logistic Regression") #lr = LogisticRegression(C = 1.0, class_weight='balanced', solver="lbfgs", max_iter=150) #lr = MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True) lr = RandomForestClassifier(n_estimators=10, random_state=12345) targets_tr = [a['Stance'] for a in training_data] targets_dev = [a['Stance'] for a in dev_data] targets_test = [a['Stance'] for a in test_data] y_pred = lr.fit(training_features, targets_tr).predict(test_features) ############################################################################## # Evaluation print("[4] Evaluating model..") score.report_score(targets_test, y_pred)
voting_result = [] temp_count = 0 temp_res = "" voting_result.append(final_prediction[i]) voting_result.append(y_pred_logistic[i]) voting_result.append(y_pred_random[i]) voting_result.append(y_pred_mnb[i]) if (voting_result.count("agree") > temp_count): temp_count = voting_result.count("agree") temp_res = "agree" if (voting_result.count("disagree") > temp_count): temp_count = voting_result.count("disagree") temp_res = "disagree" if (voting_result.count("discuss") > temp_count): temp_count = voting_result.count("discuss") temp_res = "discuss" if (voting_result.count("unrelated") > temp_count): temp_count = voting_result.count("unrelated") temp_res = "unrelated" final.append(temp_res) #saving the predicted stances for submission purpose np.savetxt("stance.csv", np.array(final), delimiter="\n", fmt='%s') targets_test = [a['Stance'] for a in test_data] # Evaluation print("[4] Evaluating model..") score.report_score(targets_test, final)
#Concatenation approach #final_pred=np.concatenate((np.matmul(test_prediction1,weight_pred_1),np.matmul(test_prediction2,weight_pred_2),np.matmul(test_prediction3,weight_pred_3)),axis=1) #Summation approach final_pred=np.matmul(test_prediction1,weight_pred_1)+np.matmul(test_prediction2,weight_pred_2)+np.matmul(test_prediction3,weight_pred_3) #=======no ensemble============ #final_pred=test_prediction1 #============================== final_pred_index = np.argmax(final_pred,1) save_predictions(base_dir,final_pred_index, file_predictions) # Calculate score golden_stance = pd.read_csv(base_dir+"/"+"test_stances_labeled.csv") prediction_stance=pd.read_csv(base_dir+"/"+"predictions_test.csv") competition_grade,agree_recall,disagree_recall,discuss_recall,unrelated_recall, agree_precision, disagree_precision, discuss_precision, unrelated_precision,all_recall, f1_agree, f1_disagree, f1_discuss, f1_unrelated, F1m=report_score(golden_stance['Stance'],prediction_stance['Prediction']) Grade.append(competition_grade) Agree_recall.append(agree_recall) Disagree_recall.append(disagree_recall) Discuss_recall.append(discuss_recall) Unrelated_recall.append(unrelated_recall) Agree_precision.append(agree_precision) Disagree_precision.append(disagree_precision) Discuss_precision.append(discuss_precision) Unrelated_precision.append(unrelated_precision) Recall.append(all_recall)
RandomForestClassifier(n_estimators=10), MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True), GradientBoostingClassifier(n_estimators=200, random_state=14128, verbose=True), KNeighborsClassifier(4), SVC(kernel="linear", C=0.025), DecisionTreeClassifier(max_depth=5), LogisticRegression(C=1e5) ] # KNeighborsClassifier(4, algorithm='kd_tree'), for n, clf in zip(names, classifiers): print(n) y_pred = clf.fit(train_features, train_labels).predict(validate_features) print(score.report_score(test_labels, y_pred)) print('\n') # ## Run Classifiers and Score Test Output # This is how well we would have scored in the actual competition names = [ "Random Forest", "Multinomial Naive Bayes", "Gradient Boosting", "K Nearest Neighbors", "Linear SVM", "Decision Tree", "Logistic Regression" ] classifiers = [ RandomForestClassifier(n_estimators=10), MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True), GradientBoostingClassifier(n_estimators=200, random_state=14128,
def cv(): K_FOLD = 10 data = load_features().astype(float) targets = load_targets() # K-Fold and Score Tracking scores = [] wscores = [] pscores = [] n_folds = 10 best_iters = [0] * n_folds # kf = GroupKFold(n_splits=K_FOLD) kf = StratifiedKFold(n_splits=K_FOLD) print('Training Model...') for fold, (train_idx, test_idx) in enumerate(kf.split(data, targets['target'])): print('\n[K = ' + str(fold + 1) + ']') # Train Model dtrain = xgb.DMatrix(data[train_idx], label=targets[train_idx]) dtest = xgb.DMatrix(data[test_idx]) watchlist = [(dtrain, 'train'), (dtest, 'eval')] bst = xgb.train( params_xgb, dtrain, num_round, watchlist, verbose_eval=10, #feval = eval_metric, #maximize = True, early_stopping_rounds=80) pred_prob_y = bst.predict(dtest).reshape(targets[test_idx].shape[0], 2) # predicted probabilities pred_y = bst.predict(dtest, ntree_limit=bst.best_ntree_limit).reshape( targets[test_idx].shape[0], 2) print('predicted probabilities: ') print(pred_y) pred_y = np.argmax(pred_y, axis=1) print('predicted label indices: ') print(pred_y) print('best iterations: ', bst.best_ntree_limit) best_iters[fold] = bst.best_ntree_limit print(pred_y) print('pred_y.shape') print(pred_y.shape) print('y_valid.shape') print(targets[test_idx].shape) predicted = [LABELS[int(a)] for a in pred_y] actual = [LABELS[int(a)] for a in targets[test_idx]] s, _ = score_submission(actual, predicted) s_perf, _ = score_submission(actual, actual) r_score = report_score(actual, predicted) wscore = float(s) / s_perf print('report_score', r_score) print('fold %s, score = %f, perfect_score %f, weighted percentage %f' % (fold, s, s_perf, wscore)) scores.append(s) pscores.append(s_perf) wscores.append(wscore) print('scores:') print(scores) print('mean score:') print(np.mean(scores)) print('perfect scores:') print(pscores) print('mean perfect score:') print(np.mean(pscores)) print('w scores:') print(wscores) print('mean w score:') print(np.mean(wscores)) print('best iters:') print(best_iters) print('mean best_iter:') m_best = np.mean(best_iters) print(m_best)
print("training the model") #clf = GridSearchCV(AdaBoostClassifier(), parameters, cv=10, verbose=2) #clf.fit(train_features, y_train) #print('Best Score: ', clf.best_score_) #print('Best Params: ', clf.best_params_) classifiers.fit(train_features, y_train) print("Prediction on dev dataset Ada") predicted = {} predicted = classifiers.predict(val_features) print(report_score(y_val, predicted)) print("Prediction on test dataset") predicted = classifiers.predict(test_features) competetion_unlabeled = pd.read_csv( 'C:\\Users\\jeswi\\competition_test_stances_unlabeled.csv') Predicted = pd.DataFrame({'Stance': predicted}) result = pd.concat([competetion_unlabeled, Predicted], axis=1, sort=False) result.to_csv('C:\\Users\\jeswi\\Desktop\\submission_LSTM_Glove.csv', index=False, encoding='utf-8') print(report_score(test_labels, predicted)) print('ADA:')
# Test the model on the FNC Test Set # test_outputs = [] test_predictions = [] labels = ['unrelated', 'agree', 'disagree', 'discuss'] for target in Y_test: test_outputs += [labels[target.tolist().index(1)]] aux = final_model.predict([X1_test, X2_test, overlapFeatures_fnc_test, refutingFeatures_fnc_test, polarityFeatures_fnc_test, handFeatures_fnc_test, \ cosFeatures_fnc_test, cosFeatures_two_test, bleu_fnc_test, rouge_fnc_test, cider_fnc_test, \ X2_test_two_sentences, overlapFeatures_fnc_two_test, refutingFeatures_fnc_two_test, polarityFeatures_fnc_two_test, handFeatures_fnc_two_test, \ bleu_two_sentences_test, rouge_two_sentences_test, cider_two_test, talos_counts_test, talos_tfidfsim_test, talos_svdHeadline_test, talos_svdBody_test, talos_svdsim_test, \ talos_w2vHeadline_test, talos_w2vBody_test, talos_w2vsim_test, talos_sentiHeadline_test, talos_sentiBody_test]) preds = [] for prediction in aux: pred = prediction.argmax() if pred == 0: one_hot = [1, 0, 0, 0] if pred == 1: one_hot = [0, 1, 0, 0] if pred == 2: one_hot = [0, 0, 1, 0] if pred == 3: one_hot = [0, 0, 0, 1] preds += [one_hot] test_predictions += [labels[prediction.argmax()]] report_score(test_outputs, test_predictions) ######################################
gc.collect() mlp_model = load_model(os.path.join(models_dir, mlp_model_file)) if sentiment == 'yes': test_features = np.concatenate( (np.load('test.tfidf.npy'), np.load('test.sent.npy')), axis=1) else: test_features = np.load('test.tfidf.npy') test_labels = np.load('test.labels.npy') print(f"\n\nShape of test set (Inputs): {test_features.shape}") print(f"Shape of test set (Labels): {test_labels.shape}\n\n") # Prediction test_predictions = mlp_model.predict(test_features) test_predictions = np.argmax(test_predictions, axis=1) test_predictions = [label_ref_rev[i] for i in test_predictions] test_labels = [label_ref_rev[i] for i in test_labels] print("Scores on test set") report_score(test_labels, test_predictions) print() print() print(classification_report(test_labels, test_predictions)) # Save predictions save_predictions(test_predictions, file_predictions)
train_hand_features = np.array(train_hand_features) test_hand_features = np.array(test_hand_features) train_features = hstack( [train_body_tfidf, train_headline_tfidf, train_hand_features]) test_features = hstack( [test_body_tfidf, test_headline_tfidf, test_hand_features]) #Extract training and test labels train_labels = list(train_df['Stance']) test_labels = list(test_df['Stance']) #Initialize random forest classifier (Scikit Learn) rf_classifier = RandomForestClassifier(n_estimators=10) y_pred = rf_classifier.fit(train_features, train_labels).predict(test_features) score.report_score(test_labels, y_pred) #Initialize multinomialnb classifier nb_classifier = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True) y_pred = nb_classifier.fit(train_features, train_labels).predict(test_features) score.report_score(test_labels, y_pred) #Add predicted labels to test dataframe test_df['RF_Predicted_Stance'] = list(y_pred) test_df2 = test_df[['Headline', 'Body_Text', 'RF_Predicted_Stance', 'Stance']] test_df2[test_df2['RF_Predicted_Stance'] == 'unrelated'] #Initialize random forest classifier (Scikit Learn) rf_classifier = RandomForestClassifier(n_estimators=10) y_pred = rf_classifier.fit(train_features, train_labels).predict(test_features) score.report_score(test_labels, y_pred)
else: training_on = training_set_stance testing_on = dev_set_stance training_set = pd.merge(training_on, bodies, how='inner', left_on='Body ID', right_index=True, sort=True, suffixes=('_x', '_y'), copy=True, indicator=True) test_set = pd.merge(testing_on, bodies, how='inner', left_on='Body ID', right_index=True, sort=True, suffixes=('_x', '_y'), copy=True, indicator=True) classifier.fit(training_set.drop('Stance', axis = 1), training_set['Stance']) #print(classifier.best_params_) print("getting predicted labels...") hypotheses = classifier.predict(test_set) #hypotheses = ['unrelated'] * len(test_set) c_r = classification_report(test_set['Stance'], hypotheses) print(c_r) print(accuracy_score(test_set['Stance'], hypotheses)) print(report_score(test_set['Stance'], hypotheses)) splits= c_r.split('\n')[2:6] fscore_mean = np.mean([float(x.strip().split()[3]) for x in splits]) print('fscore mean') print(fscore_mean) # #now = datetime.now() #date_parts = [str(x) for x in [now.month,now.day,now.year]] #time_parts = [str(x) for x in [now.hour,now.minute,now.second]] #now_str = '' #joblib.dump(classifier, 'classifiers/classifier' + now_str + '.pkl') #now_str = "-".join(date_parts) + "_" + "-".join(time_parts)
y = torch.tensor(label, dtype=torch.long, device=device) x_similarity = torch.tensor(sim, dtype=torch.float, device=device) x_entailment = torch.stack(entail, dim=0).to(device) outputs = model(x_p, x_h, x_similarity, x_entailment) out.append(outputs) y_label.append(y) outputs = torch.cat(out, dim=0) y = torch.cat(y_label) loss = criterion(outputs, y) accuracy = evaluation(outputs, y) print("Dev | Epoch:{0} | Loss:{1} | Accuracy:{2}".format( epoch + 1, loss, accuracy)) report_F1(outputs, y) report_score(outputs, y) # torch.save(model.state_dict(), 'runs/{0}/parameters_{1}'.format(argvs[1], epoch+1)) ''' # test model.eval() loss_sum = 0 accuracy_sum = 0 with torch.no_grad(): for premise, hypothesis, label in batcher(token_data["test"], test_batch): x_p = torch.tensor(premise, dtype=torch.long, device=device) x_h = torch.tensor(hypothesis, dtype=torch.long, device=device) y = torch.tensor(label, dtype=torch.long, device=device) outputs = model(x_p, x_h) loss = criterion(outputs, y) accuracy = evaluation(outputs, y) loss_sum += loss
#final_pred=np.matmul(test_prediction1,weight_pred_1)+np.matmul(test_prediction2,weight_pred_2) #=======ensemble for three========= #final_pred=np.concatenate((np.matmul(test_prediction1,weight_pred_1),np.matmul(test_prediction2,weight_pred_2),np.matmul(test_prediction3,weight_pred_3)),axis=1) #final_pred=np.matmul(test_prediction1,weight_pred_1)+np.matmul(test_prediction2,weight_pred_2)+np.matmul(test_prediction3,weight_pred_3) #=======no ensemble ======== final_pred = test_prediction1 #==================================== final_pred_index = np.argmax(final_pred, 1) save_predictions(base_dir, final_pred_index, file_predictions) # =========Calculate score================================================= golden_stance = pd.read_csv(base_dir + "/" + "test_stances_labeled.csv") prediction_stance = pd.read_csv(base_dir + "/" + "predictions_test.csv") competition_grade, agree_recall, disagree_recall, discuss_recall, unrelated_recall, all_recall = report_score( golden_stance['Stance'], prediction_stance['Prediction']) Grade.append(competition_grade) Agree.append(agree_recall) Disagree.append(disagree_recall) Discuss.append(discuss_recall) Unrelated.append(unrelated_recall) Recall.append(all_recall) print('Grade', Grade) print('Agree', Agree) print('Disagree', Disagree) print('Discuss', Discuss) print('Unrelated', Unrelated) print('All Recall', Recall)
# import packages import pandas as pd import score from sklearn.metrics import f1_score # read the data ### predicted y predict_df = pd.read_csv("competition_test_stances.csv", encoding='iso-8859-1') ### true y origin_df = pd.read_csv("tree_pred_cor2.csv", encoding='iso-8859-1') # confusion matrix and official score score.report_score(origin_df['Stance'], predict_df['Stance']) # f1_score original macro_score = f1_score(origin_df['Stance'], predict_df['Stance'], average='macro') print("Macro score: " + str(macro_score)) # f1_score weighted weighted_score = f1_score(origin_df['Stance'], predict_df['Stance'], average='weighted') print("Weighted score: " + str(weighted_score))
clf.fit(X_train, y_train) predicted = [LABELS[int(a)] for a in clf.predict(X_test)] actual = [LABELS[int(a)] for a in y_test] fold_score, _ = score_submission(actual, predicted) max_fold_score, _ = score_submission(actual, actual) score = fold_score / max_fold_score print("Score for fold " + str(fold) + " was - " + str(score)) if score > best_score: best_score = score best_fold = clf #Run on Holdout set and report the final score on the holdout set predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)] actual = [LABELS[int(a)] for a in y_holdout] print("Scores on the dev set") report_score(actual, predicted) print("") print("") #Run on competition dataset predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)] actual = [LABELS[int(a)] for a in y_competition] print("Scores on the test set") report_score(actual, predicted)
def main(): MODEL_NUM = 3 if len(sys.argv) > 1: MODEL_NUM = int(sys.argv[1]) print("[INFO] Processing training and test data") headlines, articles, stances = processing_data("processed_data.csv") headlines_test, articles_test, stances_test = processing_data( "processed_test.csv") print("[INFO] Fitting data into tokenizer") headline_tokenizer = fitting_tokenizer(headlines, headlines_test, 'headline') article_tokenizer = fitting_tokenizer(articles, articles_test, 'article') print("[INFO] Padding data to achieve max length") X_headline, X_article, y = padding(headlines, articles, stances, headline_tokenizer, article_tokenizer) X_headline_test, X_article_test, y_test = padding(headlines_test, articles_test, stances_test, headline_tokenizer, article_tokenizer) print("[INFO] Spliting out validation data") X_headline_train, X_headline_val, X_article_train, X_article_val, y_train, y_val = train_test_split( X_headline, X_article, y, random_state=10, test_size=0.1) print("[INFO] Loading GloVe embedding from file") embeddings = dict() with open(ROOT_PATH + 'glove.6B.300d.txt') as f: for line in f: tokens = line.strip().split(" ") embeddings[tokens[0]] = np.array(tokens[1:], dtype='float32') print("[INFO] Building embedding matrix for HEADLINE and ARTICLE") headline_embeddings_matrix = building_embedding_matrix( headline_tokenizer, embeddings) article_embeddings_matrix = building_embedding_matrix( article_tokenizer, embeddings) del embeddings print("[INFO] Building neural network model") headline_input = Input(shape=(MAX_HEADLINE_LEN, ), name='headline_input') headline_embedding = Embedding( output_dim=EMBEDDING_DIM, input_dim=len(headline_tokenizer.word_index) + 1, input_length=MAX_HEADLINE_LEN, weights=[headline_embeddings_matrix], trainable=False, name='headline_embedding') article_input = Input(shape=(MAX_ARTICLE_LEN, ), name='article_input') article_embedding = Embedding(output_dim=EMBEDDING_DIM, input_dim=len(article_tokenizer.word_index) + 1, input_length=MAX_ARTICLE_LEN, weights=[article_embeddings_matrix], trainable=False, name='article_embedding') if MODEL_NUM == 1: # ======= Model 1 ====== headline_lstm = LSTM(LSTM_DIM, name='headline_lstm')( headline_embedding(headline_input)) article_lstm = LSTM(LSTM_DIM, name='article_lstm')( article_embedding(article_input)) merged_vector = average([headline_lstm, article_lstm], name='merged_vector') output = Dense(4, activation='softmax', name='output_layer')(merged_vector) N_EPOCHS = EPOCH_OPTIONS[0] elif MODEL_NUM == 2: # ======= Model 2 ====== headline_lstm = LSTM(LSTM_DIM, return_state=True, name='headline_lstm') headline_outputs, state_h, state_c = headline_lstm( headline_embedding(headline_input)) headline_states = [state_h, state_c] article_lstm = LSTM(LSTM_DIM, return_sequences=False, return_state=False, name='article_lstm') article_lstm = article_lstm(article_embedding(article_input), initial_state=headline_states) dense = Dense(128, activation='relu')(article_lstm) output = Dense(4, activation='softmax', name='output_layer')(dense) N_EPOCHS = EPOCH_OPTIONS[1] else: # ======= Model 3 ====== headline_lstm = Bidirectional(LSTM(LSTM_DIM, return_sequences=False, return_state=True, name='headline_lstm'), merge_mode='concat') headline_outputs, state_h_forward, state_c_forward, state_h_backward, state_c_backward = headline_lstm( headline_embedding(headline_input)) headline_states = [ state_h_forward, state_c_forward, state_h_backward, state_c_backward ] article_lstm = Bidirectional(LSTM(LSTM_DIM, return_sequences=False, return_state=False, name='article_lstm'), merge_mode='concat') article_lstm = article_lstm(article_embedding(article_input), initial_state=headline_states) output = Dense(4, activation='softmax', name='output_layer')(article_lstm) N_EPOCHS = EPOCH_OPTIONS[2] print("[INFO] Compiling neural network model") model = Model(inputs=[headline_input, article_input], outputs=output) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) print(model.summary()) #plot_model(model, to_file=ROOT_PATH + 'model.png', show_layer_names=True, show_shapes=True) #Image(ROOT_PATH + 'model.png') print("[INFO] Training neural network model") model.fit([X_headline_train, X_article_train], y_train, batch_size=BATCH_SIZE, epochs=N_EPOCHS, validation_data=([X_headline_val, X_article_val], y_val)) print("[INFO] Predicting on test data") prediction = model.predict([X_headline_test, X_article_test], batch_size=BATCH_SIZE) outputs = [np.argmax(pred, axis=-1) for pred in prediction] predict_result = [CLASSIFICATION[output] for output in outputs] test_data = pd.read_csv(ROOT_PATH + "competition_test_stances.csv") real_result = test_data['Stance'] correct = 0 for i in range(len(predict_result)): if predict_result[i] == real_result[i]: correct += 1 print("[INFO] My accuracy = ", correct / len(predict_result)) report_score(real_result, predict_result) test_data['Stance'] = predict_result test_data.to_csv(path_or_buf=ROOT_PATH + "submission.csv", index=False)