def evaluate_model(best_fold, related_best_fold, X_holdout, y_holdout,
y_holdout_bi):
predicted = [LABELS_RELATED[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS_RELATED[int(a)] for a in y_holdout_bi]
related_X_holdout = []
related_actual = []
unrelated_actual = []
un_count = 0
for i in range(len(predicted)):
if predicted[i] == "related":
related_X_holdout.append(X_holdout[i])
related_actual.append(LABELS[int(y_holdout[i])])
else:
unrelated_actual.append(LABELS[int(y_holdout[i])])
un_count += 1
predicted = [
LABELS[int(a)] for a in related_best_fold.predict(related_X_holdout)
] + ["unrelated"] * un_count
actual = related_actual + unrelated_actual
report_score(actual, predicted)
print("")
print("")
def score(self):
predicted = [
LABELS[self.maxindex(a)] for a in self.model.predict(self.X_test)
]
actual = [LABELS[self.maxindex(a)] for a in self.Y_test]
np.savetxt("stance4.csv", predicted, delimiter=",", fmt='%s')
report_score(actual, predicted)
def run_base_classifier(train_m, train_truth, test_m, test_truth):
print('Base results:')
pred_strs = ['unrelated' for _ in test_truth]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
s = report_score(test_strs, pred_strs)
pred_strs = ['agree' for _ in test_truth]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
s = max(s, report_score(test_strs, pred_strs))
return s
def get_scores(predicted, actual):
get_labels = np.vectorize(lambda t: LABELS[int(t)])
predicted = get_labels(predicted)
actual = get_labels(actual)
print("Scores on the val set")
report_score(actual, predicted)
print("")
def score(self):
predicted = [
LABELS[self.maxindex(a)] for a in self.model.predict([
self.X_test_head_tf, self.X_test_bodies_tf,
self.X_test_head_idf, self.X_test_bodies_idf
])
]
actual = [LABELS[self.maxindex(a)] for a in self.Y_test]
np.savetxt("stance2.csv", predicted, delimiter=",", fmt='%s')
report_score(actual, predicted)
def do_reg():
d = DataSet()
folds, hold_out = kfold_split(d, n_folds=10)
fold_stances, hold_out_stances = get_stances_for_folds(d, folds, hold_out)
Xs = dict()
ys = dict()
# Load/Precompute all features now
X_holdout, y_holdout = generate_features(hold_out_stances, d, "holdout")
for fold in fold_stances:
Xs[fold], ys[fold] = generate_features(fold_stances[fold], d,
str(fold))
best_score = 0
best_fold = None
# Classifier for each fold
for fold in fold_stances:
ids = list(range(len(folds)))
del ids[fold]
X_train = np.vstack(tuple([Xs[i] for i in ids]))
y_train = np.hstack(tuple([ys[i] for i in ids]))
X_test = Xs[fold]
y_test = ys[fold]
clf_stage1 = GradientBoostingClassifier(n_estimators=200,
random_state=14128,
verbose=True)
#clf = GradientBoostingClassifier(n_estimators=50, random_state=14128, verbose=False)
# Try random forest
clf.fit(X_train, y_train)
predicted = [LABELS[int(a)] for a in clf.predict(X_test)]
actual = [LABELS[int(a)] for a in y_test]
fold_score, _ = score_submission(actual, predicted)
max_fold_score, _ = score_submission(actual, actual)
score = fold_score / max_fold_score
print("Score for fold " + str(fold) + " was - " + str(score))
if score > best_score:
best_score = score
best_fold = clf
#Run on Holdout set and report the final score on the holdout set
predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS[int(a)] for a in y_holdout]
report_score(actual, predicted)
def run_NN_classifier(train_m,
train_truth,
test_m,
test_truth,
validate=False):
print('running NN classifier, validate=' + str(validate))
hidden_layer_sizes = [5, 10, 50, 100, 1000]
activations = ['identity', 'logistic', 'tanh', 'relu']
learning_rates = [0.001, 0.01, 0.0001]
highest_fnc = 0
best_h_size = 100
best_activation = 'relu'
best_lr = 0.001
if validate:
split_train_m, split_train_truth, validate_m, validate_truth = get_validation_mat(
0.7, train_m, train_truth)
# split_train_truth = label_binarize(split_train_truth, classes=[0, 1, 2, 3])
for h in hidden_layer_sizes:
for a in activations:
for lr in learning_rates:
model = MLPClassifier(max_iter=1000,
hidden_layer_sizes=h,
activation=a,
learning_rate_init=lr)
clf = OneVsOneClassifier(model)
pred_labels = clf.fit(
split_train_m,
split_train_truth).predict(validate_m).tolist()
# pred_strs = fnc_one_hot_label_decode(pred_labels)
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in validate_truth]
fnc_s = report_score(test_strs, pred_strs, verbose=False)
if fnc_s > highest_fnc:
highest_fnc = fnc_s
best_activation = a
best_h_size = h
best_lr = lr
# train_truth = label_binarize(train_truth, classes=[0, 1, 2, 3])
model = MLPClassifier(max_iter=1000,
hidden_layer_sizes=best_h_size,
activation=best_activation,
learning_rate_init=best_lr)
clf = OneVsOneClassifier(model)
pred_labels = clf.fit(train_m, train_truth).predict(test_m).tolist()
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
print('NN results:')
f1_w = f1_score(test_strs, pred_strs, average='weighted')
f1_class = f1_score(test_strs, pred_strs, average=None)
acc = accuracy_score(test_strs, pred_strs)
print('NN f1 comp: ' + str(f1_w) + ', ' + str(f1_class))
print('NN acc: ' + str(acc))
return report_score(test_strs, pred_strs), f1_w, acc
def print_scores(test, comp, id):
pred_test = [s["Predict"] for s in test.get_labelled_stances() if s["Stance ID"] in id]
actl_test = [s["Stance"] for s in test.get_labelled_stances() if s["Stance ID"] in id]
pred_comp = [s["Predict"] for s in comp.get_labelled_stances()]
actl_comp = [s["Stance"] for s in comp.get_labelled_stances()]
print("Scores on the dev set")
report_score(actl_test,pred_test)
print("")
print("")
print("Scores on the test set")
report_score(actl_comp,pred_comp)
def predict(self):
print("test model")
self.mlp.model.load_weights('Mingjie_Chen_weights.h5')
result = self.mlp.model.predict(self.test_vec)
labels = ['unrelated', 'disagree', 'discuss', 'agree']
result_lables = [labels[np.argmax(n)] for n in result]
actual = [stance['Stance'] for stance in test_data]
# actual = [labels[np.argmax(n)] for n in test_y]
n = 0
for a, p in zip(result_lables, actual):
if a == p:
n += 1
print("test accuracy is {}".format(n / len(result_lables)))
report_score(actual, result_lables)
def mlp_predict(self,data):
# load the svm classifier
mlp_clf = pickle.load(open("mlp_classifier", "rb"))
#
total_dev_data = len(data)
folder = 'test_feature_vectors'
file_path_list = os.listdir(folder)
correct = 0
print("Testing Set!")
actual_list = []
predicted_list = []
for i, file_path in enumerate(file_path_list):
file_name = os.path.basename(file_path)
file_path = os.path.join(folder, file_path)
stance = re.findall(r'stance#(.*)#', file_name)[0]
feature_vector = np.load(file_path)
feature_vector = feature_vector.reshape(1, -1)
if (feature_vector == np.array(None)).all():
print("feature_vector is None!")
sys.exit(0)
# print("file_path: {}".format(file_path))
# print ("feature_vector: {}".format(feature_vector))
clf_stance = mlp_clf.predict(feature_vector)
if stance == clf_stance:
correct += 1
actual_list.append(stance)
predicted_list.append(clf_stance)
# print ("add correct, dev data id :{}".format(i))
# save accuracy
Accuracy = report_score(actual_list, predicted_list)
self.mlp_result.append(Accuracy)
def test_mlf(node,
data,
ols_limit,
sensitivity,
func,
num_classes,
verbose=True):
start = timeit.default_timer()
truth_labels = [t[class_label_idx] for t in data]
truth_labels_str = [t[len(t) - 1] for t in data]
predict_labels = []
predict_labels_str = []
for test_data in data:
membership_vect = [0 for _ in range(num_classes)]
test_val = test_data[sample_val_idx]
label = mlf_classify(membership_vect, node, test_val, ols_limit,
sensitivity, func)
predict_labels.append(label)
predict_labels_str.append(fnc_vect_to_label(label))
assert len(truth_labels) == len(predict_labels)
classify_time = timeit.default_timer() - start
if verbose:
print('classify time: ' + str(classify_time))
fnc_score = report_score(truth_labels_str,
predict_labels_str,
verbose=verbose)
f1_weighted = f1_score(truth_labels, predict_labels, average='weighted')
f1_class = f1_score(truth_labels, predict_labels, average=None)
acc = accuracy_score(truth_labels, predict_labels)
print('MLF f1 comp: ' + str(f1_weighted) + ', ' + str(f1_class))
print('MLF acc: ' + str(acc))
return fnc_score, predict_labels, f1_weighted, acc
def evaluate_answer(model, model_inp, true):
inv_category_dict = {
0: 'unrelated',
1: 'agree',
2: 'disagree',
3: 'discuss'
}
predicted = model.predict(model_inp)
predicted = np.argmax(predicted, axis=1)
t = np.argmax(true, axis=1)
ground = list()
pred = list()
for i in predicted:
pred.append(inv_category_dict[i])
for i in t:
ground.append(inv_category_dict[i])
score.report_score(ground, pred)
def run_LR_classifier(train_m,
train_truth,
test_m,
test_truth,
validate=False):
print('running LR classifier, validate=' + str(validate))
cs = [0.01, 0.1, 1.0, 10]
regs = ['l2']
highest_fnc = 0
best_c = 0.1
best_reg = 'l2'
if validate:
split_train_m, split_train_truth, validate_m, validate_truth = get_validation_mat(
0.7, train_m, train_truth)
for c in cs:
for reg in regs:
model = LogisticRegression(multi_class='multinomial',
solver='lbfgs',
C=c,
penalty=reg)
model.fit(split_train_m, split_train_truth)
pred_labels = model.predict(validate_m)
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in validate_truth]
fnc_s = report_score(test_strs, pred_strs, verbose=False)
if fnc_s > highest_fnc:
highest_fnc = fnc_s
best_reg = reg
best_c = c
model = LogisticRegression(multi_class='multinomial',
solver='lbfgs',
C=best_c,
penalty=best_reg)
model.fit(train_m, train_truth)
pred_labels = model.predict(test_m)
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
print('LR results:')
f1_w = f1_score(test_strs, pred_strs, average='weighted')
f1_class = f1_score(test_strs, pred_strs, average=None)
acc = accuracy_score(test_strs, pred_strs)
print('LR f1 comp: ' + str(f1_w) + ', ' + str(f1_class))
print('LR acc: ' + str(acc))
return report_score(test_strs, pred_strs), f1_w, acc
def run_NB_classifier(train_m,
train_truth,
test_m,
test_truth,
validate=False):
print('running NB classifier, validate=' + str(validate))
priors = [None, [0.5, 0.5]]
best_prior = None
highest_fnc = 0
if validate:
split_train_m, split_train_truth, validate_m, validate_truth = get_validation_mat(
0.7, train_m, train_truth)
# split_train_truth = label_binarize(split_train_truth, classes=[0, 1, 2, 3])
for pr in priors:
model = GaussianNB(priors=pr)
clf = OneVsOneClassifier(model)
pred_labels = clf.fit(
split_train_m, split_train_truth).predict(validate_m).tolist()
# pred_strs = fnc_one_hot_label_decode(pred_labels)
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in validate_truth]
fnc_s = report_score(test_strs, pred_strs, verbose=False)
if fnc_s > highest_fnc:
highest_fnc = fnc_s
best_prior = pr
# train_truth = label_binarize(train_truth, classes=[0, 1, 2, 3])
model = GaussianNB(priors=best_prior)
clf = OneVsOneClassifier(model)
pred_labels = clf.fit(train_m, train_truth).predict(test_m).tolist()
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
print('NB results:')
f1_w = f1_score(test_strs, pred_strs, average='weighted')
f1_class = f1_score(test_strs, pred_strs, average=None)
acc = accuracy_score(test_strs, pred_strs)
print('NB f1 comp: ' + str(f1_w) + ', ' + str(f1_class))
print('NB acc: ' + str(acc))
return report_score(test_strs, pred_strs), f1_w, acc
def check_score_cm(self, session, body_path, headline_path, ans_path, dataset, num_samples=100, print_to_screen=False):
logging.info("Calculating Score for %s examples in %s set..." % (str(num_samples) if num_samples != 0 else "all", dataset))
tic = time.time()
pred_list = []
actual_list = []
rem_count = num_samples
# Note here we select discard_long=False because we want to sample from the entire dataset
# That means we're truncating, rather than discarding, examples with too-long context or questions
for batch in custom_get_batch_generator(self.word2id, body_path, headline_path, ans_path, self.FLAGS.batch_size, context_len=self.FLAGS.context_len, question_len=self.FLAGS.question_len, discard_long=False):
pred_class = self.get_class(session, batch)
pred_class = pred_class.tolist()
predicted = [LABELS[int(a)] for a in pred_class]
batch_size = batch.batch_size
actual_reshaped = (batch.ans_span).reshape(batch_size,) #Reshape it to (batch_size,)
actual = [LABELS[int(a)] for a in actual_reshaped]
#score, cm = score_submission(actual, predicted)
if num_samples == 0: # All entries in all batches // for dev set....
pred_list = pred_list+predicted
actual_list = actual_list+actual
else:
if rem_count>=batch_size:
pred_list = pred_list+predicted
actual_list = actual_list+actual
rem_count = rem_count-batch_size
else:
pred_list = pred_list+predicted[:rem_count]
actual_list = actual_list+actual[:rem_count]
rem_count = 0
if rem_count == 0:
break
#HERE
score = report_score(actual_list, pred_list)
toc = time.time()
logging.info("Calculating Score/CM for %i examples in %s set took %.2f seconds" % (num_samples, dataset, toc-tic))
return score
def train_model(lrmodel, X, Y, devX, devY, devscores, feat_train, feat_dev, train_tfidf, test_tfidf):
"""
Train model, using pearsonr on dev for early stopping
"""
done = False
best = -1.0
#r = np.arange(1,5)
while not done:
# Every 100 epochs, check Pearson on development set
lrmodel.fit([X,feat_train,train_tfidf], Y, verbose=2, shuffle=False, nb_epoch = 3, validation_data=([devX,feat_dev,test_tfidf], devY))
#yhat = np.dot(lrmodel.predict(devX, verbose=2), r)
yhat = lrmodel.predict([devX,feat_dev,test_tfidf], verbose=2)
yhat = [i.argmax()for i in yhat]
string_predicted,test_stances = [],[]
for i,j in zip(yhat,devscores):
if i == 3:
string_predicted.append('unrelated')
if i == 0:
string_predicted.append('agree')
if i == 2:
string_predicted.append('discuss')
if i == 1:
string_predicted.append('disagree')
if j == 3:
test_stances.append('unrelated')
if j == 0:
test_stances.append('agree')
if j == 2:
test_stances.append('discuss')
if j == 1:
test_stances.append('disagree')
print 'using new limit value....'
#score = accuracy_score(devscores, yhat)
score = report_score(test_stances,string_predicted,val=True)
#return lrmodel
if score > best:
print score
best = score
bestlrmodel = prepare_model(ninputs=X.shape[1],n_feats=feat_train.shape[1],n_tfidf=train_tfidf.shape[1])
bestlrmodel.set_weights(lrmodel.get_weights())
else:
done = True
print '***** best model obtained with score',best,'******'
yhat = bestlrmodel.predict([devX, feat_dev, test_tfidf], verbose=2)
yhat = [i.argmax()for i in yhat]
string_predicted,test_stances = [],[]
for i,j in zip(yhat,devscores):
if i == 3:
string_predicted.append('unrelated')
if i == 0:
string_predicted.append('agree')
if i == 2:
string_predicted.append('discuss')
if i == 1:
string_predicted.append('disagree')
if j == 3:
test_stances.append('unrelated')
if j == 0:
test_stances.append('agree')
if j == 2:
test_stances.append('discuss')
if j == 1:
test_stances.append('disagree')
report_score(test_stances,string_predicted)
return bestlrmodel
def evaluate(encoder=None, seed=1234, evaltest=False, loc='./data/'):
"""
Run experiment
"""
print 'Preparing data for fnc...'
#train, dev, test, scores = load_data(loc)
#train[0], train[1], scores[0] = shuffle(train[0], train[1], scores[0], random_state=seed)
'''
trh, trb, tsh, tsb =\
load_dataset("/fnc_data/train_stances.csv", "/fnc_data/train_bodies.csv",\
"/fnc_data/competition_test_stances.csv", "/fnc_data/test_bodies.csv")
'''
train_h = np.load('/fncdata2/encode_train_head.npy')
train_b = np.load('/fncdata2/encode_train_body.npy')
test_h = np.load('/fncdata2/encode_test_head.npy')
test_b = np.load('/fncdata2/encode_test_body.npy')
score_train = np.load('/fncdata2/score_train.npy')
score_test = np.load('/fncdata2/score_test.npy')
#train_b = big_mat
#train_h, dev_h, train_b, dev_b, score_train, dev_score = split(np.array(train_h), train_b, score_train, 0.2)
print 'loading training skipthoughts...'
#trainA = encoder.encode(train_h, verbose=False, use_eos=True)
#trainB = encoder.encode(train_b, verbose=False, use_eos=True)
trainA = train_h
trainB = train_b
print 'Computing development skipthoughts...'
#devA = encoder.encode(dev_h, verbose=False, use_eos=True)
#devB = encoder.encode(dev_b, verbose=False, use_eos=True)
# devA = dev_h
# devB = dev_b
devA = test_h
devB = test_b
dev_score = score_test
print 'Computing feature combinations...'
trainF = np.c_[np.abs(trainA - trainB), trainA * trainB]
devF = np.c_[np.abs(devA - devB), devA * devB]
print 'Encoding labels...'
#trainY = encode_labels(train_labels)
#devY = encode_labels(holdout_labels)
trainY = to_categorical(score_train, 4)
devY = to_categorical(dev_score, 4)
train_Fx, test_Fx = load_features()
#fmodel = generate_feature_model(train_Fx, score_train, test_Fx, dev_score, ninputs=len(train_Fx[0]))
train_tfidf, test_tfidf = generate_tfidf()
print 'Compiling model...'
lrmodel = prepare_model(ninputs=trainF.shape[1],n_feats=train_Fx.shape[1],n_tfidf=train_tfidf.shape[1])
print 'Training...'
bestlrmodel = train_model(lrmodel, trainF, trainY, devF, devY, dev_score, train_Fx, test_Fx, train_tfidf, test_tfidf)
if evaltest:
print 'Loading test skipthoughts...'
testA = test_h
testB = test_b
print 'Computing feature combinations...'
testF = np.c_[np.abs(testA - testB), testA * testB]
yhat = bestlrmodel.predict(testF, verbose=2)
yhat = [i.argmax()for i in yhat]
string_predicted,test_stances = [],[]
for i,j in zip(yhat,score_test):
if i == 3:
string_predicted.append('unrelated')
if i == 0:
string_predicted.append('agree')
if i == 2:
string_predicted.append('discuss')
if i == 1:
string_predicted.append('disagree')
if j == 3:
test_stances.append('unrelated')
if j == 0:
test_stances.append('agree')
if j == 2:
test_stances.append('discuss')
if j == 1:
test_stances.append('disagree')
report_score(test_stances,string_predicted)
score = accuracy_score(score_test, yhat)
print 'accuracy is ..',score
# if ypp[i][1] == 1:
# a ='disagree'
# if ypp[i][2] == 1:
# a ='discuss'
# if ypp[i][3] == 1:
# a ='unrelated'
# Pr.append(a)
#
#Ac = []
#for i in range(len(ypp)):
# if y[i][0] == 1:
# a = 'agree'
# if y[i][1] == 1:
# a ='disagree'
# if y[i][2] == 1:
# a ='discuss'
# if y[i][3] == 1:
# a ='unrelated'
# Ac.append(a)
#
#ac = np.array(Ac)
#pr = np.array(Pr)
aha = score.report_score(Ac,Pr)
X_train = np.vstack(tuple([Xs[i] for i in ids]))
y_train = np.hstack(tuple([ys[i] for i in ids]))
X_test = Xs[fold]
y_test = ys[fold]
clf = GradientBoostingClassifier(n_estimators=200, random_state=14128, verbose=True)
clf.fit(X_train, y_train)
predicted = [LABELS[int(a)] for a in clf.predict(X_test)]
actual = [LABELS[int(a)] for a in y_test]
fold_score, _ = score_submission(actual, predicted)
max_fold_score, _ = score_submission(actual, actual)
score = fold_score/max_fold_score
print("Score for fold "+ str(fold) + " was - " + str(score))
if score > best_score:
best_score = score
best_fold = clf
#Run on Holdout set and report the final score on the holdout set
predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS[int(a)] for a in y_holdout]
report_score(actual,predicted)
clf.fit(X_train, y_train)
predicted = [LABELS[int(a)] for a in clf.predict(X_test)]
actual = [LABELS[int(a)] for a in y_test]
fold_score, _ = score_submission(actual, predicted)
max_fold_score, _ = score_submission(actual, actual)
score = fold_score / max_fold_score
print("Score for fold " + str(fold) + " was - " + str(score))
if score > best_score:
best_score = score
best_fold = clf
#Run on Holdout set and report the final score on the holdout set
predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS[int(a)] for a in y_holdout]
print("Scores on the dev set")
report_score(actual, predicted)
print("")
print("")
#Run on competition dataset
predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]
actual = [LABELS[int(a)] for a in y_competition]
print("Scores on the test set")
report_score(actual, predicted)
# if it is more than threshold, use mlp to make further predicrion
elif simil >= thre:
whole = [test_head + " " + test_body]
vec = count_vect.transform(whole).toarray()
appro.append(clf.predict(vec))
# return the predicted list
return appro
if __name__ == '__main__':
stoplist = stop_list("stop_list.txt")
similar_matrix, headline, articles = similarity(training_data, stoplist)
thre = train_threshold(training_data, similar_matrix, headline, articles)
print("similarity threshold:", thre)
clf, count_vect = train_related(training_data, stoplist)
appro = test(test_data, similar_matrix, clf, count_vect)
# get the actual value of test data
actual = [stance['Stance'] for stance in test_data]
count = 0
for i in range(len(actual)):
if actual[i] == appro[i]:
count += 1
print("accuracy:", count / len(actual))
report_score(actual, appro)
def run_rand_classifier(train_m, train_truth, test_m, test_truth):
pred_strs = [fnc_vect_to_label(randint(0, 3)) for _ in test_truth]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
print('Rand results:')
return report_score(test_strs, pred_strs)
if GPU_sw:
y_pred.extend(y_pred_.data.cpu().numpy())
y_target.extend(target.data.cpu().numpy())
else:
y_pred.extend(y_pred_.data.numpy())
y_target.extend(target.data.numpy())
predicted = np.argmax(np.asarray(y_pred), axis=1)
actual = np.asarray(y_target)
predicted1 = [LABELS[int(a)] for a in predicted]
actual1 = [LABELS[int(a)] for a in actual]
print("Scores on the dev set")
report_score(actual1, predicted1)
print("")
print("")
X_competition = np.array(X_competition)
y_competition = np.array(y_competition)
testVal = X_competition.shape[0] % 100
testVal = X_competition.shape[0] - testVal
test = data_utils.TensorDataset(
torch.from_numpy(X_competition[:testVal]).float(),
torch.from_numpy(y_competition[:testVal]))
test_loader = data_utils.DataLoader(test,
batch_size=batch_size,
shuffle=True)
def evaluate(self, model, X_val, y_val):
# This should probably actually be in an evaluate method
pred_related, pred_stance = model.predict(X_val)
report_score([
LABELS[np.where(x == 1)[0][0]] for x in y_val['stance_prediction']
], [LABELS[np.argmax(x)] for x in pred_stance])
def run_DT_classifier(train_m,
train_truth,
test_m,
test_truth,
validate=False):
print('running DT classifier, validate=' + str(validate))
criterions = ['gini', 'entropy']
splitters = ['best', 'random']
max_features = [None, 0.5, 0.9, 'sqrt', 'log2']
max_depths = [None, 5, 10, 50, 100]
highest_fnc = 0
best_depth = None
best_criterion = 'gini'
best_features = None
best_splitter = 'best'
if validate:
split_train_m, split_train_truth, validate_m, validate_truth = get_validation_mat(
0.7, train_m, train_truth)
# split_train_truth = label_binarize(split_train_truth, classes=[0, 1, 2, 3])
for c in criterions:
for s in splitters:
for f in max_features:
for d in max_depths:
model = DecisionTreeClassifier(criterion=c,
splitter=s,
max_features=f,
max_depth=d)
clf = OneVsOneClassifier(model)
pred_labels = clf.fit(
split_train_m,
split_train_truth).predict(validate_m).tolist()
# pred_strs = fnc_one_hot_label_decode(pred_labels)
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [
fnc_vect_to_label(d) for d in validate_truth
]
fnc_s = report_score(test_strs,
pred_strs,
verbose=False)
if fnc_s > highest_fnc:
highest_fnc = fnc_s
best_criterion = c
best_splitter = s
best_features = f
best_depth = d
# train_truth = label_binarize(train_truth, classes=[0, 1, 2, 3])
model = DecisionTreeClassifier(criterion=best_criterion,
splitter=best_splitter,
max_features=best_features,
max_depth=best_depth)
clf = OneVsOneClassifier(model)
pred_labels = clf.fit(train_m, train_truth).predict(test_m).tolist()
pred_strs = [fnc_vect_to_label(d) for d in pred_labels]
test_strs = [fnc_vect_to_label(d) for d in test_truth]
print('DT results:')
f1_w = f1_score(test_strs, pred_strs, average='weighted')
f1_class = f1_score(test_strs, pred_strs, average=None)
acc = accuracy_score(test_strs, pred_strs)
print('DT f1 comp: ' + str(f1_w) + ', ' + str(f1_class))
print('DT acc: ' + str(acc))
return report_score(test_strs, pred_strs), f1_w, acc
actual = [LABELS[int(a)] for a in y_test]
fold_score, _ = score_submission(actual, predicted)
max_fold_score, _ = score_submission(actual, actual)
score = fold_score / max_fold_score
print("Score for fold " + str(fold) + " was - " + str(score))
if score > best_score:
best_score = score
best_fold = clf
#Run on Holdout set and report the final score on the holdout set
predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS[int(a)] for a in y_holdout]
print("Scores on the dev set")
report_score(actual, predicted, type='dev')
print("")
print("")
#Run on competition dataset
predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]
actual = [LABELS[int(a)] for a in y_competition]
print("Scores on the test set")
report_score(actual, predicted, type='test')
print("Break down scores")
detailed_score(actual, predicted)
verbose=False)
clf.fit(X_train, y_train)
classifier = one_stage_classifier(clf)
final, actual, score = classifier.run(X_test, y_test_true)
print("Score for fold " + str(fold) + " was - " +
str(score))
if score > best_score:
best_score = score
best_fold1 = clf
filename = model_dir + "_" + config.mode + "_" + cval[cval_ind]
pickle.dump(best_fold1, open(filename, "wb"))
classifier = one_stage_classifier(best_fold1)
final, actual, score = classifier.run(X_holdout, y_holdout)
report_score(actual, final)
fscore.append(score)
del best_fold1, best_score
else:
for fold in fold_stances:
ids = list(range(len(folds)))
del ids[fold]
X_stg1, X_stg2, y_stg1, y_stg2 = features.consolidated_features_cval(
cval_ind, fold, ids)
X_train_stg1 = X_stg1["train"]
X_test_stg1 = X_stg1["test"]
X_holdout_stg1 = X_stg1["holdout"]
y_test_true = y_stg1["true"]
y_train_stg1 = y_stg1["train"]
y_test_stg1 = y_stg1["test"]
]
init_pred_ind = [
i for i, e in enumerate(init_pred) if e == 0
]
test = np.zeros((len(init_pred_ind), n_steps, n_input))
cc = 0
for test_ind in init_pred_ind:
test[cc, 0, :] = X_h_holdout["features"][test_ind]
test[cc, 0, :] = X_b_holdout["features"][test_ind]
cc += 1
predic = sess.run(pred,
feed_dict={
x: test,
labels: to_one_hot(y_test)
})
predic_lab = [LABELS[int(a)] for a in from_one_hot(predic)]
#actual = [LABELS[int(a)] for a in np.argmax(to_one_hot(y_test),1)]
for i, e in enumerate(init_pred_ind):
base_pred[e] = predic_lab[i]
print('confusion matrix')
report_score(base_act, base_pred)
fold_score, _ = score_submission(base_act, base_pred)
max_fold_score, _ = score_submission(base_act, base_act)
score = fold_score / max_fold_score
tune_score[l_tune, lr_tune, fold] = score
print(np.amax(tune_score, axis=2))
pickle.dump(tune_score, open("finetuning.p", "wb"))
train_co_occurance)
dev_array = combine_featues(dev_vectors_array, dev_overlap_array,
tfidf_matrix_dev, dev_similarity_array,
dev_co_occurance)
test_array = combine_featues(test_vectors_array, test_overlap_array,
tfidf_matrix_test, test_similarity_array,
test_co_occurance)
###############
#B) Classifier:
predicted = classifier(train_array, labels_train, test_array)
###############
#C) Scoring:
report_score(labels_test, predicted)
###############
#D) Learning Curves:
#plot_curve(train_array, labels_train)
###############
#E) Execution time:
print("--- %s seconds ---" % (time.time() - start_time))
best_score = score
best_fold = clf
pickle.dump(best_fold, open(params.gb_weights_file, 'wb'))
best_fold = pickle.load(open(params.gb_weights_file, 'rb'))
# Run on Holdout set and report the final score on the holdout set
if params.run_2_class:
predicted = [
LABELS_RELATED[int(a)] for a in best_fold.predict(X_holdout)
]
actual = [LABELS_RELATED[int(a)] for a in y_holdout]
else:
predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]
actual = [LABELS[int(a)] for a in y_holdout]
print("Scores on the dev set")
report_score(actual, predicted)
print("")
print("")
#Run on competition dataset
predicted_combined = []
if params.run_2_class:
predicted = [
LABELS_RELATED[int(a)] for a in best_fold.predict(X_competition)
]
else:
predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]
predicted_combined = [
a if a == "unrelated" else aD
for a, aD in zip(predicted, dl_model_pred)
def MLP_Classifier(row_body_train,
row_stance_train,
row_body_test,
row_stance_test,
head_dir_train,
body_dir_train,
label_dir_train,
cos_dir_train,
head_dir_test,
body_dir_test,
label_dir_test,
cos_dir_test,
learning_rate=0.001,
batch_size=188,
training_epoch=70,
init_bias=0.001,
mode='train',
save_model_path='../tf_model/tfidf_5000_epoch',
holdout=False):
"""
:param row_head_train: 원본 head train 파일이 있는 경로
:param row_stance_train: 원본 stance train 파일이 있는 경로
:param row_head_test: 원본 head test 파일이 있는 경로
:param row_stance_test: 원본 stance test 파일이 있는 경로
:param head_dir_train: head train pkl 파일이 있는 경로
:param body_dir_train: body train pkl 파일이 있는 경로
:param label_dir_train: y train label pkl 파일이 있는 경로
:param cos_dir_train: cos_tfidf_train pkl 파일이 있는 경로
:param head_dir_test: head test pkl 파일이 있는 경로
:param body_dir_test: body test pkl 파일이 있는 경로
:param label_dir_test: y test label pkl 파일이 있는 경로
:param learning_rate: 학습률 파라미터
:param cos_dir_test: cos_tfidf_test pkl 파일이 있는 경로
:param batch_size: 배치 사이트 파라미터
:param training_epoch: 학습 횟수인 epoch 파라미터
:param init_bias: bias 초기값 파라미터
:param mode: train, test 두 모드를 선택하는 파라미터
:param save_model_path: 모델이 저장될 경로를 입력하는 파라미터
:return:
"""
lr = learning_rate
batch_size = batch_size
training_epoch = training_epoch
hidden = (362, 942, 1071, 870, 318, 912, 247)
n_classes = 4
if mode == 'train':
X_train = make_tfidf_combined_feature_cos_5000_holdout(
row_body_train, row_stance_train, row_body_test, row_stance_test,
head_dir_train, body_dir_train, label_dir_train, cos_dir_train)
y_train = load_tfidf_y(label_dir_train)
n_input = X_train.shape[1]
else:
X_test = make_tfidf_combined_feature_cos_5000_holdout(
row_body_test, row_stance_test, row_body_test, row_stance_test,
head_dir_test, body_dir_test, label_dir_test, cos_dir_test)
y_test = load_tfidf_y(label_dir_test)
n_input = X_test.shape[1]
LABELS = ['agree', 'disagree', 'discuss', 'unrelated']
predictions_list = []
actual_list = []
graph = tf.Graph()
with graph.as_default():
X = tf.placeholder("float32", [None, n_input])
Y = tf.placeholder("float32", [None, n_classes])
learning_rate_tensor = tf.placeholder(tf.float32)
momentum = tf.placeholder(tf.float32)
layer1 = tf.nn.relu(
tf.add(tf.matmul(X, weight_variable('w1', [n_input, hidden[0]])),
bias_variable('b1', [hidden[0]], init_bias)))
layer2 = tf.nn.relu(
tf.add(
tf.matmul(layer1, weight_variable('w2',
[hidden[0], hidden[1]])),
bias_variable('b2', [hidden[1]], init_bias)))
layer3 = tf.nn.relu(
tf.add(
tf.matmul(layer2, weight_variable('w3',
[hidden[1], hidden[2]])),
bias_variable('b3', [hidden[2]], init_bias)))
layer4 = tf.nn.relu(
tf.add(
tf.matmul(layer3, weight_variable('w4',
[hidden[2], hidden[3]])),
bias_variable('b4', [hidden[3]], init_bias)))
layer5 = tf.nn.relu(
tf.add(
tf.matmul(layer4, weight_variable('w5',
[hidden[3], hidden[4]])),
bias_variable('b5', [hidden[4]], init_bias)))
layer6 = tf.nn.relu(
tf.add(
tf.matmul(layer5, weight_variable('w6',
[hidden[4], hidden[5]])),
bias_variable('b6', [hidden[5]], init_bias)))
layer7 = tf.nn.relu(
tf.add(
tf.matmul(layer6, weight_variable('w7',
[hidden[5], hidden[6]])),
bias_variable('b7', [hidden[6]], init_bias)))
logits = tf.add(
tf.matmul(layer7, weight_variable('out_w',
[hidden[6], n_classes])),
bias_variable('out_b', [n_classes], init_bias))
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits,
labels=Y))
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate_tensor).minimize(cost)
predictions = tf.argmax(logits, 1)
correct_prediction = tf.equal(predictions, tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver(max_to_keep=10)
model_path = save_model_path + str(training_epoch)
tf.reset_default_graph()
with tf.Session(graph=graph) as sess:
sess.run(tf.global_variables_initializer())
if mode == 'train':
print('Learning Started!')
calc_learning_rate = lr
for epoch in range(training_epoch):
print('epoch : ', epoch)
print(len(X_train))
momentum_start = 0.5
momentum_end = 0.99
i = 0
calc_momentum = momentum_start + (float(
(momentum_end - momentum_start) / training_epoch) * epoch)
if epoch > 0 and (epoch == 20 or epoch == 35 or epoch == 45):
calc_learning_rate = float(calc_learning_rate / 10.0)
batch_acc = 0.0
batch_cost = 0.0
batch_count = 1
# print(ep)
while i < len(X_train):
start = i
end = i + batch_size
batch_x = np.array(X_train[start:end])
batch_y = np.array(y_train[start:end])
c, acc, _ = sess.run(
[cost, accuracy, optimizer],
feed_dict={
X: batch_x,
Y: batch_y,
learning_rate_tensor: calc_learning_rate,
momentum: calc_momentum
})
# print('cost : ', c, ' accuracy : ', acc)
i += batch_size
batch_acc += acc
batch_cost += c
if i % (batch_size * 90) == 0:
print(i, '/', len(X_train))
print(' batch acc : ', batch_acc / batch_count)
print(' batch cost : ', batch_cost / batch_count)
batch_count += 1
if holdout:
i = 0
while i < len(X_test):
start = i
end = i + batch_size
batch_x = np.array(X_test[start:end])
batch_y = np.array(y_test[start:end])
c, acc, _ = sess.run(
[cost, accuracy, optimizer],
feed_dict={
X: batch_x,
Y: batch_y,
learning_rate_tensor: calc_learning_rate,
momentum: calc_momentum
})
# print('cost : ', c, ' accuracy : ', acc)
i += batch_size
batch_acc += acc
batch_cost += c
if i % (batch_size * 90) == 0:
print(i, '/', len(X_test))
print(' batch acc : ', batch_acc / batch_count)
print(' batch cost : ', batch_cost / batch_count)
batch_count += 1
saver.save(sess, model_path)
print('Training Finished!')
if mode == 'test':
saver.restore(sess, model_path)
print('model load finish!')
pred, acc = sess.run([predictions, accuracy],
feed_dict={
X: X_test,
Y: y_test
})
print('pred :', pred, ', acc :', acc)
report_score([LABELS[e] for e in np.argmax(y_test, 1)],
[LABELS[e] for e in pred])
