def train_model(model, x, y, loss_op, pred_op, train_images, train_labels):
train_op = model.training(loss_op, x, FLAGS.learning_rate,
FLAGS.decay_step, FLAGS.decay_factor)
sess = init_session()
minibatch_gen = batch_gen(FLAGS.batch_size,
train_images.shape[0],
max_batches=FLAGS.max_steps,
replace=True)
print("training model...")
start_time = time.time()
for minibatch in minibatch_gen:
batch_images, batch_labels = train_images[minibatch], \
train_labels[minibatch]
feed_dict = {x: batch_images, y: batch_labels}
_, loss_values = sess.run([train_op, loss_op], feed_dict=feed_dict)
if minibatch_gen.counter % 1000 == 0:
cur_time = time.time()
duration = (cur_time - start_time)
start_time = cur_time
print('Step %d (%.3f sec): loss = ' %
(minibatch_gen.counter, duration) + str(loss_values))
if minibatch_gen.counter % 10000 == 0:
model.save_weights(sess, FLAGS.model_dir)
evaluate(sess, x, y, pred_op, train_images, train_labels,
FLAGS.batch_size)
model.save_weights(sess, FLAGS.model_dir)
def train_model(sess,
model,
x,
y,
train_op,
loss_op,
pred_op,
weights,
train_images,
train_labels,
batch_size,
max_steps,
model_dir,
bits,
stop,
log=False):
minibatch_gen = batch_gen(batch_size,
train_images.shape[0],
max_batches=max_steps,
replace=True)
current_acc, highest_acc = 0.0, 0.0
best_weights = model.get_weights(sess, weights)
if log: print("training model...")
start_time = time.time()
for minibatch in minibatch_gen:
batch_images, batch_labels = train_images[minibatch], \
train_labels[minibatch]
feed_dict = {x: batch_images, y: batch_labels}
_, loss_values = sess.run([train_op, loss_op], feed_dict=feed_dict)
if minibatch_gen.counter % 1000 == 0:
cur_time = time.time()
duration = (cur_time - start_time)
start_time = cur_time
if log:
print('Step %d (%.3f sec): loss = ' %
(minibatch_gen.counter, duration) + str(loss_values))
model.save_weights(sess, best_weights, model_dir, num_bits=bits)
current_acc = evaluate(sess, x, y, pred_op, train_images,
train_labels, batch_size)
if current_acc >= highest_acc:
highest_acc = current_acc
best_weights = model.get_weights(sess, weights)
if (current_acc >= stop):
if log: print("Reached stopping accuracy.")
return best_weights, current_acc
yield best_weights, current_acc
if log: print("highest accuracy: %f" % highest_acc)
yield best_weights, highest_acc
def train_CNN(X, y, inp_dim, model, weights, epochs=EPOCHS, batch_size=BATCH_SIZE):
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
print cv_object
p, r, f1 = 0., 0., 0.
p1, r1, f11 = 0., 0., 0.
sentence_len = X.shape[1]
for train_index, test_index in cv_object.split(X):
if INITIALIZE_WEIGHTS_WITH == "glove":
model.layers[0].set_weights([weights])
elif INITIALIZE_WEIGHTS_WITH == "random":
shuffle_weights(model)
else:
print "ERROR!"
return
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in xrange(epochs):
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
class_weights[0] = np.where(y_temp == 0)[0].shape[0]/float(len(y_temp))
class_weights[1] = np.where(y_temp == 1)[0].shape[0]/float(len(y_temp))
class_weights[2] = np.where(y_temp == 2)[0].shape[0]/float(len(y_temp))
try:
y_temp = np_utils.to_categorical(y_temp, nb_classes=3)
except Exception as e:
print e
print y_temp
print x.shape, y.shape
loss, acc = model.train_on_batch(x, y_temp, class_weight=class_weights)
print loss, acc
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
print classification_report(y_test, y_pred)
print precision_recall_fscore_support(y_test, y_pred)
print y_pred
p += precision_score(y_test, y_pred, average='weighted')
p1 += precision_score(y_test, y_pred, average='micro')
r += recall_score(y_test, y_pred, average='weighted')
r1 += recall_score(y_test, y_pred, average='micro')
f1 += f1_score(y_test, y_pred, average='weighted')
f11 += f1_score(y_test, y_pred, average='micro')
print "macro results are"
print "average precision is %f" %(p/NO_OF_FOLDS)
print "average recall is %f" %(r/NO_OF_FOLDS)
print "average f1 is %f" %(f1/NO_OF_FOLDS)
print "micro results are"
print "average precision is %f" %(p1/NO_OF_FOLDS)
print "average recall is %f" %(r1/NO_OF_FOLDS)
print "average f1 is %f" %(f11/NO_OF_FOLDS)
def train_CNN(X, y, inp_dim, model, epochs=EPOCHS, batch_size=BATCH_SIZE):
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
print(cv_object)
p, r, f1 = 0., 0., 0.
p1, r1, f11 = 0., 0., 0.
sentence_len = X.shape[1]
for train_index, test_index in cv_object.split(X):
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in range(epochs):
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
if len(y_temp) >= 1:
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
for cw in range(len(set(y))):
class_weights[cw] = np.where(
y_temp == cw)[0].shape[0] / float(len(y_temp))
try:
y_temp = np_utils.to_categorical(y_temp,
num_classes=len(
set(y)))
except Exception as e:
print(e)
print(y_temp)
#print(x.shape, y.shape)
loss, acc = model.train_on_batch(
x, y_temp, class_weight=class_weights)
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
#print(classification_report(y_test, y_pred))
#print(precision_recall_fscore_support(y_test, y_pred))
# print(y_pred)
p += precision_score(y_test, y_pred, average='weighted')
p1 += precision_score(y_test, y_pred, average='micro')
r += recall_score(y_test, y_pred, average='weighted')
r1 += recall_score(y_test, y_pred, average='micro')
f1 += f1_score(y_test, y_pred, average='weighted')
f11 += f1_score(y_test, y_pred, average='micro')
print("macro results are")
print("average precision is %f" % (p / NO_OF_FOLDS))
print("average recall is %f" % (r / NO_OF_FOLDS))
print("average f1 is %f" % (f1 / NO_OF_FOLDS))
print("micro results are")
print("average precision is %f" % (p1 / NO_OF_FOLDS))
print("average recall is %f" % (r1 / NO_OF_FOLDS))
print("average f1 is %f" % (f11 / NO_OF_FOLDS))
return ((p / NO_OF_FOLDS), (r / NO_OF_FOLDS), (f1 / NO_OF_FOLDS))
def train_fasttext(X,
y,
model,
inp_dim,
embedding_weights,
epochs=10,
batch_size=128):
cv_object = KFold(n_splits=10, shuffle=True, random_state=42)
print cv_object
p, r, f1 = 0., 0., 0.
sentence_len = X.shape[1]
lookup_table = np.zeros_like(model.layers[0].get_weights()[0])
for train_index, test_index in cv_object.split(X):
shuffle_weights(model)
#pdb.set_trace()
#model.layers[0].set_weights([embedding_weights])
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in xrange(epochs):
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
class_weights = {}
class_weights[0] = np.where(y_temp == 0)[0].shape[0] / float(
len(y_temp))
class_weights[1] = np.where(y_temp == 1)[0].shape[0] / float(
len(y_temp))
class_weights[2] = np.where(y_temp == 2)[0].shape[0] / float(
len(y_temp))
try:
y_temp = np_utils.to_categorical(y_temp, nb_classes=3)
except Exception as e:
print e
#print x.shape, y.shape
loss, acc = model.train_on_batch(
x, y_temp) #, class_weight=class_weights)
print loss, acc
#pdb.set_trace()
lookup_table += model.layers[0].get_weights()[0]
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
print classification_report(y_test, y_pred)
print precision_recall_fscore_support(y_test, y_pred)
print y_pred
p += precision_score(y_test, y_pred, average='weighted')
r += recall_score(y_test, y_pred, average='weighted')
f1 += f1_score(y_test, y_pred, average='weighted')
print "macro results are"
print "average precision is %f" % (p / 10)
print "average recall is %f" % (r / 10)
print "average f1 is %f" % (f1 / 10)
return lookup_table / float(10)
def train_CNN(X,
y,
inp_dim,
model,
weights,
epochs=EPOCHS,
batch_size=BATCH_SIZE):
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
print(cv_object)
p, r, f1 = [], [], []
p1, r1, f11 = 0., 0., 0.
p_class, r_class, f1_class = [], [], []
sentence_len = X.shape[1]
marcro_f1, macro_r, macro_p = [], [], []
precision_scores = []
recall_scores = []
f1_scores = []
for train_index, test_index in cv_object.split(X):
if INITIALIZE_WEIGHTS_WITH == "word2vec":
model.layers[0].set_weights([weights])
elif INITIALIZE_WEIGHTS_WITH == "random":
shuffle_weights(model)
else:
print("ERROR!")
return
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in range(epochs):
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
for cw in range(len(set(tx_class))):
class_weights[cw] = np.where(
y_temp == cw)[0].shape[0] / float(len(y_temp))
try:
y_temp = np_utils.to_categorical(y_temp,
num_classes=len(
set(tx_class)))
except Exception as e:
print(e)
print(y_temp)
#print(x.shape, y.shape)
loss, acc = model.train_on_batch(x,
y_temp,
class_weight=class_weights)
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
#print(classification_report(y_test, y_pred))
#print(precision_recall_fscore_support(y_test, y_pred))
#print(y_pred)
p.append(precision_score(y_test, y_pred, average='weighted'))
p1 += precision_score(y_test, y_pred, average='micro')
p_class.append(precision_score(y_test, y_pred, average=None))
r.append(recall_score(y_test, y_pred, average='weighted'))
r1 += recall_score(y_test, y_pred, average='micro')
r_class.append(recall_score(y_test, y_pred, average=None))
f1.append(f1_score(y_test, y_pred, average='weighted'))
f11 += f1_score(y_test, y_pred, average='micro')
f1_class.append(f1_score(y_test, y_pred, average=None))
macro_p.append(precision_score(y_test, y_pred, average='macro'))
macro_r.append(recall_score(y_test, y_pred, average='macro'))
marcro_f1.append(f1_score(y_test, y_pred, average='macro'))
print("macro results are")
print("average precision is %f" % (np.array(p).mean()))
print("average recall is %f" % (np.array(r).mean()))
print("average f1 is %f" % (np.array(f1).mean()))
save_report_to_csv(
REPORT_FOLDER + 'CNN_training_report.csv',
[
'CNN',
get_model_name_by_file(POLITICS_FILE),
#weighted scores
np.array(p).mean(),
np.array(p).std() * 2,
np.array(r).mean(),
np.array(r).std() * 2,
np.array(f1).mean(),
np.array(f1).std() * 2,
#macro scores
np.array(macro_p).mean(),
np.array(macro_p).std() * 2,
np.array(macro_r).mean(),
np.array(macro_r).std() * 2,
np.array(marcro_f1).mean(),
np.array(marcro_f1).std() * 2,
#by class scores
np.array(np.array(p_class)[:, 0]).mean(),
np.array(np.array(p_class)[:, 1]).mean(),
np.array(np.array(r_class)[:, 0]).mean(),
np.array(np.array(r_class)[:, 1]).mean(),
np.array(np.array(f1_class)[:, 0]).mean(),
np.array(np.array(f1_class)[:, 1]).mean()
])
print("micro results are")
print("average precision is %f" % (p1 / NO_OF_FOLDS))
print("average recall is %f" % (r1 / NO_OF_FOLDS))
print("average f1 is %f" % (f11 / NO_OF_FOLDS))
def train_CNN(X,
y,
inp_dim,
model,
weights,
epochs=EPOCHS,
batch_size=BATCH_SIZE):
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
print cv_object
p, r, f1 = 0., 0., 0.
sentence_len = X.shape[1]
for train_index, test_index in cv_object.split(X):
if INITIALIZE_WEIGHTS_WITH == "glove":
shuffle_weights(model)
model.layers[0].set_weights([weights])
elif INITIALIZE_WEIGHTS_WITH == "random":
shuffle_weights(model)
else:
print "ERROR!"
return
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in xrange(epochs):
train_loss = 0
train_acc = 0
for i, X_batch in enumerate(batch_gen(X_temp, batch_size), 1):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:,
sentence_len] # Last column will be y due to np.hstack
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
class_weights[0] = np.where(
y_temp == 0)[0].shape[0] / float(len(y_temp))
class_weights[1] = np.where(
y_temp == 1)[0].shape[0] / float(len(y_temp))
class_weights[2] = np.where(
y_temp == 2)[0].shape[0] / float(len(y_temp))
try:
y_temp = np_utils.to_categorical(y_temp, nb_classes=3)
except Exception as e:
print e
print y_temp
_loss, _acc = model.train_on_batch(x,
y_temp,
class_weight=class_weights)
train_loss += _loss
train_acc += _acc
if i % 35 == 0:
print "Epoch: %d/%d.\tBatch: %d.\tLoss: %f.\tAccuracy: %f" % (
epoch, epochs, i, train_loss / i, train_acc / i)
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
print "\n", classification_report(y_test, y_pred)
p += precision_score(y_test, y_pred, average='weighted')
r += recall_score(y_test, y_pred, average='weighted')
f1 += f1_score(y_test, y_pred, average='weighted')
print "weighted results are"
print "average precision is %f" % (p / NO_OF_FOLDS)
print "average recall is %f" % (r / NO_OF_FOLDS)
print "average f1 is %f" % (f1 / NO_OF_FOLDS)
def train_LSTM(X,
y,
model,
inp_dim,
weights,
epochs=EPOCHS,
batch_size=BATCH_SIZE):
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
p, r, f1 = 0., 0., 0.
p1, r1, f11 = 0., 0., 0.
sentence_len = X.shape[1]
print(NO_OF_FOLDS)
for train_index, test_index in cv_object.split(X):
if INITIALIZE_WEIGHTS_WITH == "glove":
model.layers[1].set_weights([weights])
elif INITIALIZE_WEIGHTS_WITH == "random":
shuffle_weights(model)
else:
return
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in range(epochs):
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
class_weights[0] = np.where(
y_temp == 0)[0].shape[0] / float(len(y_temp))
class_weights[1] = np.where(
y_temp == 1)[0].shape[0] / float(len(y_temp))
try:
y_temp = convert_to_one_hot(y_temp, C=2)
except Exception as e:
print(e)
print(y_temp)
# print(x.shape)
# print(y_temp.shape)
# print("HERE WE GO")
loss, acc = model.train_on_batch(x,
y_temp,
class_weight=class_weights)
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
print(classification_report(y_test, y_pred))
print(precision_recall_fscore_support(y_test, y_pred))
print(y_pred)
p += precision_score(y_test, y_pred, average='weighted')
p1 += precision_score(y_test, y_pred, average='micro')
r += recall_score(y_test, y_pred, average='weighted')
r1 += recall_score(y_test, y_pred, average='micro')
f1 += f1_score(y_test, y_pred, average='weighted')
f11 += f1_score(y_test, y_pred, average='micro')
print("macro results are")
print("average precision is " + str(p / NO_OF_FOLDS))
print("average recall is " + str(r / NO_OF_FOLDS))
print("average f1 is " + str(f1 / NO_OF_FOLDS))
print("micro results are")
print("average precision is " + str(p1 / NO_OF_FOLDS))
print("average recall is " + str(r1 / NO_OF_FOLDS))
print("average f1 is " + str(f11 / NO_OF_FOLDS))
def train_LSTM(X_training, y_training, X_testing, model, inp_dim, weights,
epochs, batch_size):
# Trains lstm model by performing K-Fold Cross Validation
prec_macro, recall_macro, f1_macro = 0., 0., 0.
prec_micro, recall_micro, f1_micro = 0., 0., 0.
sentence_len = X_training.shape[1]
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
for train_index, val_index in cv_object.split(X_training):
if INITIALIZE_WEIGHTS_WITH == "glove":
model.layers[0].set_weights([weights])
elif INITIALIZE_WEIGHTS_WITH == "random":
shuffle_weights(model)
else:
print("ERROR!")
return
X_train, y_train = X_training[train_index], y_training[train_index]
X_val, y_val = X_training[val_index], y_training[val_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in range(epochs):
loss, acc = -1, -1
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
class_weights[0] = np.where(
y_temp == 0)[0].shape[0] / float(len(y_temp))
class_weights[1] = np.where(
y_temp == 1)[0].shape[0] / float(len(y_temp))
y_temp = np_utils.to_categorical(y_temp)
loss, acc = model.train_on_batch(x,
y_temp,
class_weight=class_weights)
print("Epoch: {}\tLoss: {}\tAccuracy: {}".format(
epoch, round(loss, 5), round(acc, 5)))
y_pred = model.predict_on_batch(X_val)
y_pred = np.argmax(y_pred, axis=1)
print(classification_report(y_val, y_pred))
print("")
prec_macro += precision_score(y_val, y_pred, average='weighted')
prec_micro += precision_score(y_val, y_pred, average='micro')
recall_macro += recall_score(y_val, y_pred, average='weighted')
recall_micro += recall_score(y_val, y_pred, average='micro')
f1_macro += f1_score(y_val, y_pred, average='weighted')
f1_micro += f1_score(y_val, y_pred, average='micro')
X_id = [(x["id"], x["text"]) for x in test_tweets]
y_pred = model.predict_on_batch(X_testing)
y_pred = np.argmax(y_pred, axis=1)
writePred(list(zip(X_id, y_pred)),
path='../predictions/',
filename='T2.csv')
print("\n\n## FINAL RESULTS ##")
print("MACRO Results: ")
print("Avg. Precision: %f" % (prec_macro / NO_OF_FOLDS))
print("Avg. Recall: %f" % (recall_macro / NO_OF_FOLDS))
print("Avg. Macro-F1: %f" % (f1_macro / NO_OF_FOLDS))
print("\nMICRO results: ")
print("Avg. Precision: %f" % (prec_micro / NO_OF_FOLDS))
print("Avg. Recall: %f" % (recall_micro / NO_OF_FOLDS))
print("Avg. Micro-F1: %f" % (f1_micro / NO_OF_FOLDS))
def train_CNN(X, y, inp_dim, model, weights, epochs=EPOCHS, batch_size=BATCH_SIZE):
cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42)
print(cv_object)
p, r, f1 = 0., 0., 0.
p1, r1, f11 = 0., 0., 0.
sentence_len = X.shape[1]
for train_index, test_index in cv_object.split(X):
if INITIALIZE_WEIGHTS_WITH == "word2vec":
model.layers[0].set_weights([weights])
elif INITIALIZE_WEIGHTS_WITH == "random":
shuffle_weights(model)
else:
print("ERROR!")
return
X_train, y_train = X[train_index], y[train_index]
X_test, y_test = X[test_index], y[test_index]
y_train = y_train.reshape((len(y_train), 1))
X_temp = np.hstack((X_train, y_train))
for epoch in range(epochs):
for X_batch in batch_gen(X_temp, batch_size):
x = X_batch[:, :sentence_len]
y_temp = X_batch[:, sentence_len]
class_weights = None
if SCALE_LOSS_FUN:
class_weights = {}
for cw in range(len(set(tw_class))):
class_weights[cw] = np.where(y_temp == cw)[0].shape[
0]/float(len(y_temp))
try:
y_temp = np_utils.to_categorical(
y_temp, num_classes=len(set(tw_class)))
except Exception as e:
print(e)
print(y_temp)
#print(x.shape, y.shape)
loss, acc = model.train_on_batch(
x, y_temp, class_weight=class_weights)
y_pred = model.predict_on_batch(X_test)
y_pred = np.argmax(y_pred, axis=1)
print(classification_report(y_test, y_pred))
print(precision_recall_fscore_support(y_test, y_pred))
print(y_pred)
p += precision_score(y_test, y_pred, average='weighted')
p1 += precision_score(y_test, y_pred, average='micro')
r += recall_score(y_test, y_pred, average='weighted')
r1 += recall_score(y_test, y_pred, average='micro')
f1 += f1_score(y_test, y_pred, average='weighted')
f11 += f1_score(y_test, y_pred, average='micro')
print("macro results are")
print("average precision is %f" % (p / NO_OF_FOLDS))
print("average recall is %f" % (r / NO_OF_FOLDS))
print("average f1 is %f" % (f1 / NO_OF_FOLDS))
print("micro results are")
print("average precision is %f" % (p1 / NO_OF_FOLDS))
print("average recall is %f" % (r1 / NO_OF_FOLDS))
print("average f1 is %f" % (f11 / NO_OF_FOLDS))
return ((p / NO_OF_FOLDS), (r / NO_OF_FOLDS), (f1 / NO_OF_FOLDS))
