def train_model(model, x, y, loss_op, pred_op, train_images, train_labels): train_op = model.training(loss_op, x, FLAGS.learning_rate, FLAGS.decay_step, FLAGS.decay_factor) sess = init_session() minibatch_gen = batch_gen(FLAGS.batch_size, train_images.shape[0], max_batches=FLAGS.max_steps, replace=True) print("training model...") start_time = time.time() for minibatch in minibatch_gen: batch_images, batch_labels = train_images[minibatch], \ train_labels[minibatch] feed_dict = {x: batch_images, y: batch_labels} _, loss_values = sess.run([train_op, loss_op], feed_dict=feed_dict) if minibatch_gen.counter % 1000 == 0: cur_time = time.time() duration = (cur_time - start_time) start_time = cur_time print('Step %d (%.3f sec): loss = ' % (minibatch_gen.counter, duration) + str(loss_values)) if minibatch_gen.counter % 10000 == 0: model.save_weights(sess, FLAGS.model_dir) evaluate(sess, x, y, pred_op, train_images, train_labels, FLAGS.batch_size) model.save_weights(sess, FLAGS.model_dir)
def train_model(sess, model, x, y, train_op, loss_op, pred_op, weights, train_images, train_labels, batch_size, max_steps, model_dir, bits, stop, log=False): minibatch_gen = batch_gen(batch_size, train_images.shape[0], max_batches=max_steps, replace=True) current_acc, highest_acc = 0.0, 0.0 best_weights = model.get_weights(sess, weights) if log: print("training model...") start_time = time.time() for minibatch in minibatch_gen: batch_images, batch_labels = train_images[minibatch], \ train_labels[minibatch] feed_dict = {x: batch_images, y: batch_labels} _, loss_values = sess.run([train_op, loss_op], feed_dict=feed_dict) if minibatch_gen.counter % 1000 == 0: cur_time = time.time() duration = (cur_time - start_time) start_time = cur_time if log: print('Step %d (%.3f sec): loss = ' % (minibatch_gen.counter, duration) + str(loss_values)) model.save_weights(sess, best_weights, model_dir, num_bits=bits) current_acc = evaluate(sess, x, y, pred_op, train_images, train_labels, batch_size) if current_acc >= highest_acc: highest_acc = current_acc best_weights = model.get_weights(sess, weights) if (current_acc >= stop): if log: print("Reached stopping accuracy.") return best_weights, current_acc yield best_weights, current_acc if log: print("highest accuracy: %f" % highest_acc) yield best_weights, highest_acc
def train_CNN(X, y, inp_dim, model, weights, epochs=EPOCHS, batch_size=BATCH_SIZE): cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) print cv_object p, r, f1 = 0., 0., 0. p1, r1, f11 = 0., 0., 0. sentence_len = X.shape[1] for train_index, test_index in cv_object.split(X): if INITIALIZE_WEIGHTS_WITH == "glove": model.layers[0].set_weights([weights]) elif INITIALIZE_WEIGHTS_WITH == "random": shuffle_weights(model) else: print "ERROR!" return X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in xrange(epochs): for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] class_weights = None if SCALE_LOSS_FUN: class_weights = {} class_weights[0] = np.where(y_temp == 0)[0].shape[0]/float(len(y_temp)) class_weights[1] = np.where(y_temp == 1)[0].shape[0]/float(len(y_temp)) class_weights[2] = np.where(y_temp == 2)[0].shape[0]/float(len(y_temp)) try: y_temp = np_utils.to_categorical(y_temp, nb_classes=3) except Exception as e: print e print y_temp print x.shape, y.shape loss, acc = model.train_on_batch(x, y_temp, class_weight=class_weights) print loss, acc y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) print classification_report(y_test, y_pred) print precision_recall_fscore_support(y_test, y_pred) print y_pred p += precision_score(y_test, y_pred, average='weighted') p1 += precision_score(y_test, y_pred, average='micro') r += recall_score(y_test, y_pred, average='weighted') r1 += recall_score(y_test, y_pred, average='micro') f1 += f1_score(y_test, y_pred, average='weighted') f11 += f1_score(y_test, y_pred, average='micro') print "macro results are" print "average precision is %f" %(p/NO_OF_FOLDS) print "average recall is %f" %(r/NO_OF_FOLDS) print "average f1 is %f" %(f1/NO_OF_FOLDS) print "micro results are" print "average precision is %f" %(p1/NO_OF_FOLDS) print "average recall is %f" %(r1/NO_OF_FOLDS) print "average f1 is %f" %(f11/NO_OF_FOLDS)
def train_CNN(X, y, inp_dim, model, epochs=EPOCHS, batch_size=BATCH_SIZE): cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) print(cv_object) p, r, f1 = 0., 0., 0. p1, r1, f11 = 0., 0., 0. sentence_len = X.shape[1] for train_index, test_index in cv_object.split(X): X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in range(epochs): for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] if len(y_temp) >= 1: class_weights = None if SCALE_LOSS_FUN: class_weights = {} for cw in range(len(set(y))): class_weights[cw] = np.where( y_temp == cw)[0].shape[0] / float(len(y_temp)) try: y_temp = np_utils.to_categorical(y_temp, num_classes=len( set(y))) except Exception as e: print(e) print(y_temp) #print(x.shape, y.shape) loss, acc = model.train_on_batch( x, y_temp, class_weight=class_weights) y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) #print(classification_report(y_test, y_pred)) #print(precision_recall_fscore_support(y_test, y_pred)) # print(y_pred) p += precision_score(y_test, y_pred, average='weighted') p1 += precision_score(y_test, y_pred, average='micro') r += recall_score(y_test, y_pred, average='weighted') r1 += recall_score(y_test, y_pred, average='micro') f1 += f1_score(y_test, y_pred, average='weighted') f11 += f1_score(y_test, y_pred, average='micro') print("macro results are") print("average precision is %f" % (p / NO_OF_FOLDS)) print("average recall is %f" % (r / NO_OF_FOLDS)) print("average f1 is %f" % (f1 / NO_OF_FOLDS)) print("micro results are") print("average precision is %f" % (p1 / NO_OF_FOLDS)) print("average recall is %f" % (r1 / NO_OF_FOLDS)) print("average f1 is %f" % (f11 / NO_OF_FOLDS)) return ((p / NO_OF_FOLDS), (r / NO_OF_FOLDS), (f1 / NO_OF_FOLDS))
def train_fasttext(X, y, model, inp_dim, embedding_weights, epochs=10, batch_size=128): cv_object = KFold(n_splits=10, shuffle=True, random_state=42) print cv_object p, r, f1 = 0., 0., 0. sentence_len = X.shape[1] lookup_table = np.zeros_like(model.layers[0].get_weights()[0]) for train_index, test_index in cv_object.split(X): shuffle_weights(model) #pdb.set_trace() #model.layers[0].set_weights([embedding_weights]) X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in xrange(epochs): for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] class_weights = {} class_weights[0] = np.where(y_temp == 0)[0].shape[0] / float( len(y_temp)) class_weights[1] = np.where(y_temp == 1)[0].shape[0] / float( len(y_temp)) class_weights[2] = np.where(y_temp == 2)[0].shape[0] / float( len(y_temp)) try: y_temp = np_utils.to_categorical(y_temp, nb_classes=3) except Exception as e: print e #print x.shape, y.shape loss, acc = model.train_on_batch( x, y_temp) #, class_weight=class_weights) print loss, acc #pdb.set_trace() lookup_table += model.layers[0].get_weights()[0] y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) print classification_report(y_test, y_pred) print precision_recall_fscore_support(y_test, y_pred) print y_pred p += precision_score(y_test, y_pred, average='weighted') r += recall_score(y_test, y_pred, average='weighted') f1 += f1_score(y_test, y_pred, average='weighted') print "macro results are" print "average precision is %f" % (p / 10) print "average recall is %f" % (r / 10) print "average f1 is %f" % (f1 / 10) return lookup_table / float(10)
def train_CNN(X, y, inp_dim, model, weights, epochs=EPOCHS, batch_size=BATCH_SIZE): cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) print(cv_object) p, r, f1 = [], [], [] p1, r1, f11 = 0., 0., 0. p_class, r_class, f1_class = [], [], [] sentence_len = X.shape[1] marcro_f1, macro_r, macro_p = [], [], [] precision_scores = [] recall_scores = [] f1_scores = [] for train_index, test_index in cv_object.split(X): if INITIALIZE_WEIGHTS_WITH == "word2vec": model.layers[0].set_weights([weights]) elif INITIALIZE_WEIGHTS_WITH == "random": shuffle_weights(model) else: print("ERROR!") return X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in range(epochs): for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] class_weights = None if SCALE_LOSS_FUN: class_weights = {} for cw in range(len(set(tx_class))): class_weights[cw] = np.where( y_temp == cw)[0].shape[0] / float(len(y_temp)) try: y_temp = np_utils.to_categorical(y_temp, num_classes=len( set(tx_class))) except Exception as e: print(e) print(y_temp) #print(x.shape, y.shape) loss, acc = model.train_on_batch(x, y_temp, class_weight=class_weights) y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) #print(classification_report(y_test, y_pred)) #print(precision_recall_fscore_support(y_test, y_pred)) #print(y_pred) p.append(precision_score(y_test, y_pred, average='weighted')) p1 += precision_score(y_test, y_pred, average='micro') p_class.append(precision_score(y_test, y_pred, average=None)) r.append(recall_score(y_test, y_pred, average='weighted')) r1 += recall_score(y_test, y_pred, average='micro') r_class.append(recall_score(y_test, y_pred, average=None)) f1.append(f1_score(y_test, y_pred, average='weighted')) f11 += f1_score(y_test, y_pred, average='micro') f1_class.append(f1_score(y_test, y_pred, average=None)) macro_p.append(precision_score(y_test, y_pred, average='macro')) macro_r.append(recall_score(y_test, y_pred, average='macro')) marcro_f1.append(f1_score(y_test, y_pred, average='macro')) print("macro results are") print("average precision is %f" % (np.array(p).mean())) print("average recall is %f" % (np.array(r).mean())) print("average f1 is %f" % (np.array(f1).mean())) save_report_to_csv( REPORT_FOLDER + 'CNN_training_report.csv', [ 'CNN', get_model_name_by_file(POLITICS_FILE), #weighted scores np.array(p).mean(), np.array(p).std() * 2, np.array(r).mean(), np.array(r).std() * 2, np.array(f1).mean(), np.array(f1).std() * 2, #macro scores np.array(macro_p).mean(), np.array(macro_p).std() * 2, np.array(macro_r).mean(), np.array(macro_r).std() * 2, np.array(marcro_f1).mean(), np.array(marcro_f1).std() * 2, #by class scores np.array(np.array(p_class)[:, 0]).mean(), np.array(np.array(p_class)[:, 1]).mean(), np.array(np.array(r_class)[:, 0]).mean(), np.array(np.array(r_class)[:, 1]).mean(), np.array(np.array(f1_class)[:, 0]).mean(), np.array(np.array(f1_class)[:, 1]).mean() ]) print("micro results are") print("average precision is %f" % (p1 / NO_OF_FOLDS)) print("average recall is %f" % (r1 / NO_OF_FOLDS)) print("average f1 is %f" % (f11 / NO_OF_FOLDS))
def train_CNN(X, y, inp_dim, model, weights, epochs=EPOCHS, batch_size=BATCH_SIZE): cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) print cv_object p, r, f1 = 0., 0., 0. sentence_len = X.shape[1] for train_index, test_index in cv_object.split(X): if INITIALIZE_WEIGHTS_WITH == "glove": shuffle_weights(model) model.layers[0].set_weights([weights]) elif INITIALIZE_WEIGHTS_WITH == "random": shuffle_weights(model) else: print "ERROR!" return X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in xrange(epochs): train_loss = 0 train_acc = 0 for i, X_batch in enumerate(batch_gen(X_temp, batch_size), 1): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] # Last column will be y due to np.hstack class_weights = None if SCALE_LOSS_FUN: class_weights = {} class_weights[0] = np.where( y_temp == 0)[0].shape[0] / float(len(y_temp)) class_weights[1] = np.where( y_temp == 1)[0].shape[0] / float(len(y_temp)) class_weights[2] = np.where( y_temp == 2)[0].shape[0] / float(len(y_temp)) try: y_temp = np_utils.to_categorical(y_temp, nb_classes=3) except Exception as e: print e print y_temp _loss, _acc = model.train_on_batch(x, y_temp, class_weight=class_weights) train_loss += _loss train_acc += _acc if i % 35 == 0: print "Epoch: %d/%d.\tBatch: %d.\tLoss: %f.\tAccuracy: %f" % ( epoch, epochs, i, train_loss / i, train_acc / i) y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) print "\n", classification_report(y_test, y_pred) p += precision_score(y_test, y_pred, average='weighted') r += recall_score(y_test, y_pred, average='weighted') f1 += f1_score(y_test, y_pred, average='weighted') print "weighted results are" print "average precision is %f" % (p / NO_OF_FOLDS) print "average recall is %f" % (r / NO_OF_FOLDS) print "average f1 is %f" % (f1 / NO_OF_FOLDS)
def train_LSTM(X, y, model, inp_dim, weights, epochs=EPOCHS, batch_size=BATCH_SIZE): cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) p, r, f1 = 0., 0., 0. p1, r1, f11 = 0., 0., 0. sentence_len = X.shape[1] print(NO_OF_FOLDS) for train_index, test_index in cv_object.split(X): if INITIALIZE_WEIGHTS_WITH == "glove": model.layers[1].set_weights([weights]) elif INITIALIZE_WEIGHTS_WITH == "random": shuffle_weights(model) else: return X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in range(epochs): for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] class_weights = None if SCALE_LOSS_FUN: class_weights = {} class_weights[0] = np.where( y_temp == 0)[0].shape[0] / float(len(y_temp)) class_weights[1] = np.where( y_temp == 1)[0].shape[0] / float(len(y_temp)) try: y_temp = convert_to_one_hot(y_temp, C=2) except Exception as e: print(e) print(y_temp) # print(x.shape) # print(y_temp.shape) # print("HERE WE GO") loss, acc = model.train_on_batch(x, y_temp, class_weight=class_weights) y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) print(classification_report(y_test, y_pred)) print(precision_recall_fscore_support(y_test, y_pred)) print(y_pred) p += precision_score(y_test, y_pred, average='weighted') p1 += precision_score(y_test, y_pred, average='micro') r += recall_score(y_test, y_pred, average='weighted') r1 += recall_score(y_test, y_pred, average='micro') f1 += f1_score(y_test, y_pred, average='weighted') f11 += f1_score(y_test, y_pred, average='micro') print("macro results are") print("average precision is " + str(p / NO_OF_FOLDS)) print("average recall is " + str(r / NO_OF_FOLDS)) print("average f1 is " + str(f1 / NO_OF_FOLDS)) print("micro results are") print("average precision is " + str(p1 / NO_OF_FOLDS)) print("average recall is " + str(r1 / NO_OF_FOLDS)) print("average f1 is " + str(f11 / NO_OF_FOLDS))
def train_LSTM(X_training, y_training, X_testing, model, inp_dim, weights, epochs, batch_size): # Trains lstm model by performing K-Fold Cross Validation prec_macro, recall_macro, f1_macro = 0., 0., 0. prec_micro, recall_micro, f1_micro = 0., 0., 0. sentence_len = X_training.shape[1] cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) for train_index, val_index in cv_object.split(X_training): if INITIALIZE_WEIGHTS_WITH == "glove": model.layers[0].set_weights([weights]) elif INITIALIZE_WEIGHTS_WITH == "random": shuffle_weights(model) else: print("ERROR!") return X_train, y_train = X_training[train_index], y_training[train_index] X_val, y_val = X_training[val_index], y_training[val_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in range(epochs): loss, acc = -1, -1 for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] class_weights = None if SCALE_LOSS_FUN: class_weights = {} class_weights[0] = np.where( y_temp == 0)[0].shape[0] / float(len(y_temp)) class_weights[1] = np.where( y_temp == 1)[0].shape[0] / float(len(y_temp)) y_temp = np_utils.to_categorical(y_temp) loss, acc = model.train_on_batch(x, y_temp, class_weight=class_weights) print("Epoch: {}\tLoss: {}\tAccuracy: {}".format( epoch, round(loss, 5), round(acc, 5))) y_pred = model.predict_on_batch(X_val) y_pred = np.argmax(y_pred, axis=1) print(classification_report(y_val, y_pred)) print("") prec_macro += precision_score(y_val, y_pred, average='weighted') prec_micro += precision_score(y_val, y_pred, average='micro') recall_macro += recall_score(y_val, y_pred, average='weighted') recall_micro += recall_score(y_val, y_pred, average='micro') f1_macro += f1_score(y_val, y_pred, average='weighted') f1_micro += f1_score(y_val, y_pred, average='micro') X_id = [(x["id"], x["text"]) for x in test_tweets] y_pred = model.predict_on_batch(X_testing) y_pred = np.argmax(y_pred, axis=1) writePred(list(zip(X_id, y_pred)), path='../predictions/', filename='T2.csv') print("\n\n## FINAL RESULTS ##") print("MACRO Results: ") print("Avg. Precision: %f" % (prec_macro / NO_OF_FOLDS)) print("Avg. Recall: %f" % (recall_macro / NO_OF_FOLDS)) print("Avg. Macro-F1: %f" % (f1_macro / NO_OF_FOLDS)) print("\nMICRO results: ") print("Avg. Precision: %f" % (prec_micro / NO_OF_FOLDS)) print("Avg. Recall: %f" % (recall_micro / NO_OF_FOLDS)) print("Avg. Micro-F1: %f" % (f1_micro / NO_OF_FOLDS))
def train_CNN(X, y, inp_dim, model, weights, epochs=EPOCHS, batch_size=BATCH_SIZE): cv_object = KFold(n_splits=NO_OF_FOLDS, shuffle=True, random_state=42) print(cv_object) p, r, f1 = 0., 0., 0. p1, r1, f11 = 0., 0., 0. sentence_len = X.shape[1] for train_index, test_index in cv_object.split(X): if INITIALIZE_WEIGHTS_WITH == "word2vec": model.layers[0].set_weights([weights]) elif INITIALIZE_WEIGHTS_WITH == "random": shuffle_weights(model) else: print("ERROR!") return X_train, y_train = X[train_index], y[train_index] X_test, y_test = X[test_index], y[test_index] y_train = y_train.reshape((len(y_train), 1)) X_temp = np.hstack((X_train, y_train)) for epoch in range(epochs): for X_batch in batch_gen(X_temp, batch_size): x = X_batch[:, :sentence_len] y_temp = X_batch[:, sentence_len] class_weights = None if SCALE_LOSS_FUN: class_weights = {} for cw in range(len(set(tw_class))): class_weights[cw] = np.where(y_temp == cw)[0].shape[ 0]/float(len(y_temp)) try: y_temp = np_utils.to_categorical( y_temp, num_classes=len(set(tw_class))) except Exception as e: print(e) print(y_temp) #print(x.shape, y.shape) loss, acc = model.train_on_batch( x, y_temp, class_weight=class_weights) y_pred = model.predict_on_batch(X_test) y_pred = np.argmax(y_pred, axis=1) print(classification_report(y_test, y_pred)) print(precision_recall_fscore_support(y_test, y_pred)) print(y_pred) p += precision_score(y_test, y_pred, average='weighted') p1 += precision_score(y_test, y_pred, average='micro') r += recall_score(y_test, y_pred, average='weighted') r1 += recall_score(y_test, y_pred, average='micro') f1 += f1_score(y_test, y_pred, average='weighted') f11 += f1_score(y_test, y_pred, average='micro') print("macro results are") print("average precision is %f" % (p / NO_OF_FOLDS)) print("average recall is %f" % (r / NO_OF_FOLDS)) print("average f1 is %f" % (f1 / NO_OF_FOLDS)) print("micro results are") print("average precision is %f" % (p1 / NO_OF_FOLDS)) print("average recall is %f" % (r1 / NO_OF_FOLDS)) print("average f1 is %f" % (f11 / NO_OF_FOLDS)) return ((p / NO_OF_FOLDS), (r / NO_OF_FOLDS), (f1 / NO_OF_FOLDS))