def train(params, input_X, input_Y, key_array, model_dir): train_X, train_Y, val_X, val_Y = separate_train_and_val(input_X, input_Y) path_final, word_embeddings = None, None with open(model_dir + 'train_log', 'a') as timelog: with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1, use_per_session_threads=True)) as sess: cnn, loss, train_step, sess, saver = set_up_model(sess, params, key_array) best_dev_loss, init_time = initial_prints(timelog, saver, sess, model_dir, val_X, val_Y, key_array, params) for epoch in range(params['EPOCHS']): cnn = epoch_train(train_X, train_Y, key_array, params, cnn, sess, train_step) epoch_write_statements(timelog, init_time, epoch) path = saver.save(sess, model_dir + 'temp_cnn_eval_epoch%i' %(epoch)) dev_loss = cnn_eval.float_entropy(path, val_X, val_Y, key_array, params) timelog.write('\ndev accuracy: %g'%dev_loss) if dev_loss < best_dev_loss: timelog.write('\nnew best model') best_dev_loss = dev_loss path_final, word_embeddings = return_current_state(cnn, saver, sess, model_dir + 'cnn_final%i' %epoch) #early stop if accuracy drops significantly elif dev_loss > best_dev_loss + .02: break timelog.write('\ntypes:' + str(type(path_final)) + str(type(word_embeddings))) remove_chkpt_files(epoch, model_dir) if (path_final == None or word_embeddings == None): timelog.write('failure to train, returning current state') path_final, word_embeddings = return_current_state(cnn, saver, sess, model_dir + 'cnn_final%i' %epoch) return path_final, word_embeddings
def train(params, input_X, input_Y, word_vec_array, model_dir): train_X, train_Y, val_X, val_Y = separate_train_and_val(input_X, input_Y) path_final, word_embeddings = None, None with open(model_dir + 'train_log', 'a') as timelog: # with tf.Graph().as_default(), tf.Session() as sess: cnn, loss, train_step, saver = set_up_model(params, word_vec_array) best_dev_loss, init_time = initial_prints(timelog, saver, cnn.sess, model_dir, val_X, val_Y, word_vec_array, params) for epoch in range(params['EPOCHS']): cnn = epoch_train(train_X, train_Y, word_vec_array, params, cnn, train_step, timelog) epoch_write_statements(timelog, init_time, epoch) start_save_time = time.time() path = saver.save(cnn.sess, model_dir + 'temp_cnn_eval_epoch%i' % (epoch)) total_save_time = time.time() - start_save_time timelog.write( "\nthe amount of time it takes to save the model at epoch " + str(epoch) + ": %g" % (total_save_time)) float_entropy_init_time = time.time() dev_loss = cnn_eval.float_entropy(path, val_X, val_Y, word_vec_array, params) float_entropy_time = time.time() - float_entropy_init_time timelog.write( '\ndev cross entropy, best cross entropy: %g, %g (it took %g ' % (dev_loss, best_dev_loss, float_entropy_time) + 'seconds to compute)') timelog.flush() start_save_best_time = time.time() if dev_loss < best_dev_loss: timelog.write('\nnew best model, saving model in ' + (model_dir + 'cnn_final%i' % epoch)) best_dev_loss = dev_loss path_final = saver.save(cnn.sess, model_dir + 'cnn_final%i' % epoch) word_embeddings = cnn.word_embeddings.eval(session=cnn.sess) #early stop if accuracy drops significantly elif dev_loss > best_dev_loss + .02: break total_save_best_time = time.time() - start_save_best_time timelog.write( '\nhow long saving the best model at the end takes: %g' % (total_save_best_time)) remove_chkpt_files(epoch, model_dir) if (path_final is None or word_embeddings is None): timelog.write('failure to train, returning current state') path_final = saver.save(cnn.sess, model_dir + 'cnn_final%i' % epoch) word_embeddings = cnn.word_embeddings.eval(session=cnn.sess) return path_final, word_embeddings
def initial_prints(timelog, saver, sess, model_dir, val_X, val_Y, key_array, params): timelog.write('\n\n\nNew Model:') init_time = time.time() timelog.write(str(init_time)) path = saver.save(sess, model_dir + 'temp_cnn_eval_epoch%i' %0) best_dev_loss = cnn_eval.float_entropy(path, val_X, val_Y, key_array, params) timelog.write( '\ndebug dev loss %g' %best_dev_loss) timelog.write('\n%g'%time.clock()) return best_dev_loss, init_time