Exemple #1
0
def train(params, input_X, input_Y, key_array, model_dir):
    train_X, train_Y, val_X, val_Y = separate_train_and_val(input_X, input_Y)
    path_final, word_embeddings = None, None
    with open(model_dir + 'train_log', 'a') as timelog:
        with tf.Graph().as_default(), tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=1,
                                      intra_op_parallelism_threads=1, use_per_session_threads=True)) as sess:
            cnn, loss, train_step, sess, saver = set_up_model(sess, params, key_array)
            best_dev_loss, init_time = initial_prints(timelog, saver, sess, model_dir, val_X, val_Y, key_array, params)

            for epoch in range(params['EPOCHS']):
                cnn = epoch_train(train_X, train_Y, key_array, params, cnn, sess, train_step)
                epoch_write_statements(timelog, init_time, epoch)

                path = saver.save(sess, model_dir + 'temp_cnn_eval_epoch%i' %(epoch))
                dev_loss = cnn_eval.float_entropy(path, val_X, val_Y, key_array, params)
                timelog.write('\ndev accuracy: %g'%dev_loss)
                if dev_loss < best_dev_loss:
                    timelog.write('\nnew best model')
                    best_dev_loss = dev_loss
                    path_final, word_embeddings = return_current_state(cnn, saver, sess,
                                                    model_dir + 'cnn_final%i' %epoch)
                #early stop if accuracy drops significantly
                elif dev_loss > best_dev_loss + .02:
                    break

            timelog.write('\ntypes:' + str(type(path_final)) + str(type(word_embeddings)))
            remove_chkpt_files(epoch, model_dir)
            if (path_final == None or word_embeddings == None):
                timelog.write('failure to train, returning current state')
                path_final, word_embeddings = return_current_state(cnn, saver, sess,
                                                    model_dir + 'cnn_final%i' %epoch)

    return path_final, word_embeddings
Exemple #2
0
def train(params, input_X, input_Y, word_vec_array, model_dir):

    train_X, train_Y, val_X, val_Y = separate_train_and_val(input_X, input_Y)
    path_final, word_embeddings = None, None
    with open(model_dir + 'train_log', 'a') as timelog:
        # with tf.Graph().as_default(), tf.Session() as sess:
        cnn, loss, train_step, saver = set_up_model(params, word_vec_array)
        best_dev_loss, init_time = initial_prints(timelog, saver, cnn.sess,
                                                  model_dir, val_X, val_Y,
                                                  word_vec_array, params)

        for epoch in range(params['EPOCHS']):
            cnn = epoch_train(train_X, train_Y, word_vec_array, params, cnn,
                              train_step, timelog)
            epoch_write_statements(timelog, init_time, epoch)

            start_save_time = time.time()
            path = saver.save(cnn.sess,
                              model_dir + 'temp_cnn_eval_epoch%i' % (epoch))
            total_save_time = time.time() - start_save_time
            timelog.write(
                "\nthe amount of time it takes to save the model at epoch " +
                str(epoch) + ": %g" % (total_save_time))

            float_entropy_init_time = time.time()
            dev_loss = cnn_eval.float_entropy(path, val_X, val_Y,
                                              word_vec_array, params)
            float_entropy_time = time.time() - float_entropy_init_time
            timelog.write(
                '\ndev cross entropy, best cross entropy: %g, %g  (it took %g '
                % (dev_loss, best_dev_loss, float_entropy_time) +
                'seconds to compute)')
            timelog.flush()

            start_save_best_time = time.time()
            if dev_loss < best_dev_loss:
                timelog.write('\nnew best model, saving model in ' +
                              (model_dir + 'cnn_final%i' % epoch))
                best_dev_loss = dev_loss
                path_final = saver.save(cnn.sess,
                                        model_dir + 'cnn_final%i' % epoch)
                word_embeddings = cnn.word_embeddings.eval(session=cnn.sess)
            #early stop if accuracy drops significantly
            elif dev_loss > best_dev_loss + .02:
                break
            total_save_best_time = time.time() - start_save_best_time
            timelog.write(
                '\nhow long saving the best model at the end takes: %g' %
                (total_save_best_time))

        remove_chkpt_files(epoch, model_dir)

        if (path_final is None or word_embeddings is None):
            timelog.write('failure to train, returning current state')
            path_final = saver.save(cnn.sess,
                                    model_dir + 'cnn_final%i' % epoch)
            word_embeddings = cnn.word_embeddings.eval(session=cnn.sess)

    return path_final, word_embeddings
Exemple #3
0
def initial_prints(timelog, saver, sess, model_dir, val_X, val_Y, key_array, params):
    timelog.write('\n\n\nNew Model:')
    init_time = time.time()
    timelog.write(str(init_time))

    path = saver.save(sess, model_dir + 'temp_cnn_eval_epoch%i' %0)
    best_dev_loss = cnn_eval.float_entropy(path, val_X, val_Y, key_array, params)

    timelog.write( '\ndebug dev loss %g' %best_dev_loss)
    timelog.write('\n%g'%time.clock())
    return best_dev_loss, init_time