def main(): parser = argparse.ArgumentParser( description='Tuning with bi-directional LSTM-CNN-CRF') parser.add_argument('--fine_tune', action='store_true', help='Fine tune the word embeddings') parser.add_argument('--embedding', choices=['word2vec', 'glove', 'senna', 'random'], help='Embedding for words', required=True) parser.add_argument('--embedding_dict', default=None, help='path for embedding dict') parser.add_argument('--batch_size', type=int, default=10, help='Number of sentences in each batch') parser.add_argument('--num_units', type=int, default=100, help='Number of hidden units in LSTM') parser.add_argument('--num_filters', type=int, default=20, help='Number of filters in CNN') parser.add_argument('--learning_rate', type=float, default=0.1, help='Learning rate') parser.add_argument('--decay_rate', type=float, default=0.1, help='Decay rate of learning rate') parser.add_argument('--grad_clipping', type=float, default=0, help='Gradient clipping') parser.add_argument('--gamma', type=float, default=1e-6, help='weight for regularization') parser.add_argument('--peepholes', action='store_true', help='Peepholes for LSTM') parser.add_argument('--oov', choices=['random', 'embedding'], help='Embedding for oov word', required=True) parser.add_argument('--update', choices=['sgd', 'momentum', 'nesterov', 'adadelta'], help='update algorithm', default='sgd') parser.add_argument('--regular', choices=['none', 'l2'], help='regularization for training', required=True) parser.add_argument('--dropout', action='store_true', help='Apply dropout layers') parser.add_argument('--patience', type=int, default=5, help='Patience for early stopping') parser.add_argument('--output_prediction', action='store_true', help='Output predictions to temp files') parser.add_argument( '--train') # "data/POS-penn/wsj/split1/wsj1.train.original" parser.add_argument( '--dev') # "data/POS-penn/wsj/split1/wsj1.dev.original" parser.add_argument( '--test') # "data/POS-penn/wsj/split1/wsj1.test.original" args = parser.parse_args() def construct_input_layer(): if fine_tune: layer_input = lasagne.layers.InputLayer(shape=(None, max_length), input_var=input_var, name='input') layer_embedding = lasagne.layers.EmbeddingLayer( layer_input, input_size=alphabet_size, output_size=embedd_dim, W=embedd_table, name='embedding') return layer_embedding else: layer_input = lasagne.layers.InputLayer(shape=(None, max_length, embedd_dim), input_var=input_var, name='input') return layer_input def construct_char_input_layer(): layer_char_input = lasagne.layers.InputLayer(shape=(None, max_sent_length, max_char_length), input_var=char_input_var, name='char-input') layer_char_input = lasagne.layers.reshape(layer_char_input, (-1, [2])) layer_char_embedding = lasagne.layers.EmbeddingLayer( layer_char_input, input_size=char_alphabet_size, output_size=char_embedd_dim, W=char_embedd_table, name='char_embedding') layer_char_input = lasagne.layers.DimshuffleLayer(layer_char_embedding, pattern=(0, 2, 1)) return layer_char_input logger = utils.get_logger("BiLSTM-CNN-CRF") fine_tune = args.fine_tune oov = args.oov regular = args.regular embedding = args.embedding embedding_path = args.embedding_dict train_path = args.train dev_path = args.dev test_path = args.test update_algo = args.update grad_clipping = args.grad_clipping peepholes = args.peepholes num_filters = args.num_filters gamma = args.gamma output_predict = args.output_prediction dropout = args.dropout X_train, Y_train, mask_train, X_dev, Y_dev, mask_dev, X_test, Y_test, mask_test, \ embedd_table, label_alphabet, \ C_train, C_dev, C_test, char_embedd_table = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, oov=oov, fine_tune=fine_tune, embedding=embedding, embedding_path=embedding_path, use_character=True) num_labels = label_alphabet.size() - 1 logger.info("constructing network...") # create variables target_var = T.imatrix(name='targets') mask_var = T.matrix(name='masks', dtype=theano.config.floatX) if fine_tune: input_var = T.imatrix(name='inputs') num_data, max_length = X_train.shape alphabet_size, embedd_dim = embedd_table.shape else: input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) num_data, max_length, embedd_dim = X_train.shape char_input_var = T.itensor3(name='char-inputs') num_data_char, max_sent_length, max_char_length = C_train.shape char_alphabet_size, char_embedd_dim = char_embedd_table.shape assert (max_length == max_sent_length) assert (num_data == num_data_char) # construct input and mask layers layer_incoming1 = construct_char_input_layer() layer_incoming2 = construct_input_layer() layer_mask = lasagne.layers.InputLayer(shape=(None, max_length), input_var=mask_var, name='mask') # construct bi-rnn-cnn num_units = args.num_units bi_lstm_cnn_crf = build_BiLSTM_CNN_CRF(layer_incoming1, layer_incoming2, num_units, num_labels, mask=layer_mask, grad_clipping=grad_clipping, peepholes=peepholes, num_filters=num_filters, dropout=dropout) logger.info("Network structure: hidden=%d, filter=%d" % (num_units, num_filters)) # compute loss num_tokens = mask_var.sum(dtype=theano.config.floatX) # get outpout of bi-lstm-cnn-crf shape [batch, length, num_labels, num_labels] energies_train = lasagne.layers.get_output(bi_lstm_cnn_crf) energies_eval = lasagne.layers.get_output(bi_lstm_cnn_crf, deterministic=True) loss_train = crf_loss(energies_train, target_var, mask_var).mean() loss_eval = crf_loss(energies_eval, target_var, mask_var).mean() # l2 regularization? if regular == 'l2': l2_penalty = lasagne.regularization.regularize_network_params( bi_lstm_cnn_crf, lasagne.regularization.l2) loss_train = loss_train + gamma * l2_penalty _, corr_train = crf_accuracy(energies_train, target_var) corr_train = (corr_train * mask_var).sum(dtype=theano.config.floatX) prediction_eval, corr_eval = crf_accuracy(energies_eval, target_var) corr_eval = (corr_eval * mask_var).sum(dtype=theano.config.floatX) # Create update expressions for training. # hyper parameters to tune: learning rate, momentum, regularization. batch_size = args.batch_size learning_rate = 1.0 if update_algo == 'adadelta' else args.learning_rate decay_rate = args.decay_rate momentum = 0.9 params = lasagne.layers.get_all_params(bi_lstm_cnn_crf, trainable=True) updates = utils.create_updates(loss_train, params, update_algo, learning_rate, momentum=momentum) # Compile a function performing a training step on a mini-batch train_fn = theano.function( [input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # Compile a second function evaluating the loss and accuracy of network eval_fn = theano.function( [input_var, target_var, mask_var, char_input_var], [loss_eval, corr_eval, num_tokens, prediction_eval]) # Finally, launch the training loop. logger.info( "Start training: %s with regularization: %s(%f), dropout: %s, fine tune: %s (#training data: %d, batch size: %d, clip: %.1f, peepholes: %s)..." \ % ( update_algo, regular, (0.0 if regular == 'none' else gamma), dropout, fine_tune, num_data, batch_size, grad_clipping, peepholes)) num_batches = num_data / batch_size num_epochs = 50 best_loss = 1e+12 best_acc = 0.0 best_epoch_loss = 0 best_epoch_acc = 0 best_loss_test_err = 0. best_loss_test_corr = 0. best_acc_test_err = 0. best_acc_test_corr = 0. stop_count = 0 lr = learning_rate patience = args.patience for epoch in range(1, num_epochs + 1): print 'Epoch %d (learning rate=%.4f, decay rate=%.4f): ' % (epoch, lr, decay_rate) logger.info('Epoch %d (learning rate=%.4f, decay rate=%.4f): ' % (epoch, lr, decay_rate)) train_err = 0.0 train_corr = 0.0 train_total = 0 train_inst = 0 start_time = time.time() num_back = 0 train_batches = 0 for batch in utils.iterate_minibatches(X_train, Y_train, masks=mask_train, char_inputs=C_train, batch_size=batch_size, shuffle=True): inputs, targets, masks, char_inputs = batch err, corr, num = train_fn(inputs, targets, masks, char_inputs) train_err += err * inputs.shape[0] train_corr += corr train_total += num train_inst += inputs.shape[0] train_batches += 1 time_ave = (time.time() - start_time) / train_batches time_left = (num_batches - train_batches) * time_ave # update log #sys.stdout.write("\b" * num_back) log_info = 'train: %d/%d loss: %.4f, acc: %.2f%%, time left (estimated): %.2fs' % ( min(train_batches * batch_size, num_data), num_data, train_err / train_inst, train_corr * 100 / train_total, time_left) #sys.stdout.write(log_info) num_back = len(log_info) logger.info(log_info) # update training log after each epoch assert train_inst == num_data # sys.stdout.write("\b" * num_back) # print 'train: %d/%d loss: %.4f, acc: %.2f%%, time: %.2fs' % ( # min(train_batches * batch_size, num_data), num_data, # train_err / num_data, train_corr * 100 / train_total, time.time() - start_time) logger.info('train: %d/%d loss: %.4f, acc: %.2f%%, time: %.2fs' % (min(train_batches * batch_size, num_data), num_data, train_err / num_data, train_corr * 100 / train_total, time.time() - start_time)) #evaluate performance on dev data dev_err = 0.0 dev_corr = 0.0 dev_total = 0 dev_inst = 0 for batch in utils.iterate_minibatches(X_dev, Y_dev, masks=mask_dev, char_inputs=C_dev, batch_size=batch_size): inputs, targets, masks, char_inputs = batch err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) dev_err += err * inputs.shape[0] dev_corr += corr dev_total += num dev_inst += inputs.shape[0] if output_predict: utils.output_predictions(predictions, targets, masks, 'tmp3/dev%d' % epoch, label_alphabet, is_flattened=False) # print 'dev loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( # dev_err / dev_inst, dev_corr, dev_total, dev_corr * 100 / dev_total) logger.info('dev loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (dev_err / dev_inst, dev_corr, dev_total, dev_corr * 100 / dev_total)) logger.info( 'dev_err: %.4f, best_loss: %.4f, best_acc: %.4f, dev_corr: %.4f, dev_total: %.4f, (dev_corr/dev_total): %.4f' % (dev_err, best_loss, best_acc, dev_corr, dev_total, dev_corr / dev_total)) if best_loss < dev_err and best_acc > dev_corr / dev_total: stop_count += 1 else: update_loss = False update_acc = False stop_count = 0 if best_loss > dev_err: update_loss = True best_loss = dev_err best_epoch_loss = epoch if best_acc < dev_corr / dev_total: update_acc = True best_acc = dev_corr / dev_total best_epoch_acc = epoch # # evaluate on test data when better performance detected # test_err = 0.0 # test_corr = 0.0 # test_total = 0 # test_inst = 0 # for batch in utils.iterate_minibatches(X_test, Y_test, masks=mask_test, char_inputs=C_test, # batch_size=batch_size): # inputs, targets, masks, char_inputs = batch # err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) # test_err += err * inputs.shape[0] # test_corr += corr # test_total += num # test_inst += inputs.shape[0] # if output_predict: # utils.output_predictions(predictions, targets, masks, 'tmp3/test%d' % epoch, label_alphabet, # is_flattened=False) # # print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( # # test_err / test_inst, test_corr, test_total, test_corr * 100 / test_total) # logger.info('test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( # test_err / test_inst, test_corr, test_total, test_corr * 100 / test_total)) # if update_loss: # best_loss_test_err = test_err # best_loss_test_corr = test_corr # if update_acc: # best_acc_test_err = test_err # best_acc_test_corr = test_corr logger.info('stop_count: %.4f' % (stop_count)) # stop if dev acc decrease 3 time straightly. if stop_count == patience: break # re-compile a function with new learning rate for training if update_algo != 'adadelta': lr = learning_rate / (1.0 + epoch * decay_rate) updates = utils.create_updates(loss_train, params, update_algo, lr, momentum=momentum) train_fn = theano.function( [input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # evaluate on test data when better performance detected test_err = 0.0 test_corr = 0.0 test_total = 0 test_inst = 0 for batch in utils.iterate_minibatches(X_test, Y_test, masks=mask_test, char_inputs=C_test, batch_size=batch_size): inputs, targets, masks, char_inputs = batch err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) test_err += err * inputs.shape[0] test_corr += corr test_total += num test_inst += inputs.shape[0] if output_predict: utils.output_predictions(predictions, targets, masks, 'tmp4/test%d' % epoch, label_alphabet, is_flattened=False) # print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( # test_err / test_inst, test_corr, test_total, test_corr * 100 / test_total) logger.info('test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (test_err / test_inst, test_corr, test_total, test_corr * 100 / test_total)) if update_loss: best_loss_test_err = test_err best_loss_test_corr = test_corr if update_acc: best_acc_test_err = test_err best_acc_test_corr = test_corr # print best performance on test data. logger.info("final best loss test performance (at epoch %d)" % best_epoch_loss) logger.info("final best acc test performance (at epoch %d)" % best_epoch_acc) # print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( # best_loss_test_err / test_inst, best_loss_test_corr, test_total, best_loss_test_corr * 100 / test_total) logger.info('test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (best_loss_test_err / test_inst, best_loss_test_corr, test_total, best_loss_test_corr * 100 / test_total)) # print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( # best_acc_test_err / test_inst, best_acc_test_corr, test_total, best_acc_test_corr * 100 / test_total) logger.info('test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % (best_acc_test_err / test_inst, best_acc_test_corr, test_total, best_acc_test_corr * 100 / test_total))
def main(): parser = argparse.ArgumentParser(description='Tuning with bi-directional LSTM-CNN-CRF') parser.add_argument('--fine_tune', action='store_true', help='Fine tune the word embeddings') parser.add_argument('--embedding', choices=['word2vec', 'glove', 'senna', 'random'], help='Embedding for words', required=True) parser.add_argument('--embedding_dict', default=None, help='path for embedding dict') parser.add_argument('--batch_size', type=int, default=10, help='Number of sentences in each batch') parser.add_argument('--num_units', type=int, default=100, help='Number of hidden units in LSTM') parser.add_argument('--num_filters', type=int, default=20, help='Number of filters in CNN') parser.add_argument('--learning_rate', type=float, default=0.1, help='Learning rate') parser.add_argument('--decay_rate', type=float, default=0.1, help='Decay rate of learning rate') parser.add_argument('--grad_clipping', type=float, default=0, help='Gradient clipping') parser.add_argument('--gamma', type=float, default=1e-6, help='weight for regularization') parser.add_argument('--peepholes', action='store_true', help='Peepholes for LSTM') parser.add_argument('--oov', choices=['random', 'embedding'], help='Embedding for oov word', required=True) parser.add_argument('--update', choices=['sgd', 'momentum', 'nesterov', 'adadelta'], help='update algorithm', default='sgd') parser.add_argument('--regular', choices=['none', 'l2'], help='regularization for training', required=True) parser.add_argument('--dropout', action='store_true', help='Apply dropout layers') parser.add_argument('--patience', type=int, default=5, help='Patience for early stopping') parser.add_argument('--output_prediction', default='true', action='store_true', help='Output predictions to temp files') parser.add_argument('--train') # "data/POS-penn/wsj/split1/wsj1.train.original" parser.add_argument('--dev') # "data/POS-penn/wsj/split1/wsj1.dev.original" parser.add_argument('--test') # "data/POS-penn/wsj/split1/wsj1.test.original" parser.add_argument("--model") # model name args = parser.parse_args() def construct_input_layer(): if fine_tune: layer_input = lasagne.layers.InputLayer(shape=(None, max_length), input_var=input_var, name='input') layer_embedding = lasagne.layers.EmbeddingLayer(layer_input, input_size=alphabet_size, output_size=embedd_dim, W=embedd_table, name='embedding') return layer_embedding else: layer_input = lasagne.layers.InputLayer(shape=(None, max_length, embedd_dim), input_var=input_var, name='input') return layer_input def construct_char_input_layer(): layer_char_input = lasagne.layers.InputLayer(shape=(None, max_sent_length, max_char_length), input_var=char_input_var, name='char-input') layer_char_input = lasagne.layers.reshape(layer_char_input, (-1, [2])) layer_char_embedding = lasagne.layers.EmbeddingLayer(layer_char_input, input_size=char_alphabet_size, output_size=char_embedd_dim, W=char_embedd_table, name='char_embedding') layer_char_input = lasagne.layers.DimshuffleLayer(layer_char_embedding, pattern=(0, 2, 1)) return layer_char_input logger = utils.get_logger("BiLSTM-CNN-CRF") fine_tune = args.fine_tune oov = args.oov regular = args.regular embedding = args.embedding embedding_path = args.embedding_dict train_path = args.train dev_path = args.dev test_path = args.test update_algo = args.update grad_clipping = args.grad_clipping peepholes = args.peepholes num_filters = args.num_filters gamma = args.gamma output_predict = args.output_prediction dropout = args.dropout modelname = args.model # 读取数据训练集,dev集,测试集 X_train, Y_train, mask_train, X_dev, Y_dev, mask_dev, X_test, Y_test, mask_test, \ embedd_table, label_alphabet, word_alphabet, \ C_train, C_dev, C_test, char_embedd_table = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, oov=oov, fine_tune=fine_tune, embedding=embedding, embedding_path=embedding_path, use_character=True) print 'label_alphabet' for i in range(label_alphabet.size()): print i print label_alphabet.get_instance(i) # print Y_test, Y_test.shape; sys.exit(1) my_size = data_processor.MAX_LENGTH_TRAIN my_size = data_processor.MY_MAX_LENGTH print "\tMY_SIZE", my_size, data_processor.MAX_LENGTH_TRAIN # my_size = data_processor.MAX_LENGTH_DEV print "\tMYSIZE", my_size num_labels = label_alphabet.size() - 1 # 构建网络 logger.info("constructing network...") # create variables target_var = T.imatrix(name='targets') mask_var = T.matrix(name='masks', dtype=theano.config.floatX) if fine_tune: input_var = T.imatrix(name='inputs') num_data, max_length = X_train.shape alphabet_size, embedd_dim = embedd_table.shape else: input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) num_data, max_length, embedd_dim = X_train.shape char_input_var = T.itensor3(name='char-inputs') num_data_char, max_sent_length, max_char_length = C_train.shape char_alphabet_size, char_embedd_dim = char_embedd_table.shape assert (max_length == max_sent_length) assert (num_data == num_data_char) # 构建输入层 # construct input and mask layers logger.info("construct input and mask layers...") layer_incoming1 = construct_char_input_layer() layer_incoming2 = construct_input_layer() layer_mask = lasagne.layers.InputLayer(shape=(None, max_length), input_var=mask_var, name='mask') # construct bi-rnn-cnn logger.info("construct bi-rnn-cnn...") num_units = args.num_units bi_lstm_cnn_crf = build_BiLSTM_CNN_CRF(layer_incoming1, layer_incoming2, num_units, num_labels, mask=layer_mask, grad_clipping=grad_clipping, peepholes=peepholes, num_filters=num_filters, dropout=dropout) logger.info("Network structure: hidden=%d, filter=%d" % (num_units, num_filters)) # compute loss num_tokens = mask_var.sum(dtype=theano.config.floatX) # get outpout of bi-lstm-cnn-crf shape [batch, length, num_labels, num_labels] energies_train = lasagne.layers.get_output(bi_lstm_cnn_crf) energies_eval = lasagne.layers.get_output(bi_lstm_cnn_crf, deterministic=True) loss_train = crf_loss(energies_train, target_var, mask_var).mean() # print loss_train; sys.exit(1) loss_eval = crf_loss(energies_eval, target_var, mask_var).mean() # l2 regularization? if regular == 'l2': l2_penalty = lasagne.regularization.regularize_network_params(bi_lstm_cnn_crf, lasagne.regularization.l2) loss_train = loss_train + gamma * l2_penalty _, corr_train = crf_accuracy(energies_train, target_var) corr_train = (corr_train * mask_var).sum(dtype=theano.config.floatX) prediction_eval, corr_eval = crf_accuracy(energies_eval, target_var) corr_eval = (corr_eval * mask_var).sum(dtype=theano.config.floatX) # Create update expressions for training. # hyper parameters to tune: learning rate, momentum, regularization. batch_size = args.batch_size learning_rate = 1.0 if update_algo == 'adadelta' else args.learning_rate decay_rate = args.decay_rate momentum = 0.9 params = lasagne.layers.get_all_params(bi_lstm_cnn_crf, trainable=True) updates = utils.create_updates(loss_train, params, update_algo, learning_rate, momentum=momentum) # Compile a function performing a training step on a mini-batch train_fn = theano.function([input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # Compile a second function evaluating the loss and accuracy of network eval_fn = theano.function([input_var, target_var, mask_var, char_input_var], [loss_eval, corr_eval, num_tokens, prediction_eval]) # Finally, launch the training loop. logger.info( "Start training: %s with regularization: %s(%f), dropout: %s, fine tune: %s (#training data: %d, batch size: %d, clip: %.1f, peepholes: %s)..." \ % ( update_algo, regular, (0.0 if regular == 'none' else gamma), dropout, fine_tune, num_data, batch_size, grad_clipping, peepholes)) num_batches = num_data / batch_size num_epochs = 1000 # num_epochs = 1 best_loss = 1e+12 best_acc = 0.0 best_epoch_loss = 0 best_epoch_acc = 0 best_loss_test_err = 0. best_loss_test_corr = 0. best_acc_test_err = 0. best_acc_test_corr = 0. stop_count = 0 lr = learning_rate patience = args.patience for epoch in range(1, num_epochs + 1): print 'Epoch %d (learning rate=%.4f, decay rate=%.4f): ' % (epoch, lr, decay_rate) train_err = 0.0 train_corr = 0.0 train_total = 0 train_inst = 0 start_time = time.time() num_back = 0 train_batches = 0 for batch in utils.iterate_minibatches(X_train, Y_train, masks=mask_train, char_inputs=C_train, batch_size=batch_size, shuffle=True): inputs, targets, masks, char_inputs = batch err, corr, num = train_fn(inputs, targets, masks, char_inputs) train_err += err * inputs.shape[0] train_corr += corr train_total += num train_inst += inputs.shape[0] train_batches += 1 time_ave = (time.time() - start_time) / train_batches time_left = (num_batches - train_batches) * time_ave # update log sys.stdout.write("\b" * num_back) log_info = 'train: %d/%d loss: %.4f, acc: %.2f%%, time left (estimated): %.2fs' % ( min(train_batches * batch_size, num_data), num_data, train_err / train_inst, train_corr * 100 / train_total, time_left) sys.stdout.write(log_info) num_back = len(log_info) # update training log after each epoch assert train_inst == num_data sys.stdout.write("\b" * num_back) print 'train: %d/%d loss: %.4f, acc: %.2f%%, time: %.2fs' % ( min(train_batches * batch_size, num_data), num_data, train_err / num_data, train_corr * 100 / train_total, time.time() - start_time) # evaluate performance on dev data dev_err = 0.0 dev_corr = 0.0 dev_total = 0 dev_inst = 0 test_err_sentence = 0 my_f1 = {} my_prs = [] my_trs = [] for batch in utils.iterate_minibatches(X_dev, Y_dev, masks=mask_dev, char_inputs=C_dev, batch_size=batch_size): inputs, targets, masks, char_inputs = batch err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) # print "-->HERE COMES THE PREDS",predictions # print predictions.shape,type(predictions); for i in xrange(batch_size): try: input_clear = [word_alphabet.get_instance(y) for y in list(inputs[i, :])] except IndexError: continue if len(input_clear) == 0: continue try: dev_size = input_clear.index(None) except ValueError: dev_size = my_size # print dev_size my_trs += list(targets[i, :dev_size]) my_prs += list(predictions[i, :dev_size]) # for j in xrange(len(targets[i,:dev_size])): # pr = predictions[i,j] # tr = targets[i,j] # my_f1[(pr,tr)] = my_f1.get((pr,tr),0)+1 # input_clear = [word_alphabet.get_instance(y) for y in list(inputs[0,:])] # print input_clear # my_f1 = f1_score(my_trs,my_prs,average="macro") # print [label_alphabet.get_instance(y+1) for y in list(targets[0,:])] # print targets[0,:],predictions[0,:],inputs.shape[0],my_f1; sys.exit(1) # print err,inputs.shape[0] dev_err += err * inputs.shape[0] dev_corr += corr dev_total += num dev_inst += inputs.shape[0] if output_predict: utils.output_predictions(predictions, targets, masks, 'tmp/dev%d' % epoch, label_alphabet, is_flattened=False) dev_f1 = f1_score(my_trs, my_prs, average="macro") classify_report = metrics.classification_report(my_trs, my_prs) print 'dev classify_report' print classify_report print 'dev loss: %.4f, corr: %d, total: %d, acc: %.2f%%, f1: %.4f' % ( dev_err / dev_inst, dev_corr, dev_total, dev_corr * 100 / dev_total, dev_f1) # CHANGE THIS IF NECESSARY # MODEL SELECTION ON DEV CRITERION, SE useF1 = True useLoss = False criterion = dev_f1 if useF1 else dev_corr / dev_total if best_loss < dev_err and best_acc > criterion: stop_count += 1 else: update_loss = False update_acc = False stop_count = 0 if best_loss > dev_err: update_loss = True best_loss = dev_err best_epoch_loss = epoch if best_acc < criterion: update_acc = True best_acc = criterion best_epoch_acc = epoch else: if useLoss == False: continue # evaluate on test data when better performance detected test_err = 0.0 test_corr = 0.0 test_total = 0 test_inst = 0 test_err_sentence = 0 test_sentences = 0 print "#SAVING MODEL" np.savez(modelname, *lasagne.layers.get_all_param_values(bi_lstm_cnn_crf)) test_prs = [] test_trs = [] for batch in utils.iterate_minibatches(X_test, Y_test, masks=mask_test, char_inputs=C_test, batch_size=batch_size): inputs, targets, masks, char_inputs = batch err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) # print "-->HERE COMES THE PREDS",predictions # print predictions.shape,type(predictions); for i in xrange(batch_size): try: input_clear = [word_alphabet.get_instance(y) for y in list(inputs[i, :])] except IndexError: continue if len(input_clear) == 0: continue try: test_size = input_clear.index(None) except ValueError: test_size = my_size # print dev_size test_trs += list(targets[i, :test_size]) test_prs += list(predictions[i, :test_size]) # print predictions # SE # print "AAA",inputs[0],len(inputs[0]),inputs[0][0] input_clear = [word_alphabet.get_instance(y) for x in inputs for y in list(x)] # predictions,dir(label_alphabet),label_alphabet.get_instance(4) # SE target_clear = [label_alphabet.get_instance(y + 1) for x in targets for y in list(x)] target_clear_pred = [label_alphabet.get_instance(y + 1) for x in predictions for y in list(x)] # SE # print my_size # comment this out # my_size = 652 # print input_clear; sys.exit(1) # print input_clear for ii in range(batch_size): Z = input_clear[ii * my_size:(ii + 1) * my_size] if len(Z) == 0: continue try: size = Z.index(None) except ValueError: size = my_size # print size itruth = input_clear[ii * my_size:(ii + 1) * my_size][:size] EMPTY = "EMPTY" EMPTY = "EMPTY_EMPTY" otruth = filter(lambda z: z != EMPTY, target_clear[ii * my_size:(ii + 1) * my_size][:size]) opred = filter(lambda z: z != EMPTY, target_clear_pred[ii * my_size:(ii + 1) * my_size][:size]) if otruth == opred: test_err_sentence += 1 # print "CORRECT",itruth,otruth,opred print "#CORRECT %%%", len(itruth), len(opred), len(otruth) printout(itruth, opred, otruth) print else: print "#WRONG %%%" # ,itruth,otruth,opred printout(itruth, opred, otruth) print test_sentences += 1 test_err += err * inputs.shape[0] test_corr += corr test_total += num test_inst += inputs.shape[0] if output_predict: utils.output_predictions(predictions, targets, masks, 'tmp/test%d' % epoch, label_alphabet, is_flattened=False) test_f1 = f1_score(test_trs, test_prs, average="macro") test_classify_report = metrics.classification_report(test_trs, test_prs) print 'label_alphabet' for i in range(label_alphabet.size()): print i print label_alphabet.get_instance(i) print 'Epoch %d test classify_report' % epoch print test_classify_report print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%% f1: %.4f' % ( test_err / test_inst, test_corr, test_total, test_corr * 100 / test_total, test_f1), test_err_sentence * 1.0 / test_sentences, test_err_sentence, test_sentences if update_loss: best_loss_test_err = test_err best_loss_test_corr = test_corr if update_acc: best_acc_test_err = test_err best_acc_test_corr = test_corr # stop if dev acc decrease 3 time straightly. if stop_count == patience: break # re-compile a function with new learning rate for training if update_algo != 'adadelta': lr = learning_rate / (1.0 + epoch * decay_rate) updates = utils.create_updates(loss_train, params, update_algo, lr, momentum=momentum) train_fn = theano.function([input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # print best performance on test data. logger.info("final best loss test performance (at epoch %d)" % best_epoch_loss) print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( best_loss_test_err / test_inst, best_loss_test_corr, test_total, best_loss_test_corr * 100 / test_total) logger.info("final best acc test performance (at epoch %d)" % best_epoch_acc) print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( best_acc_test_err / test_inst, best_acc_test_corr, test_total, best_acc_test_corr * 100 / test_total)
def main(): parser = argparse.ArgumentParser(description='Tuning with bi-directional LSTM-CNN-CRF') parser.add_argument('--fine_tune', action='store_true', help='Fine tune the word embeddings') parser.add_argument('--embedding', choices=['word2vec', 'glove', 'senna', 'random'], help='Embedding for words', required=True) parser.add_argument('--embedding_dict', default=None, help='path for embedding dict') parser.add_argument('--batch_size', type=int, default=10, help='Number of sentences in each batch') parser.add_argument('--num_units', type=int, default=100, help='Number of hidden units in LSTM') parser.add_argument('--num_filters', type=int, default=20, help='Number of filters in CNN') parser.add_argument('--learning_rate', type=float, default=0.1, help='Learning rate') parser.add_argument('--decay_rate', type=float, default=0.1, help='Decay rate of learning rate') parser.add_argument('--grad_clipping', type=float, default=0, help='Gradient clipping') parser.add_argument('--gamma', type=float, default=1e-6, help='weight for regularization') parser.add_argument('--peepholes', action='store_true', help='Peepholes for LSTM') parser.add_argument('--oov', choices=['random', 'embedding'], help='Embedding for oov word', required=True) parser.add_argument('--update', choices=['sgd', 'momentum', 'nesterov', 'adadelta'], help='update algorithm', default='sgd') parser.add_argument('--regular', choices=['none', 'l2'], help='regularization for training', required=True) parser.add_argument('--dropout', action='store_true', help='Apply dropout layers') parser.add_argument('--patience', type=int, default=5, help='Patience for early stopping') parser.add_argument('--output_prediction', action='store_true', help='Output predictions to temp files') parser.add_argument('--train') # "data/POS-penn/wsj/split1/wsj1.train.original" parser.add_argument('--dev') # "data/POS-penn/wsj/split1/wsj1.dev.original" parser.add_argument('--test') # "data/POS-penn/wsj/split1/wsj1.test.original" args = parser.parse_args() def construct_input_layer(): if fine_tune: layer_input = lasagne.layers.InputLayer(shape=(None, max_length), input_var=input_var, name='input') layer_embedding = lasagne.layers.EmbeddingLayer(layer_input, input_size=alphabet_size, output_size=embedd_dim, W=embedd_table, name='embedding') return layer_embedding else: layer_input = lasagne.layers.InputLayer(shape=(None, max_length, embedd_dim), input_var=input_var, name='input') return layer_input def construct_char_input_layer(): layer_char_input = lasagne.layers.InputLayer(shape=(None, max_sent_length, max_char_length), input_var=char_input_var, name='char-input') layer_char_input = lasagne.layers.reshape(layer_char_input, (-1, [2])) layer_char_embedding = lasagne.layers.EmbeddingLayer(layer_char_input, input_size=char_alphabet_size, output_size=char_embedd_dim, W=char_embedd_table, name='char_embedding') layer_char_input = lasagne.layers.DimshuffleLayer(layer_char_embedding, pattern=(0, 2, 1)) return layer_char_input logger = utils.get_logger("BiLSTM-CNN-CRF") fine_tune = args.fine_tune oov = args.oov regular = args.regular embedding = args.embedding embedding_path = args.embedding_dict train_path = args.train dev_path = args.dev test_path = args.test update_algo = args.update grad_clipping = args.grad_clipping peepholes = args.peepholes num_filters = args.num_filters gamma = args.gamma output_predict = args.output_prediction dropout = args.dropout X_train, Y_train, mask_train, X_dev, Y_dev, mask_dev, X_test, Y_test, mask_test, \ embedd_table, label_alphabet, \ C_train, C_dev, C_test, char_embedd_table = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, oov=oov, fine_tune=fine_tune, embedding=embedding, embedding_path=embedding_path, use_character=True) num_labels = label_alphabet.size() - 1 logger.info("constructing network...") # create variables target_var = T.imatrix(name='targets') mask_var = T.matrix(name='masks', dtype=theano.config.floatX) if fine_tune: input_var = T.imatrix(name='inputs') num_data, max_length = X_train.shape alphabet_size, embedd_dim = embedd_table.shape else: input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) num_data, max_length, embedd_dim = X_train.shape char_input_var = T.itensor3(name='char-inputs') num_data_char, max_sent_length, max_char_length = C_train.shape char_alphabet_size, char_embedd_dim = char_embedd_table.shape assert (max_length == max_sent_length) assert (num_data == num_data_char) # construct input and mask layers layer_incoming1 = construct_char_input_layer() layer_incoming2 = construct_input_layer() layer_mask = lasagne.layers.InputLayer(shape=(None, max_length), input_var=mask_var, name='mask') # construct bi-rnn-cnn num_units = args.num_units bi_lstm_cnn_crf = build_BiLSTM_CNN_CRF(layer_incoming1, layer_incoming2, num_units, num_labels, mask=layer_mask, grad_clipping=grad_clipping, peepholes=peepholes, num_filters=num_filters, dropout=dropout) logger.info("Network structure: hidden=%d, filter=%d" % (num_units, num_filters)) # compute loss num_tokens = mask_var.sum(dtype=theano.config.floatX) # get outpout of bi-lstm-cnn-crf shape [batch, length, num_labels, num_labels] energies_train = lasagne.layers.get_output(bi_lstm_cnn_crf) energies_eval = lasagne.layers.get_output(bi_lstm_cnn_crf, deterministic=True) loss_train = crf_loss(energies_train, target_var, mask_var).mean() loss_eval = crf_loss(energies_eval, target_var, mask_var).mean() # l2 regularization? if regular == 'l2': l2_penalty = lasagne.regularization.regularize_network_params(bi_lstm_cnn_crf, lasagne.regularization.l2) loss_train = loss_train + gamma * l2_penalty _, corr_train = crf_accuracy(energies_train, target_var) corr_train = (corr_train * mask_var).sum(dtype=theano.config.floatX) prediction_eval, corr_eval = crf_accuracy(energies_eval, target_var) corr_eval = (corr_eval * mask_var).sum(dtype=theano.config.floatX) # Create update expressions for training. # hyper parameters to tune: learning rate, momentum, regularization. batch_size = args.batch_size learning_rate = 1.0 if update_algo == 'adadelta' else args.learning_rate decay_rate = args.decay_rate momentum = 0.9 params = lasagne.layers.get_all_params(bi_lstm_cnn_crf, trainable=True) updates = utils.create_updates(loss_train, params, update_algo, learning_rate, momentum=momentum) # Compile a function performing a training step on a mini-batch train_fn = theano.function([input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # Compile a second function evaluating the loss and accuracy of network eval_fn = theano.function([input_var, target_var, mask_var, char_input_var], [loss_eval, corr_eval, num_tokens, prediction_eval]) # Finally, launch the training loop. logger.info( "Start training: %s with regularization: %s(%f), dropout: %s, fine tune: %s (#training data: %d, batch size: %d, clip: %.1f, peepholes: %s)..." \ % ( update_algo, regular, (0.0 if regular == 'none' else gamma), dropout, fine_tune, num_data, batch_size, grad_clipping, peepholes)) num_batches = num_data / batch_size num_epochs = 1000 best_loss = 1e+12 best_acc = 0.0 best_epoch_loss = 0 best_epoch_acc = 0 best_loss_test_err = 0. best_loss_test_corr = 0. best_acc_test_err = 0. best_acc_test_corr = 0. stop_count = 0 lr = learning_rate patience = args.patience for epoch in range(1, num_epochs + 1): print 'Epoch %d (learning rate=%.4f, decay rate=%.4f): ' % (epoch, lr, decay_rate) train_err = 0.0 train_corr = 0.0 train_total = 0 train_inst = 0 start_time = time.time() num_back = 0 train_batches = 0 for batch in utils.iterate_minibatches(X_train, Y_train, masks=mask_train, char_inputs=C_train, batch_size=batch_size, shuffle=True): inputs, targets, masks, char_inputs = batch err, corr, num = train_fn(inputs, targets, masks, char_inputs) train_err += err * inputs.shape[0] train_corr += corr train_total += num train_inst += inputs.shape[0] train_batches += 1 time_ave = (time.time() - start_time) / train_batches time_left = (num_batches - train_batches) * time_ave # update log sys.stdout.write("\b" * num_back) log_info = 'train: %d/%d loss: %.4f, acc: %.2f%%, time left (estimated): %.2fs' % ( min(train_batches * batch_size, num_data), num_data, train_err / train_inst, train_corr * 100 / train_total, time_left) sys.stdout.write(log_info) num_back = len(log_info) # update training log after each epoch assert train_inst == num_data sys.stdout.write("\b" * num_back) print 'train: %d/%d loss: %.4f, acc: %.2f%%, time: %.2fs' % ( min(train_batches * batch_size, num_data), num_data, train_err / num_data, train_corr * 100 / train_total, time.time() - start_time) # evaluate performance on dev data dev_err = 0.0 dev_corr = 0.0 dev_total = 0 dev_inst = 0 for batch in utils.iterate_minibatches(X_dev, Y_dev, masks=mask_dev, char_inputs=C_dev, batch_size=batch_size): inputs, targets, masks, char_inputs = batch err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) dev_err += err * inputs.shape[0] dev_corr += corr dev_total += num dev_inst += inputs.shape[0] if output_predict: utils.output_predictions(predictions, targets, masks, 'tmp/dev%d' % epoch, label_alphabet, is_flattened=False) print 'dev loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( dev_err / dev_inst, dev_corr, dev_total, dev_corr * 100 / dev_total) if best_loss < dev_err and best_acc > dev_corr / dev_total: stop_count += 1 else: update_loss = False update_acc = False stop_count = 0 if best_loss > dev_err: update_loss = True best_loss = dev_err best_epoch_loss = epoch if best_acc < dev_corr / dev_total: update_acc = True best_acc = dev_corr / dev_total best_epoch_acc = epoch # evaluate on test data when better performance detected test_err = 0.0 test_corr = 0.0 test_total = 0 test_inst = 0 for batch in utils.iterate_minibatches(X_test, Y_test, masks=mask_test, char_inputs=C_test, batch_size=batch_size): inputs, targets, masks, char_inputs = batch err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) test_err += err * inputs.shape[0] test_corr += corr test_total += num test_inst += inputs.shape[0] if output_predict: utils.output_predictions(predictions, targets, masks, 'tmp/test%d' % epoch, label_alphabet, is_flattened=False) print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( test_err / test_inst, test_corr, test_total, test_corr * 100 / test_total) if update_loss: best_loss_test_err = test_err best_loss_test_corr = test_corr if update_acc: best_acc_test_err = test_err best_acc_test_corr = test_corr # stop if dev acc decrease 3 time straightly. if stop_count == patience: break # re-compile a function with new learning rate for training if update_algo != 'adadelta': lr = learning_rate / (1.0 + epoch * decay_rate) updates = utils.create_updates(loss_train, params, update_algo, lr, momentum=momentum) train_fn = theano.function([input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # print best performance on test data. logger.info("final best loss test performance (at epoch %d)" % best_epoch_loss) print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( best_loss_test_err / test_inst, best_loss_test_corr, test_total, best_loss_test_corr * 100 / test_total) logger.info("final best acc test performance (at epoch %d)" % best_epoch_acc) print 'test loss: %.4f, corr: %d, total: %d, acc: %.2f%%' % ( best_acc_test_err / test_inst, best_acc_test_corr, test_total, best_acc_test_corr * 100 / test_total)
def main(): parser = argparse.ArgumentParser( description='Tuning with bi-directional LSTM-CNN-CRF') parser.add_argument('--fine_tune', action='store_true', help='Fine tune the word embeddings') parser.add_argument('--embedding', choices=['word2vec', 'glove', 'senna', 'random'], help='Embedding for words', required=True) parser.add_argument('--embedding_dict', default=None, help='path for embedding dict') parser.add_argument('--batch_size', type=int, default=10, help='Number of sentences in each batch') parser.add_argument('--num_units', type=int, default=100, help='Number of hidden units in LSTM') parser.add_argument('--num_filters', type=int, default=20, help='Number of filters in CNN') parser.add_argument('--learning_rate', type=float, default=0.1, help='Learning rate') parser.add_argument('--decay_rate', type=float, default=0.1, help='Decay rate of learning rate') parser.add_argument('--grad_clipping', type=float, default=0, help='Gradient clipping') parser.add_argument('--gamma', type=float, default=1e-6, help='weight for regularization') parser.add_argument('--peepholes', action='store_true', help='Peepholes for LSTM') parser.add_argument('--oov', choices=['random', 'embedding'], help='Embedding for oov word', required=True) parser.add_argument('--update', choices=['sgd', 'momentum', 'nesterov', 'adadelta'], help='update algorithm', default='sgd') parser.add_argument('--regular', choices=['none', 'l2'], help='regularization for training', required=True) parser.add_argument('--dropout', action='store_true', help='Apply dropout layers') parser.add_argument('--patience', type=int, default=5, help='Patience for early stopping') parser.add_argument('--output_prediction', action='store_true', help='Output predictions to temp files') parser.add_argument( '--train') # "data/POS-penn/wsj/split1/wsj1.train.original" parser.add_argument( '--dev') # "data/POS-penn/wsj/split1/wsj1.dev.original" parser.add_argument( '--test') # "data/POS-penn/wsj/split1/wsj1.test.original" parser.add_argument('--realtest') parser.add_argument('--mymodel') args = parser.parse_args() def construct_input_layer(): if fine_tune: layer_input = lasagne.layers.InputLayer(shape=(None, max_length), input_var=input_var, name='input') layer_embedding = lasagne.layers.EmbeddingLayer( layer_input, input_size=alphabet_size, output_size=embedd_dim, W=embedd_table, name='embedding') return layer_embedding else: layer_input = lasagne.layers.InputLayer(shape=(None, max_length, embedd_dim), input_var=input_var, name='input') return layer_input def construct_char_input_layer(): layer_char_input = lasagne.layers.InputLayer(shape=(None, max_sent_length, max_char_length), input_var=char_input_var, name='char-input') layer_char_input = lasagne.layers.reshape(layer_char_input, (-1, [2])) layer_char_embedding = lasagne.layers.EmbeddingLayer( layer_char_input, input_size=char_alphabet_size, output_size=char_embedd_dim, W=char_embedd_table, name='char_embedding') layer_char_input = lasagne.layers.DimshuffleLayer(layer_char_embedding, pattern=(0, 2, 1)) return layer_char_input logger = utils.get_logger("BiLSTM-CNN-CRF") fine_tune = args.fine_tune oov = args.oov regular = args.regular embedding = args.embedding embedding_path = args.embedding_dict train_path = args.train dev_path = args.dev test_path = args.test real_test_path = args.realtest update_algo = args.update grad_clipping = args.grad_clipping peepholes = args.peepholes num_filters = args.num_filters gamma = args.gamma output_predict = args.output_prediction dropout = args.dropout mymodel = args.mymodel print "Model is", mymodel, test_path, real_test_path X_train, Y_train, mask_train, X_dev, Y_dev, mask_dev, X_test, Y_test, mask_test, _X_real_test, _Y_real_test, _mask_real_test, \ embedd_table, label_alphabet, word_alphabet, \ C_train, C_dev, C_test, _C_real_test, char_embedd_table = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, test_path, oov=oov, fine_tune=fine_tune, embedding=embedding, embedding_path=embedding_path, use_character=True) _X_train, _Y_train, _mask_train, _X_dev, _Y_dev, _mask_dev, _X_test, _Y_test, _mask_test, X_real_test, Y_real_test, mask_real_test, \ _embedd_table, _label_alphabet, _word_alphabet, \ _C_train, _C_dev, _C_test, C_real_test, _char_embedd_table = data_processor.load_dataset_sequence_labeling(train_path, dev_path, test_path, real_test_path, oov=oov,fine_tune=fine_tune, embedding=embedding, embedding_path=embedding_path,use_character=True) #print _C_train.shape,_C_dev.shape,_C_test.shape,C_real_test.shape; #sys.exit(1) my_size = data_processor.MAX_LENGTH_TRAIN my_size = data_processor.MY_MAX_LENGTH #my_size = data_processor.MAX_LENGTH_DEV print "\tMYSIZE", my_size, C_real_test.shape, C_test.shape, C_train.shape num_labels = label_alphabet.size() - 1 logger.info("constructing network...") # create variables target_var = T.imatrix(name='targets') mask_var = T.matrix(name='masks', dtype=theano.config.floatX) if fine_tune: input_var = T.imatrix(name='inputs') num_data, max_length = X_train.shape alphabet_size, embedd_dim = embedd_table.shape else: input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) num_data, max_length, embedd_dim = X_train.shape char_input_var = T.itensor3(name='char-inputs') num_data_char, max_sent_length, max_char_length = C_train.shape char_alphabet_size, char_embedd_dim = char_embedd_table.shape assert (max_length == max_sent_length) assert (num_data == num_data_char) # construct input and mask layers layer_incoming1 = construct_char_input_layer() layer_incoming2 = construct_input_layer() layer_mask = lasagne.layers.InputLayer(shape=(None, max_length), input_var=mask_var, name='mask') # construct bi-rnn-cnn num_units = args.num_units bi_lstm_cnn_crf = build_BiLSTM_CNN_CRF(layer_incoming1, layer_incoming2, num_units, num_labels, mask=layer_mask, grad_clipping=grad_clipping, peepholes=peepholes, num_filters=num_filters, dropout=dropout) # bi_lstm_cnn_crf = None logger.info("Network structure: hidden=%d, filter=%d" % (num_units, num_filters)) # compute loss num_tokens = mask_var.sum(dtype=theano.config.floatX) # get outpout of bi-lstm-cnn-crf shape [batch, length, num_labels, num_labels] energies_train = lasagne.layers.get_output(bi_lstm_cnn_crf) energies_eval = lasagne.layers.get_output(bi_lstm_cnn_crf, deterministic=True) loss_train = crf_loss(energies_train, target_var, mask_var).mean() loss_eval = crf_loss(energies_eval, target_var, mask_var).mean() # l2 regularization? if regular == 'l2': l2_penalty = lasagne.regularization.regularize_network_params( bi_lstm_cnn_crf, lasagne.regularization.l2) loss_train = loss_train + gamma * l2_penalty _, corr_train = crf_accuracy(energies_train, target_var) corr_train = (corr_train * mask_var).sum(dtype=theano.config.floatX) prediction_eval, corr_eval = crf_accuracy(energies_eval, target_var) corr_eval = (corr_eval * mask_var).sum(dtype=theano.config.floatX) # Create update expressions for training. # hyper parameters to tune: learning rate, momentum, regularization. batch_size = args.batch_size learning_rate = 1.0 if update_algo == 'adadelta' else args.learning_rate decay_rate = args.decay_rate momentum = 0.9 params = lasagne.layers.get_all_params(bi_lstm_cnn_crf, trainable=True) updates = utils.create_updates(loss_train, params, update_algo, learning_rate, momentum=momentum) # Compile a function performing a training step on a mini-batch train_fn = theano.function( [input_var, target_var, mask_var, char_input_var], [loss_train, corr_train, num_tokens], updates=updates) # Compile a second function evaluating the loss and accuracy of network eval_fn = theano.function( [input_var, target_var, mask_var, char_input_var], [loss_eval, corr_eval, num_tokens, prediction_eval]) my_prediction_eval = my_crf_accuracy(energies_eval) my_eval_fn = theano.function([input_var, mask_var, char_input_var], [my_prediction_eval]) # Finally, launch the training loop. logger.info( "Start training: %s with regularization: %s(%f), dropout: %s, fine tune: %s (#training data: %d, batch size: %d, clip: %.1f, peepholes: %s)..." \ % ( update_algo, regular, (0.0 if regular == 'none' else gamma), dropout, fine_tune, num_data, batch_size, grad_clipping, peepholes)) num_batches = num_data / batch_size num_epochs = 1000 best_loss = 1e+12 best_acc = 0.0 best_epoch_loss = 0 best_epoch_acc = 0 best_loss_test_err = 0. best_loss_test_corr = 0. best_acc_test_err = 0. best_acc_test_corr = 0. stop_count = 0 lr = learning_rate patience = args.patience print "#LOADING MODEL" #np.savez("model.npz",*lasagne.layers.get_all_param_values(bi_lstm_cnn_crf)) # just load the data, see here: # https://github.com/Lasagne/Lasagne/blob/master/examples/mnist.py #try: mymodel = sys.argv[1] #except IndexError: # mymodel = "models.npz" with np.load(mymodel) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(bi_lstm_cnn_crf, param_values) correct = 0 total = 0 print dir(bi_lstm_cnn_crf) #print bi_lstm_cnn_crf.predict([1,2,3,4]) #sys.exit(1) print X_real_test.shape, Y_real_test.shape, C_real_test.shape, mask_real_test.shape # that's a stupid hack #C_real_test = C_real_test[:len(X_real_test)] #print X_real_test[0:1] #print my_eval_fn(X_real_test[0:1],mask_real_test[0:1],C_real_test[0:1]) #sys.exit(1) for batch in utils.iterate_minibatches(X_real_test, Y_real_test, masks=mask_real_test, char_inputs=C_real_test, batch_size=batch_size): inputs, targets, masks, char_inputs = batch #print inputs,targets,masks,char_inputs; sys.exit(1) err, corr, num, predictions = eval_fn(inputs, targets, masks, char_inputs) #print predictions # SE input_clear = [ word_alphabet.get_instance(y) for x in inputs for y in list(x) ] # predictions,dir(label_alphabet),label_alphabet.get_instance(4) # SE target_clear = [ label_alphabet.get_instance(y + 1) for x in targets for y in list(x) ] target_clear_pred = [ label_alphabet.get_instance(y + 1) for x in predictions for y in list(x) ] # SE #print my_size # comment this out #my_size = 557 #print input_clear; sys.exit(1) for ii in range(batch_size): Z = input_clear[ii * my_size:(ii + 1) * my_size] if len(Z) == 0: continue try: size = Z.index(None) except ValueError: size = my_size #print size itruth = input_clear[ii * my_size:(ii + 1) * my_size][:size] otruth = filter( lambda z: z != "EMPTY", target_clear[ii * my_size:(ii + 1) * my_size][:size]) opred = filter( lambda z: z != "EMPTY", target_clear_pred[ii * my_size:(ii + 1) * my_size][:size]) total += len(opred) correct += len( filter(lambda x: x == True, [otruth[jj] == opred[jj] for jj in xrange(len(opred))])) if otruth == opred: #test_err_sentence += 1 #print "CORRECT",itruth,otruth,opred #print "#CORRECT %%%" printout(itruth, opred, otruth) print else: #print "#WRONG %%%" #itruth,otruth,opred printout(itruth, opred, otruth) print print correct, total, correct * 1.0 / total