def train(args): print '\nNEURAL POS TAGGER START\n' print '\tINITIAL EMBEDDING\t%s %s' % (args.word_list, args.emb_list) print '\tWORD\t\t\tEmb Dim: %d Hidden Dim: %d' % (args.w_emb_dim, args.w_hidden_dim) print '\tCHARACTER\t\tEmb Dim: %d Hidden Dim: %d' % (args.c_emb_dim, args.c_hidden_dim) print '\tOPTIMIZATION\t\tMethod: %s Learning Rate: %f\n' % (args.opt, args.lr) print '\tMINI-BATCH: %d\n' % args.batch_size """ load data """ print 'Loading data sets...\n' train_corpus, vocab_word, vocab_char, vocab_tag, _ = io_utils.load_conll(args.train_data) """ limit data set """ train_corpus = train_corpus[:args.data_size] train_corpus.sort(key=lambda a: len(a)) dev_corpus = None if args.dev_data: dev_corpus, dev_vocab_word, dev_vocab_char, dev_vocab_tag, _ = io_utils.load_conll(args.dev_data) for w in dev_vocab_word.i2w: if args.vocab_size is None or vocab_word.size() < args.vocab_size: vocab_word.add_word(w) for c in dev_vocab_char.i2w: vocab_char.add_word(c) for t in dev_vocab_tag.i2w: vocab_tag.add_word(t) if args.save: io_utils.dump_data(vocab_word, 'vocab_word') io_utils.dump_data(vocab_char, 'vocab_char') io_utils.dump_data(vocab_tag, 'vocab_tag') """ load word embeddings """ init_w_emb = None if args.emb_list: print '\tLoading pre-trained word embeddings...\n' init_w_emb = io_utils.load_init_emb(args.emb_list, args.word_list, vocab_word) w_emb_dim = init_w_emb.shape[1] else: print '\tUse random-initialized word embeddings...\n' w_emb_dim = args.w_emb_dim """ converting into ids """ print '\nConverting into IDs...\n' tr_x, tr_c, tr_b, tr_y = convert_into_ids(train_corpus, vocab_word, vocab_char, vocab_tag) tr_x, tr_y, tr_b = set_minibatch(tr_x, tr_y, args.batch_size) tr_x, tr_y = shared_samples(tr_x, tr_y) dev_x = None dev_y = None if args.dev_data: dev_x, dev_c, dev_b, dev_y = convert_into_ids(dev_corpus, vocab_word, vocab_char, vocab_tag) dev_x, dev_y, dev_b = set_minibatch(dev_x, dev_y, 1) dev_x, dev_y = shared_samples(dev_x, dev_y) print '\tTrain Sentences: %d Dev Sentences: %d' % (len(train_corpus), len(dev_corpus)) else: print '\tTrain Sentences: %d' % len(train_corpus) print '\tWord size: %d Char size: %d' % (vocab_word.size(), vocab_char.size()) """ set model parameters """ hidden_dim = args.w_hidden_dim output_dim = vocab_tag.size() window = args.window opt = args.opt """ symbol definition """ print '\tCompiling Theano Code...' bos = T.iscalar('bos') eos = T.iscalar('eos') n_words = T.iscalar('n_words') batch_size = T.iscalar('batch_size') x = T.imatrix('x') y = T.ivector('y') lr = T.fscalar('lr') """ tagger set up """ tagger = Model(x=x, y=y, n_words=n_words, batch_size=batch_size, lr=lr, init_emb=init_w_emb, vocab_size=vocab_word.size(), emb_dim=w_emb_dim, hidden_dim=hidden_dim, output_dim=output_dim, opt=opt, window=window) train_f = theano.function( inputs=[bos, eos, n_words, batch_size, lr], outputs=[tagger.nll, tagger.result], updates=tagger.updates, givens={ x: tr_x[bos: eos], y: tr_y[bos: eos] }, mode='FAST_RUN' ) dev_f = theano.function( inputs=[bos, eos, n_words, batch_size], outputs=tagger.result, givens={ x: dev_x[bos: eos], y: dev_y[bos: eos] }, mode='FAST_RUN' ) def _train(): for epoch in xrange(args.epoch): _lr = args.lr / float(epoch+1) indices = range(len(tr_b)) random.shuffle(indices) print '\nEpoch: %d' % (epoch + 1) print '\tBatch Index: ', start = time.time() total = 0.0 correct = 0 losses = 0.0 for i, index in enumerate(indices): if i % 100 == 0 and i != 0: print i, sys.stdout.flush() boundary = tr_b[index] loss, corrects = train_f(boundary[0], boundary[1], boundary[2],boundary[3], _lr) assert math.isnan(loss) is False, i total += len(corrects) correct += np.sum(corrects) losses += loss end = time.time() print '\tTime: %f seconds' % (end - start) print '\tNegative Log Likelihood: %f' % losses print '\tAccuracy:%f Total:%d Correct:%d' % ((correct / total), total, correct) _dev(dev_f) def _dev(model): print '\tBatch Index: ', start = time.time() total = 0.0 correct = 0 for index in xrange(len(dev_b)): if index % 100 == 0 and index != 0: print index, sys.stdout.flush() boundary = dev_b[index] corrects = model(boundary[0], boundary[1], boundary[2], boundary[3]) total += len(corrects) correct += np.sum(corrects) end = time.time() print '\tTime: %f seconds' % (end - start) print '\tAccuracy:%f Total:%d Correct:%d' % ((correct / total), total, correct) _train()
def main(argv): print '\nSYSTEM START' print '\nMODE: Training' ################### # PREPROCESS DATA # ################### """ Load initial embedding file """ vocab_word = Vocab() emb = None if argv.init_emb: print '\n\tInitial Embedding Loading...' emb, vocab_word = load_init_emb(init_emb=argv.init_emb) print '\t\tVocabulary Size: %d' % vocab_word.size() """ Load corpora """ print '\n\tLoading Corpora...' tr_corpus, tr_doc_names, vocab_word = load_conll(path=argv.train_data, vocab=vocab_word, data_size=argv.data_size) dev_corpus, dev_doc_names, _ = load_conll(path=argv.dev_data, vocab=vocab_word, data_size=argv.data_size) print '\t\tTrain Documents: %d' % len(tr_corpus) print '\t\tDev Documents: %d' % len(dev_corpus) """ Extract gold mentions CoNLL-2012: Train=155,560, Dev=19,156, Test=19,764 """ # gold_mentions: 1D: n_doc, 2D: n_sents, 3D: n_mentions: elem=(bos, eos) # gold_corefs: 1D: n_doc, 2D: n_sents, 3D: n_mentions: elem=coref_id print '\n\tExtracting Gold Mentions...' print '\t\tTRAIN', tr_gold_ments = get_gold_mentions(tr_corpus, check=argv.check) print '\t\tDEV ', dev_gold_ments = get_gold_mentions(dev_corpus) """ Extract cand mentions """ # cand_mentions: 1D: n_doc, 2D: n_sents, 3D: n_mentions; elem=(bos, eos) print '\n\tExtracting Cand Mentions...' print '\t\tTRAIN', tr_cand_ments = get_cand_mentions(tr_corpus, check=argv.check) print '\t\tDEV ', dev_cand_ments = get_cand_mentions(dev_corpus) """ Convert words into IDs """ print '\n\tConverting Words into IDs...' print '\t\tVocabulary Size: %d' % vocab_word.size() tr_word_ids = convert_words_into_ids(corpus=tr_corpus, vocab_word=vocab_word) dev_word_ids = convert_words_into_ids(corpus=dev_corpus, vocab_word=vocab_word) """ Set word ids for mentions """ tr_gold_ments = set_word_id_for_ment(tr_word_ids, tr_gold_ments) tr_cand_ments = set_word_id_for_ment(tr_word_ids, tr_cand_ments) dev_gold_ments = set_word_id_for_ment(dev_word_ids, dev_gold_ments) dev_cand_ments = set_word_id_for_ment(dev_word_ids, dev_cand_ments) """ Set coref ids for cand mentions """ tr_cand_ments = set_cand_ment_coref(tr_gold_ments, tr_cand_ments) dev_cand_ments = set_cand_ment_coref(dev_gold_ments, dev_cand_ments) """ Check the coverage: Coverage 95.0%, Rate 1:3.5 by Berkeley System """ print '\n\tChecking the Coverage of the Candidate Mentions...' check_coverage_of_cand_mentions(tr_gold_ments, tr_cand_ments) check_coverage_of_cand_mentions(dev_gold_ments, dev_cand_ments) """ Extract features """ print '\n\tExtracting features...' """ phi = (span, word, ctx, dist, label, position) span : 1D: n_doc, 2D: n_ments, 3D: n_cand_ants, 4D: limit * 2; elem=word id word : 1D: n_doc, 2D: n_ments, 3D: n_cand_ants, 4D: [m_first, m_last, a_first, a_last]; elem=word id ctx : 1D: n_doc, 2D: n_ments, 3D: n_cand_ants, 4D: window * 2 * 2; elem=word id dist : 1D: n_doc, 2D: n_ments, 3D: n_cand_ants; elem=sent dist label : 1D: n_doc, 2D: n_ments; elem=0/1 position: 1D: n_doc, 2D: n_ments, 3D: n_cand_ants; elem=(sent_m_i, span_m, sent_a_i, span_a) """ tr_phi, tr_posit = get_features(tr_cand_ments, False, argv.n_cands) dev_phi, dev_posit = get_features(dev_cand_ments, True, argv.n_cands) """ Count the number of features """ n_tr_phi_total = reduce(lambda a, b: a + reduce(lambda c, d: c + len(d), b, 0), tr_phi, 0) n_tr_phi_t = reduce(lambda a, b: a + reduce(lambda c, d: c + reduce(lambda e, f: e + np.sum(f[-1]), d, 0), b, 0), tr_phi, 0) n_tr_phi_f = n_tr_phi_total - n_tr_phi_t n_dev_phi_total = reduce(lambda a, b: a + reduce(lambda c, d: c + len(d), b, 0), dev_phi, 0) n_dev_phi_t = reduce(lambda a, b: a + reduce(lambda c, d: c + reduce(lambda e, f: e + np.sum(f[-1]), d, 0), b, 0), dev_phi, 0) n_dev_phi_f = n_dev_phi_total - n_dev_phi_t print '\t\tTrain Features Total: %d\tRate: P:N\t%d:%d' % (n_tr_phi_total, n_tr_phi_t, n_tr_phi_f) print '\t\tDev Features Total: %d\tRate: P:N\t%d:%d' % (n_dev_phi_total, n_dev_phi_t, n_dev_phi_f) """ Convert into the Theano format """ print '\n\tConverting features into the Theano Format...' """ samples = (span, word, ctx, dist, label) span : 1D: n_doc * n_ments * n_cand_ants, 2D: limit * 2; elem=word id word : 1D: n_doc * n_ments * n_cand_ants, 2D: [m_first, m_last, a_first, a_last]; elem=word id ctx : 1D: n_doc * n_ments * n_cand_ants, 2D: window * 2 * 2; elem=word id dist : 1D: n_doc * n_ments * n_cand_ants; elem=sent dist label : 1D: n_doc * n_ments * n_cand_ants; elem=0/1 indices: 1D: n_doc * n_ments; elem=(bos, eos) """ tr_samples, tr_indices = theano_format(tr_phi) dev_samples, dev_indices = theano_format(dev_phi) ###################### # BUILD ACTUAL MODEL # ###################### print '\nBuilding the model...' model = set_model(argv, vocab_word, emb) bos = T.iscalar('bos') eos = T.iscalar('eos') train_f = theano.function( inputs=[bos, eos], outputs=[model.nll, model.correct, model.correct_t, model.correct_f, model.total_p, model.total_r], updates=model.updates, givens={ model.x_span: tr_samples[0][bos: eos], model.x_word: tr_samples[1][bos: eos], model.x_ctx : tr_samples[2][bos: eos], model.x_dist: tr_samples[3][bos: eos], model.x_slen: tr_samples[4][bos: eos], model.y : tr_samples[5][bos: eos] }, mode='FAST_RUN' ) dev_f = theano.function( inputs=[bos, eos], outputs=[model.y_hat_index, model.p_y_hat, model.correct, model.correct_t, model.correct_f, model.total_p, model.total_r], givens={ model.x_span: dev_samples[0][bos: eos], model.x_word: dev_samples[1][bos: eos], model.x_ctx : dev_samples[2][bos: eos], model.x_dist: dev_samples[3][bos: eos], model.x_slen: dev_samples[4][bos: eos], model.y : dev_samples[5][bos: eos] }, mode='FAST_RUN' ) ############### # TRAIN MODEL # ############### batch_size = argv.batch n_batches = n_tr_phi_total / batch_size indices = range(n_batches) print 'Training START\n' print 'Mini-Batch Samples: %d\n' % n_batches for epoch in xrange(argv.epoch): random.shuffle(indices) print '\nEpoch: %d' % (epoch + 1) print 'TRAIN' print '\tIndex: ', start = time.time() total_loss = 0. correct = np.zeros(9, dtype='float32') correct_t = np.zeros(9, dtype='float32') correct_f = np.zeros(9, dtype='float32') total = 0. total_r = np.zeros(9, dtype='float32') total_p = np.zeros(9, dtype='float32') for i, index in enumerate(indices): if i % 1000 == 0 and i != 0: print '%d' % i, sys.stdout.flush() loss, crr, crr_t, crr_f, ttl_p, ttl_r = train_f(index * batch_size, (index+1) * batch_size) assert not math.isnan(loss), 'Index: %d Batch Index: %d' % (i, index) total_loss += loss correct += crr correct_t += crr_t correct_f += crr_f total += batch_size total_p += ttl_p total_r += ttl_r end = time.time() print '\n\tTime: %f seconds' % (end - start) show_results(total, total_p, total_r, correct, correct_t, correct_f, total_loss) predict(epoch, dev_f, dev_corpus, dev_doc_names, dev_indices, dev_posit)
def train(args): print '\nNEURAL POS TAGGER START\n' print '\tINITIAL EMBEDDING\t%s %s' % (args.word_list, args.emb_list) print '\tWORD\t\t\tEmb Dim: %d Hidden Dim: %d' % (args.w_emb_dim, args.w_hidden_dim) print '\tCHARACTER\t\tEmb Dim: %d Hidden Dim: %d' % (args.c_emb_dim, args.c_hidden_dim) print '\tOPTIMIZATION\t\tMethod: %s Learning Rate: %f\n' % (args.opt, args.lr) """ load data """ print 'Loading data sets...\n' train_corpus, vocab_word, vocab_char, vocab_tag, max_char_len = load_train_data(args) dev_corpus, dev_vocab_word, dev_vocab_char, dev_vocab_tag, dev_max_char_len = load_dev_data(args) if dev_corpus: for w in dev_vocab_word.i2w: if args.vocab_size is None or vocab_word.size() < args.vocab_size: vocab_word.add_word(w) for c in dev_vocab_char.i2w: vocab_char.add_word(c) for t in dev_vocab_tag.i2w: vocab_tag.add_word(t) if args.save: io_utils.dump_data(vocab_word, 'vocab_word') io_utils.dump_data(vocab_char, 'vocab_char') io_utils.dump_data(vocab_tag, 'vocab_tag') """ load word embeddings """ init_w_emb = None if args.emb_list: print '\tLoading pre-trained word embeddings...\n' init_w_emb = io_utils.load_init_emb(args.emb_list, args.word_list, vocab_word) w_emb_dim = init_w_emb.shape[1] else: print '\tUse random-initialized word embeddings...\n' w_emb_dim = args.w_emb_dim """ limit data set """ train_corpus = train_corpus[:args.data_size] train_corpus.sort(key=lambda a: len(a)) """ converting into ids """ print '\nConverting into IDs...\n' tr_x, tr_c, tr_b, tr_y = preprocessor.convert_into_ids(train_corpus, vocab_word, vocab_char, vocab_tag) if args.dev_data: dev_x, dev_c, dev_b, dev_y = preprocessor.convert_into_ids(dev_corpus, vocab_word, vocab_char, vocab_tag) print '\tTrain Sentences: %d Dev Sentences: %d' % (len(train_corpus), len(dev_corpus)) else: print '\tTrain Sentences: %d' % len(train_corpus) print '\tWord size: %d Char size: %d' % (vocab_word.size(), vocab_char.size()) """ tagger set up """ tagger = set_model(args, init_w_emb, w_emb_dim, vocab_word, vocab_char, vocab_tag) train_f = theano.function( inputs=tagger.input, outputs=[tagger.nll, tagger.result], updates=tagger.updates, mode='FAST_RUN' ) dev_f = theano.function( inputs=tagger.input[:-1], outputs=tagger.result, mode='FAST_RUN' ) def _train(): print '\nTRAINING START\n' for epoch in xrange(args.epoch): _lr = args.lr / float(epoch+1) indices = range(len(tr_x)) random.shuffle(indices) print '\nEpoch: %d' % (epoch + 1) print '\n\tTrain set' print '\t\tBatch Index: ', start = time.time() total = 0.0 correct = 0 losses = 0.0 for i, index in enumerate(indices): if i % 100 == 0 and i != 0: print i, sys.stdout.flush() if args.model == 'char': loss, corrects = train_f(tr_x[index], tr_c[index], tr_b[index], tr_y[index], _lr) else: loss, corrects = train_f(tr_x[index], tr_y[index], _lr) assert math.isnan(loss) is False, index total += len(corrects) correct += np.sum(corrects) losses += loss end = time.time() print '\n\t\tTime: %f seconds' % (end - start) print '\t\tNegative Log Likelihood: %f' % losses print '\t\tAccuracy:%f Total:%d Correct:%d' % ((correct / total), total, correct) if args.save: io_utils.dump_data(tagger, 'model-%s.epoch-%d' % (args.model, epoch+1)) _dev(dev_f) def _dev(_dev_f): print '\n\tDev set' print '\t\tBatch Index: ', start = time.time() total = 0.0 correct = 0 for index in xrange(len(dev_x)): if index % 100 == 0 and index != 0: print index, sys.stdout.flush() if args.model == 'char': corrects = _dev_f(dev_x[index], dev_c[index], dev_b[index], dev_y[index]) else: corrects = _dev_f(dev_x[index], dev_y[index]) total += len(corrects) correct += np.sum(corrects) end = time.time() print '\n\t\tTime: %f seconds' % (end - start) print '\t\tAccuracy:%f Total:%d Correct:%d' % ((correct / total), total, correct) _train()
def train(args): print '\nNEURAL POS TAGGER START\n' print '\tINITIAL EMBEDDING\t%s %s' % (args.word_list, args.emb_list) print '\tWORD\t\t\tEmb Dim: %d Hidden Dim: %d' % (args.w_emb_dim, args.w_hidden_dim) print '\tCHARACTER\t\tEmb Dim: %d Hidden Dim: %d' % (args.c_emb_dim, args.c_hidden_dim) print '\tOPTIMIZATION\t\tMethod: %s Learning Rate: %f\n' % (args.opt, args.lr) """ load data """ print 'Loading data sets...\n' train_corpus, vocab_word, vocab_char, vocab_tag, max_char_len = load_train_data( args) dev_corpus, dev_vocab_word, dev_vocab_char, dev_vocab_tag, dev_max_char_len = load_dev_data( args) if dev_corpus: for w in dev_vocab_word.i2w: if args.vocab_size is None or vocab_word.size() < args.vocab_size: vocab_word.add_word(w) for c in dev_vocab_char.i2w: vocab_char.add_word(c) for t in dev_vocab_tag.i2w: vocab_tag.add_word(t) if args.save: io_utils.dump_data(vocab_word, 'vocab_word') io_utils.dump_data(vocab_char, 'vocab_char') io_utils.dump_data(vocab_tag, 'vocab_tag') """ load word embeddings """ init_w_emb = None if args.emb_list: print '\tLoading pre-trained word embeddings...\n' init_w_emb = io_utils.load_init_emb(args.emb_list, args.word_list, vocab_word) w_emb_dim = init_w_emb.shape[1] else: print '\tUse random-initialized word embeddings...\n' w_emb_dim = args.w_emb_dim """ limit data set """ train_corpus = train_corpus[:args.data_size] train_corpus.sort(key=lambda a: len(a)) """ converting into ids """ print '\nConverting into IDs...\n' tr_x, tr_c, tr_b, tr_y = preprocessor.convert_into_ids( train_corpus, vocab_word, vocab_char, vocab_tag) if args.dev_data: dev_x, dev_c, dev_b, dev_y = preprocessor.convert_into_ids( dev_corpus, vocab_word, vocab_char, vocab_tag) print '\tTrain Sentences: %d Dev Sentences: %d' % (len(train_corpus), len(dev_corpus)) else: print '\tTrain Sentences: %d' % len(train_corpus) print '\tWord size: %d Char size: %d' % (vocab_word.size(), vocab_char.size()) """ tagger set up """ tagger = set_model(args, init_w_emb, w_emb_dim, vocab_word, vocab_char, vocab_tag) train_f = theano.function(inputs=tagger.input, outputs=[tagger.nll, tagger.result], updates=tagger.updates, mode='FAST_RUN') dev_f = theano.function(inputs=tagger.input[:-1], outputs=tagger.result, mode='FAST_RUN') def _train(): print '\nTRAINING START\n' for epoch in xrange(args.epoch): _lr = args.lr / float(epoch + 1) indices = range(len(tr_x)) random.shuffle(indices) print '\nEpoch: %d' % (epoch + 1) print '\n\tTrain set' print '\t\tBatch Index: ', start = time.time() total = 0.0 correct = 0 losses = 0.0 for i, index in enumerate(indices): if i % 100 == 0 and i != 0: print i, sys.stdout.flush() if args.model == 'char': loss, corrects = train_f(tr_x[index], tr_c[index], tr_b[index], tr_y[index], _lr) else: loss, corrects = train_f(tr_x[index], tr_y[index], _lr) assert math.isnan(loss) is False, index total += len(corrects) correct += np.sum(corrects) losses += loss end = time.time() print '\n\t\tTime: %f seconds' % (end - start) print '\t\tNegative Log Likelihood: %f' % losses print '\t\tAccuracy:%f Total:%d Correct:%d' % ( (correct / total), total, correct) if args.save: io_utils.dump_data( tagger, 'model-%s.epoch-%d' % (args.model, epoch + 1)) _dev(dev_f) def _dev(_dev_f): print '\n\tDev set' print '\t\tBatch Index: ', start = time.time() total = 0.0 correct = 0 for index in xrange(len(dev_x)): if index % 100 == 0 and index != 0: print index, sys.stdout.flush() if args.model == 'char': corrects = _dev_f(dev_x[index], dev_c[index], dev_b[index], dev_y[index]) else: corrects = _dev_f(dev_x[index], dev_y[index]) total += len(corrects) correct += np.sum(corrects) end = time.time() print '\n\t\tTime: %f seconds' % (end - start) print '\t\tAccuracy:%f Total:%d Correct:%d' % ( (correct / total), total, correct) _train()
def train(args): print '\nNEURAL POS TAGGER START\n' print '\tINITIAL EMBEDDING\t%s %s' % (args.word_list, args.emb_list) print '\tWORD\t\t\tEmb Dim: %d Hidden Dim: %d' % (args.w_emb_dim, args.w_hidden_dim) print '\tCHARACTER\t\tEmb Dim: %d Hidden Dim: %d' % (args.c_emb_dim, args.c_hidden_dim) print '\tOPTIMIZATION\t\tMethod: %s Learning Rate: %f\n' % (args.opt, args.lr) print '\tMINI-BATCH: %d\n' % args.batch_size """ load data """ print 'Loading data sets...\n' train_corpus, vocab_word, vocab_char, vocab_tag, max_char_len = io_utils.load_conll( args.train_data) """ limit data set """ train_corpus = train_corpus[:args.data_size] train_corpus.sort(key=lambda a: len(a)) dev_corpus = None if args.dev_data: dev_corpus, dev_vocab_word, dev_vocab_char, dev_vocab_tag, max_char_len_dev = io_utils.load_conll( args.dev_data) for w in dev_vocab_word.i2w: if args.vocab_size is None or vocab_word.size() < args.vocab_size: vocab_word.add_word(w) for c in dev_vocab_char.i2w: vocab_char.add_word(c) for t in dev_vocab_tag.i2w: vocab_tag.add_word(t) if args.save: io_utils.dump_data(vocab_word, 'vocab_word') io_utils.dump_data(vocab_char, 'vocab_char') io_utils.dump_data(vocab_tag, 'vocab_tag') """ load pre-trained embeddings """ init_w_emb = None if args.emb_list: print '\tLoading word embeddings...\n' init_w_emb = io_utils.load_init_emb(args.emb_list, args.word_list, vocab_word) w_emb_dim = init_w_emb.shape[1] else: w_emb_dim = args.w_emb_dim """ converting into ids """ print '\nConverting into IDs...\n' tr_x, tr_c, tr_b, tr_y = convert_into_ids(train_corpus, vocab_word, vocab_char, vocab_tag, max_char_len) tr_x, tr_c, tr_y, tr_b = set_minibatch(tr_x, tr_c, tr_y, max_char_len, args.batch_size) tr_x, tr_c, tr_y = shared_samples(tr_x, tr_c, tr_y) if args.dev_data: dev_x, dev_c, dev_b, dev_y = convert_into_ids(dev_corpus, vocab_word, vocab_char, vocab_tag, max_char_len_dev) dev_x, dev_c, dev_y, dev_b = set_minibatch(dev_x, dev_c, dev_y, max_char_len_dev, 1) dev_x, dev_c, dev_y = shared_samples(dev_x, dev_c, dev_y) print '\tTrain Sentences: %d Dev Sentences: %d' % (len(train_corpus), len(dev_corpus)) else: print '\tTrain Sentences: %d' % len(train_corpus) print '\tWord size: %d Char size: %d' % (vocab_word.size(), vocab_char.size()) """ set model parameters """ w_hidden_dim = args.w_hidden_dim c_emb_dim = args.c_emb_dim c_hidden_dim = args.c_hidden_dim output_dim = vocab_tag.size() window = args.window opt = args.opt """ symbol definition """ print '\tCompiling Theano Code...' bos = T.iscalar('bos') eos = T.iscalar('eos') n_words = T.iscalar('n_words') batch_size = T.iscalar('batch_size') x = T.imatrix('x') c = T.itensor4('c') y = T.ivector('y') lr = T.fscalar('lr') """ tagger set up """ tagger = Model(x=x, c=c, y=y, n_words=n_words, batch_size=batch_size, lr=lr, init_emb=init_w_emb, vocab_w_size=vocab_word.size(), w_emb_dim=w_emb_dim, w_hidden_dim=w_hidden_dim, c_emb_dim=c_emb_dim, c_hidden_dim=c_hidden_dim, output_dim=output_dim, vocab_c_size=vocab_char.size(), window=window, opt=opt) train_f = theano.function(inputs=[bos, eos, n_words, batch_size, lr], outputs=[tagger.nll, tagger.result], updates=tagger.updates, givens={ x: tr_x[bos:eos], c: tr_c[bos:eos], y: tr_y[bos:eos] }, mode='FAST_RUN') dev_f = theano.function(inputs=[bos, eos, n_words, batch_size], outputs=tagger.result, givens={ x: dev_x[bos:eos], c: dev_c[bos:eos], y: dev_y[bos:eos] }, mode='FAST_RUN') def _train(): for epoch in xrange(args.epoch): _lr = args.lr / float(epoch + 1) indices = range(len(tr_b)) random.shuffle(indices) print '\nEpoch: %d' % (epoch + 1) print '\tBatch Index: ', start = time.time() total = 0.0 correct = 0 losses = 0.0 for i, index in enumerate(indices): if i % 100 == 0 and i != 0: print i, sys.stdout.flush() boundary = tr_b[index] loss, corrects = train_f(boundary[0], boundary[1], boundary[2], boundary[3], _lr) assert math.isnan(loss) is False, i total += len(corrects) correct += np.sum(corrects) losses += loss end = time.time() print '\tTime: %f seconds' % (end - start) print '\tNegative Log Likelihood: %f' % losses print '\tAccuracy:%f Total:%d Correct:%d' % ( (correct / total), total, correct) _dev(dev_f) def _dev(model): print '\tBatch Index: ', start = time.time() total = 0.0 correct = 0 for index in xrange(len(dev_b)): if index % 100 == 0 and index != 0: print index, sys.stdout.flush() boundary = dev_b[index] corrects = model(boundary[0], boundary[1], boundary[2], boundary[3]) total += len(corrects) correct += np.sum(corrects) end = time.time() print '\tTime: %f seconds' % (end - start) print '\tAccuracy:%f Total:%d Correct:%d' % ( (correct / total), total, correct) _train()