def main(debug): """Main function Args: debug : whether to use a fraction of the data. Make sure to set to False when you're ready to train your model for real! """ print(80 * "=") print("INITIALIZING") print(80 * "=") torch.manual_seed(1234) if torch.cuda.is_available(): torch.set_default_tensor_type(torch.cuda.FloatTensor) torch.cuda.manual_seed_all(1234) print('Running on GPU: {}.'.format(torch.cuda.get_device_name())) else: print('Running on CPU.') config = Config() data = load_and_preprocess_data(max_batch_size=config.batch_size, transition_cache=0 if debug else None) transducer, word_embeddings, train_data = data[:3] dev_sents, dev_arcs = data[3:5] test_sents, test_arcs = data[5:] config.n_word_ids = len(transducer.id2word) + 1 # plus null config.n_tag_ids = len(transducer.id2tag) + 1 config.n_deprel_ids = len(transducer.id2deprel) + 1 config.embed_size = word_embeddings.shape[1] for (word_batch, tag_batch, deprel_batch), td_batch in \ train_data.get_iterator(shuffled=False): config.n_word_features = word_batch.shape[-1] config.n_tag_features = tag_batch.shape[-1] config.n_deprel_features = deprel_batch.shape[-1] config.n_classes = td_batch.shape[-1] break print('# word features: {}'.format(config.n_word_features)) print('# tag features: {}'.format(config.n_tag_features)) print('# deprel features: {}'.format(config.n_deprel_features)) print('# classes: {}'.format(config.n_classes)) if debug: dev_sents = dev_sents[:500] dev_arcs = dev_arcs[:500] test_sents = test_sents[:500] test_arcs = test_arcs[:500] print(80 * "=") print("TRAINING") print(80 * "=") weights_file = Path('weights.pt') print('Best weights will be saved to:', weights_file) model = ParserModel(transducer, config, word_embeddings) if torch.cuda.is_available(): model = model.cuda() best_las = 0. trnbar_fmt = '{l_bar}{bar}| [{elapsed}<{remaining}, {rate_fmt}{postfix}]' with tqdm(desc='Training', total=config.n_epochs, leave=False, unit='epoch', position=0, bar_format=trnbar_fmt) as progbar: for epoch in range(config.n_epochs): if debug: trn_loss = model.fit_epoch(list(islice(train_data, 32)), epoch, progbar, config.batch_size) else: trn_loss = model.fit_epoch(train_data, epoch, progbar) tqdm.write('Epoch {:>2} training loss: {:.3g}'.format( epoch + 1, trn_loss)) stdout.flush() dev_las, dev_uas = model.evaluate(dev_sents, dev_arcs) best = dev_las > best_las if best: best_las = dev_las if not debug: torch.save(model.state_dict(), str(weights_file)) tqdm.write(' validation LAS: {:.3f}{} UAS: {:.3f}'.format( dev_las, ' (BEST!)' if best else ' ', dev_uas)) if not debug: print() print(80 * "=") print("TESTING") print(80 * "=") print("Restoring the best model weights found on the dev set.") model.load_state_dict(torch.load(str(weights_file))) stdout.flush() las, uas = model.evaluate(test_sents, test_arcs) if las: print('Test LAS: {:.3f}'.format(las), end=' ') print('UAS: {:.3f}'.format(uas)) print("Done.") return 0
def main(debug): '''Main function Args: debug : whether to use a fraction of the data. Make sure to set to False when you're ready to train your model for real! ''' print(80 * "=") print("INITIALIZING") print(80 * "=") config = Config() data = load_and_preprocess_data(max_batch_size=config.batch_size) transducer, word_embeddings, train_data = data[:3] dev_sents, dev_arcs = data[3:5] test_sents, test_arcs = data[5:] config.n_word_ids = len(transducer.id2word) + 1 # plus null config.n_tag_ids = len(transducer.id2tag) + 1 config.n_deprel_ids = len(transducer.id2deprel) + 1 config.embed_size = word_embeddings.shape[1] for (word_batch, tag_batch, deprel_batch), td_batch in \ train_data.get_iterator(shuffled=False): config.n_word_features = word_batch.shape[-1] config.n_tag_features = tag_batch.shape[-1] config.n_deprel_features = deprel_batch.shape[-1] config.n_classes = td_batch.shape[-1] break print(config.n_tag_ids, config.n_word_ids) print('Word feat size: {}, tag feat size: {}, deprel feat size: {}, ' 'classes size: {}'.format(config.n_word_features, config.n_tag_features, config.n_deprel_features, config.n_classes)) if debug: dev_sents = dev_sents[:500] dev_arcs = dev_arcs[:500] test_sents = test_sents[:500] test_arcs = test_arcs[:500] if not debug: weight_file = NamedTemporaryFile(suffix='.weights') with tf.Graph().as_default(), tf.Session() as session: print("Building model...", end=' ') start = time.time() model = ParserModel(transducer, session, config, word_embeddings) print("took {:.2f} seconds\n".format(time.time() - start)) init = tf.global_variables_initializer() session.run(init) saver = None if debug else tf.train.Saver() print(80 * "=") print("TRAINING") print(80 * "=") best_las = 0. for epoch in range(config.n_epochs): print('Epoch {}'.format(epoch)) if debug: model.fit_epoch(list(islice(train_data, 3)), config.batch_size) else: model.fit_epoch(train_data) stdout.flush() dev_las, dev_uas = model.eval(dev_sents, dev_arcs) best = dev_las > best_las if best: best_las = dev_las if not debug: saver.save(session, weight_file.name) print('Validation LAS: ', end='') print('{:.2f}{}'.format(dev_las, ' (BEST!), ' if best else ', ')) print('Validation UAS: ', end='') print('{:.2f}'.format(dev_uas)) if not debug: print() print(80 * "=") print("TESTING") print(80 * "=") print("Restoring the best model weights found on the dev set") saver.restore(session, weight_file.name) stdout.flush() las, uas = model.eval(test_sents, test_arcs) if las: print("Test LAS: ", end='') print('{:.2f}'.format(las), end=', ') print("Test UAS: ", end='') print('{:.2f}'.format(uas)) print("Done!") return 0