def main(debug):
"""Main function
Args:
debug :
whether to use a fraction of the data. Make sure to set to False
when you're ready to train your model for real!
"""
print(80 * "=")
print("INITIALIZING")
print(80 * "=")
torch.manual_seed(1234)
if torch.cuda.is_available():
torch.set_default_tensor_type(torch.cuda.FloatTensor)
torch.cuda.manual_seed_all(1234)
print('Running on GPU: {}.'.format(torch.cuda.get_device_name()))
else:
print('Running on CPU.')
config = Config()
data = load_and_preprocess_data(max_batch_size=config.batch_size,
transition_cache=0 if debug else None)
transducer, word_embeddings, train_data = data[:3]
dev_sents, dev_arcs = data[3:5]
test_sents, test_arcs = data[5:]
config.n_word_ids = len(transducer.id2word) + 1 # plus null
config.n_tag_ids = len(transducer.id2tag) + 1
config.n_deprel_ids = len(transducer.id2deprel) + 1
config.embed_size = word_embeddings.shape[1]
for (word_batch, tag_batch, deprel_batch), td_batch in \
train_data.get_iterator(shuffled=False):
config.n_word_features = word_batch.shape[-1]
config.n_tag_features = tag_batch.shape[-1]
config.n_deprel_features = deprel_batch.shape[-1]
config.n_classes = td_batch.shape[-1]
break
print('# word features: {}'.format(config.n_word_features))
print('# tag features: {}'.format(config.n_tag_features))
print('# deprel features: {}'.format(config.n_deprel_features))
print('# classes: {}'.format(config.n_classes))
if debug:
dev_sents = dev_sents[:500]
dev_arcs = dev_arcs[:500]
test_sents = test_sents[:500]
test_arcs = test_arcs[:500]
print(80 * "=")
print("TRAINING")
print(80 * "=")
weights_file = Path('weights.pt')
print('Best weights will be saved to:', weights_file)
model = ParserModel(transducer, config, word_embeddings)
if torch.cuda.is_available():
model = model.cuda()
best_las = 0.
trnbar_fmt = '{l_bar}{bar}| [{elapsed}<{remaining}, {rate_fmt}{postfix}]'
with tqdm(desc='Training',
total=config.n_epochs,
leave=False,
unit='epoch',
position=0,
bar_format=trnbar_fmt) as progbar:
for epoch in range(config.n_epochs):
if debug:
trn_loss = model.fit_epoch(list(islice(train_data, 32)), epoch,
progbar, config.batch_size)
else:
trn_loss = model.fit_epoch(train_data, epoch, progbar)
tqdm.write('Epoch {:>2} training loss: {:.3g}'.format(
epoch + 1, trn_loss))
stdout.flush()
dev_las, dev_uas = model.evaluate(dev_sents, dev_arcs)
best = dev_las > best_las
if best:
best_las = dev_las
if not debug:
torch.save(model.state_dict(), str(weights_file))
tqdm.write(' validation LAS: {:.3f}{} UAS: {:.3f}'.format(
dev_las, ' (BEST!)' if best else ' ', dev_uas))
if not debug:
print()
print(80 * "=")
print("TESTING")
print(80 * "=")
print("Restoring the best model weights found on the dev set.")
model.load_state_dict(torch.load(str(weights_file)))
stdout.flush()
las, uas = model.evaluate(test_sents, test_arcs)
if las:
print('Test LAS: {:.3f}'.format(las), end=' ')
print('UAS: {:.3f}'.format(uas))
print("Done.")
return 0
def main(debug):
'''Main function
Args:
debug :
whether to use a fraction of the data. Make sure to set to False
when you're ready to train your model for real!
'''
print(80 * "=")
print("INITIALIZING")
print(80 * "=")
config = Config()
data = load_and_preprocess_data(max_batch_size=config.batch_size)
transducer, word_embeddings, train_data = data[:3]
dev_sents, dev_arcs = data[3:5]
test_sents, test_arcs = data[5:]
config.n_word_ids = len(transducer.id2word) + 1 # plus null
config.n_tag_ids = len(transducer.id2tag) + 1
config.n_deprel_ids = len(transducer.id2deprel) + 1
config.embed_size = word_embeddings.shape[1]
for (word_batch, tag_batch, deprel_batch), td_batch in \
train_data.get_iterator(shuffled=False):
config.n_word_features = word_batch.shape[-1]
config.n_tag_features = tag_batch.shape[-1]
config.n_deprel_features = deprel_batch.shape[-1]
config.n_classes = td_batch.shape[-1]
break
print(config.n_tag_ids, config.n_word_ids)
print('Word feat size: {}, tag feat size: {}, deprel feat size: {}, '
'classes size: {}'.format(config.n_word_features,
config.n_tag_features,
config.n_deprel_features,
config.n_classes))
if debug:
dev_sents = dev_sents[:500]
dev_arcs = dev_arcs[:500]
test_sents = test_sents[:500]
test_arcs = test_arcs[:500]
if not debug:
weight_file = NamedTemporaryFile(suffix='.weights')
with tf.Graph().as_default(), tf.Session() as session:
print("Building model...", end=' ')
start = time.time()
model = ParserModel(transducer, session, config, word_embeddings)
print("took {:.2f} seconds\n".format(time.time() - start))
init = tf.global_variables_initializer()
session.run(init)
saver = None if debug else tf.train.Saver()
print(80 * "=")
print("TRAINING")
print(80 * "=")
best_las = 0.
for epoch in range(config.n_epochs):
print('Epoch {}'.format(epoch))
if debug:
model.fit_epoch(list(islice(train_data, 3)), config.batch_size)
else:
model.fit_epoch(train_data)
stdout.flush()
dev_las, dev_uas = model.eval(dev_sents, dev_arcs)
best = dev_las > best_las
if best:
best_las = dev_las
if not debug:
saver.save(session, weight_file.name)
print('Validation LAS: ', end='')
print('{:.2f}{}'.format(dev_las, ' (BEST!), ' if best else ', '))
print('Validation UAS: ', end='')
print('{:.2f}'.format(dev_uas))
if not debug:
print()
print(80 * "=")
print("TESTING")
print(80 * "=")
print("Restoring the best model weights found on the dev set")
saver.restore(session, weight_file.name)
stdout.flush()
las, uas = model.eval(test_sents, test_arcs)
if las:
print("Test LAS: ", end='')
print('{:.2f}'.format(las), end=', ')
print("Test UAS: ", end='')
print('{:.2f}'.format(uas))
print("Done!")
return 0
