def train_lookup_model():
parser = init_argparser()
opt = parser.parse_args()
default_settings = opt.default_settings
# Add machine-task to path
# gets the lookupt task from tasks
T = get_task("lookup", is_mini=True)
print("Got Task")
parameters = T.default_params[default_settings]
train_path = T.train_path
valid_path = T.valid_path
test_paths = T.test_paths
# # Prepare logging and data set
init_logging(parameters)
src, tgt, train, dev, monitor_data = prepare_iters(
parameters, train_path, test_paths, valid_path,
parameters['batch_size'])
# Prepare model
seq2seq, output_vocab = initialize_model(parameters, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
# Prepare training
losses = [NLLLoss(ignore_index=pad)]
for loss in losses:
loss.to(device)
loss_weights = [1.]
metrics = [SequenceAccuracy(ignore_index=pad)]
trainer = SupervisedTrainer(expt_dir='../models')
# Train
print("Training")
seq2seq, _ = trainer.train(seq2seq,
train,
num_epochs=20,
dev_data=dev,
monitor_data=monitor_data,
optimizer='adam',
checkpoint_path='../models',
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=10,
print_every=10)
def get_iters(task_name="long_lookup",
batch_size=1,
eval_batch_size=512,
max_len=50):
T = get_task(task_name)
train_path = T.train_path
valid_path = T.valid_path
test_paths = T.test_paths
# generate training and testing data
train_dataset, valid_dataset, test_datasets, src, tgt = get_train_valid_tests(
train_path, valid_path, test_paths, max_len)
train_iter = get_standard_batch_iterator(train_dataset, batch_size)
valid_iter = get_standard_batch_iterator(valid_dataset, eval_batch_size)
tests_iters = OrderedDict()
for i, dataset in enumerate(test_paths):
tests_iters[dataset] = get_standard_batch_iterator(
test_datasets[i], eval_batch_size)
return train_iter, valid_iter, tests_iters, src, tgt
def test_get_lookup_table_task(self):
task = get_task("lookup")
self.assertEqual(task.name, "lookup")
def test_get_symbol_rewriting_task(self):
task = get_task("symbol_rewriting")
self.assertEqual(task.name, "Symbol Rewriting")
from machine.tasks import get_task
task_a = get_task("long_lookup_reverse")
task_b = get_task("lookup")
task_c = get_task("symbol_rewriting")
print(task_a)
print(task_b)
print(task_c)