def train_on(dataset, params):
print("Using hyperparameter configuration:", params)
losses = []
state_dicts = []
kfold = StratifiedKFold(dataset, k=10, grouping=origin_of)
for train, val in kfold:
# TODO: Figure how much of the following code we can put outside the loop
vocab = Vocabulary.from_instances(dataset)
# TODO: Figure out the best parameters here
elmo = Elmo(cached_path(OPTIONS_FILE),
cached_path(WEIGHTS_FILE),
num_output_representations=2,
dropout=params["dropout"]
) # TODO: Does dropout refer to the LSTM or ELMo?
word_embeddings = ELMoTextFieldEmbedder({"tokens": elmo})
# TODO: Figure out the best parameters here
lstm = PytorchSeq2VecWrapper(
torch.nn.LSTM(input_size=elmo.get_output_dim(),
hidden_size=64,
num_layers=params["num_layers"],
batch_first=True))
model = RuseModel(word_embeddings, lstm, vocab)
optimizer = optim.Adam(model.parameters())
# TODO: What kind of iterator should be used?
iterator = BucketIterator(batch_size=params["batch_size"],
sorting_keys=[("mt_sent", "num_tokens"),
("ref_sent", "num_tokens")])
iterator.index_with(vocab)
# TODO: Figure out best hyperparameters
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
cuda_device=0,
train_dataset=train,
validation_dataset=val,
patience=5,
num_epochs=100)
trainer.train()
# TODO: Better way to access the validation loss?
loss, _ = trainer._validation_loss()
losses.append(loss)
state_dicts.append(model.state_dict())
mean_loss = np.mean(losses)
print("Mean validation loss was:", mean_loss)
return TrainResults(cv_loss=mean_loss, state_dicts=state_dicts)
def train_one_epoch(
trainer: Trainer,
epoch_count: int) -> Tuple[Dict[str, float], Dict[str, float]]:
train_metrics: Dict[str, float] = {}
val_metrics: Dict[str, float] = {}
this_epoch_val_metric: float = None
metrics: Dict[str, float] = {}
train_metrics = trainer._train_epoch(epoch_count)
if trainer._validation_data is not None:
with torch.no_grad():
# We have a validation set, so compute all the metrics on it.
val_loss, num_batches = trainer._validation_loss()
val_metrics = training_util.get_metrics(trainer.model,
val_loss,
num_batches,
reset=True)
this_epoch_val_metric = val_metrics[trainer._validation_metric]
for key, value in train_metrics.items():
metrics["training_" + key] = value
for key, value in val_metrics.items():
metrics["validation_" + key] = value
if trainer._serialization_dir:
dump_metrics(
os.path.join(trainer._serialization_dir,
f"metrics_epoch_{epoch_count}.json"), metrics)
# The Scheduler API is agnostic to whether your schedule requires a validation metric -
# if it doesn't, the validation metric passed here is ignored.
if trainer._learning_rate_scheduler:
trainer._learning_rate_scheduler.step(this_epoch_val_metric,
epoch_count)
if trainer._momentum_scheduler:
trainer._momentum_scheduler.step(this_epoch_val_metric, epoch_count)
#trainer._save_checkpoint(epoch_count)
return train_metrics, val_metrics
with open(args.model_path, 'rb') as inp:
model.load_state_dict(torch.load(inp, map_location='cpu'))
optimizer = torch.optim.Adam(model.parameters())
iterator = BucketIterator(batch_size=1,
sorting_keys=[("source", "num_tokens")])
iterator.index_with(vocab)
trainer = Trainer(model=model,
optimizer=optimizer,
iterator=iterator,
train_dataset=None,
validation_dataset=val_dataset,
patience=10,
num_epochs=0)
np.random.seed(0)
with torch.no_grad():
val_loss, num_batches = trainer._validation_loss(
) # ну да, не очень, но коротко
val_metrics = training_util.get_metrics(trainer.model,
val_loss,
num_batches,
reset=True)
print(val_metrics)
if args.file is not None:
with open(args.file, 'a') as out:
out.write(
str(args.bpe) + ';' + str(args.mistakes_rate) + ';' +
str(val_metrics['perplexity']) + '\n')
