def train_on(dataset, params): print("Using hyperparameter configuration:", params) losses = [] state_dicts = [] kfold = StratifiedKFold(dataset, k=10, grouping=origin_of) for train, val in kfold: # TODO: Figure how much of the following code we can put outside the loop vocab = Vocabulary.from_instances(dataset) # TODO: Figure out the best parameters here elmo = Elmo(cached_path(OPTIONS_FILE), cached_path(WEIGHTS_FILE), num_output_representations=2, dropout=params["dropout"] ) # TODO: Does dropout refer to the LSTM or ELMo? word_embeddings = ELMoTextFieldEmbedder({"tokens": elmo}) # TODO: Figure out the best parameters here lstm = PytorchSeq2VecWrapper( torch.nn.LSTM(input_size=elmo.get_output_dim(), hidden_size=64, num_layers=params["num_layers"], batch_first=True)) model = RuseModel(word_embeddings, lstm, vocab) optimizer = optim.Adam(model.parameters()) # TODO: What kind of iterator should be used? iterator = BucketIterator(batch_size=params["batch_size"], sorting_keys=[("mt_sent", "num_tokens"), ("ref_sent", "num_tokens")]) iterator.index_with(vocab) # TODO: Figure out best hyperparameters trainer = Trainer(model=model, optimizer=optimizer, iterator=iterator, cuda_device=0, train_dataset=train, validation_dataset=val, patience=5, num_epochs=100) trainer.train() # TODO: Better way to access the validation loss? loss, _ = trainer._validation_loss() losses.append(loss) state_dicts.append(model.state_dict()) mean_loss = np.mean(losses) print("Mean validation loss was:", mean_loss) return TrainResults(cv_loss=mean_loss, state_dicts=state_dicts)
def train_one_epoch( trainer: Trainer, epoch_count: int) -> Tuple[Dict[str, float], Dict[str, float]]: train_metrics: Dict[str, float] = {} val_metrics: Dict[str, float] = {} this_epoch_val_metric: float = None metrics: Dict[str, float] = {} train_metrics = trainer._train_epoch(epoch_count) if trainer._validation_data is not None: with torch.no_grad(): # We have a validation set, so compute all the metrics on it. val_loss, num_batches = trainer._validation_loss() val_metrics = training_util.get_metrics(trainer.model, val_loss, num_batches, reset=True) this_epoch_val_metric = val_metrics[trainer._validation_metric] for key, value in train_metrics.items(): metrics["training_" + key] = value for key, value in val_metrics.items(): metrics["validation_" + key] = value if trainer._serialization_dir: dump_metrics( os.path.join(trainer._serialization_dir, f"metrics_epoch_{epoch_count}.json"), metrics) # The Scheduler API is agnostic to whether your schedule requires a validation metric - # if it doesn't, the validation metric passed here is ignored. if trainer._learning_rate_scheduler: trainer._learning_rate_scheduler.step(this_epoch_val_metric, epoch_count) if trainer._momentum_scheduler: trainer._momentum_scheduler.step(this_epoch_val_metric, epoch_count) #trainer._save_checkpoint(epoch_count) return train_metrics, val_metrics
with open(args.model_path, 'rb') as inp: model.load_state_dict(torch.load(inp, map_location='cpu')) optimizer = torch.optim.Adam(model.parameters()) iterator = BucketIterator(batch_size=1, sorting_keys=[("source", "num_tokens")]) iterator.index_with(vocab) trainer = Trainer(model=model, optimizer=optimizer, iterator=iterator, train_dataset=None, validation_dataset=val_dataset, patience=10, num_epochs=0) np.random.seed(0) with torch.no_grad(): val_loss, num_batches = trainer._validation_loss( ) # ну да, не очень, но коротко val_metrics = training_util.get_metrics(trainer.model, val_loss, num_batches, reset=True) print(val_metrics) if args.file is not None: with open(args.file, 'a') as out: out.write( str(args.bpe) + ';' + str(args.mistakes_rate) + ';' + str(val_metrics['perplexity']) + '\n')