def test_no_val_step_end(tmpdir): """ Test that val step + val epoch end """ model = DeterministicModel() model.training_step = model.training_step_dict_return model.validation_step = model.validation_step_dict_return model.validation_step_end = None model.validation_epoch_end = model.validation_epoch_end trainer = Trainer(default_root_dir=tmpdir, weights_summary=None, limit_train_batches=2, limit_val_batches=3, num_sanity_val_steps=0, max_epochs=2) trainer.fit(model) # out are the results of the full loop # eval_results are output of _evaluate callback_metrics, eval_results = trainer.run_evaluation(test_mode=False) assert len(callback_metrics) == 1 assert len(callback_metrics[0]) == 6 assert len(eval_results) == 1 eval_results = eval_results[0] assert 'val_step_end' not in eval_results assert eval_results['val_epoch_end'] == 1233 for k in ['val_loss', 'log', 'progress_bar']: assert k in eval_results # ensure all the keys ended up as candidates for callbacks assert len(trainer.logger_connector.callback_metrics) in [8, 9] # make sure correct steps were called assert model.validation_step_called assert not model.validation_step_end_called assert model.validation_epoch_end_called
def test_validation_step_dict_return(tmpdir): """ Test that val step can return a dict with all the expected keys and they end up in the correct place """ model = DeterministicModel() model.training_step = model.training_step_dict_return model.validation_step = model.validation_step_dict_return model.validation_step_end = None model.validation_epoch_end = None trainer = Trainer(default_root_dir=tmpdir, weights_summary=None, limit_train_batches=2, limit_val_batches=2, max_epochs=2) trainer.fit(model) # out are the results of the full loop # eval_results are output of _evaluate callback_metrics, eval_results = trainer.run_evaluation(test_mode=False) assert len(callback_metrics) == 1 assert len(callback_metrics[0]) == 5 assert len(eval_results) == 2 assert eval_results[0]['log']['log_acc1'] == 12 assert eval_results[1]['log']['log_acc1'] == 13 for k in ['val_loss', 'log', 'progress_bar']: assert k in eval_results[0] assert k in eval_results[1] # ensure all the keys ended up as candidates for callbacks assert len(trainer.logger_connector.callback_metrics) in [7, 8] # make sure correct steps were called assert model.validation_step_called assert not model.validation_step_end_called assert not model.validation_epoch_end_called
def test_val_step_epoch_step_metrics(tmpdir): """ Make sure the logged + pbar metrics are allocated accordingly at every step when requested """ # enable internal debugging actions os.environ['PL_DEV_DEBUG'] = '1' model = DeterministicModel() model.training_step = model.training_step_result_log_epoch_and_step_for_callbacks model.training_step_end = None model.training_epoch_end = None model.validation_step = model.validation_step_result_epoch_step_metrics model.validation_step_end = None model.validation_epoch_end = None batches = 3 epochs = 3 trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, log_every_n_steps=1, limit_train_batches=batches, limit_val_batches=batches, weights_summary=None, ) trainer.fit(model) assert len(trainer.logger_connector.callback_metrics) == 11 expected_metrics = { 'early_stop_on', 'checkpoint_on', 'val_step_pbar_acc', 'val_step_pbar_acc_epoch', 'val_step_log_acc', 'val_step_log_acc_epoch', 'val_step_log_pbar_acc', 'val_step_log_pbar_acc_epoch', 'val_step_batch_idx', 'val_step_batch_idx_epoch' } expected_metrics.add('debug_epoch') seen_metrics = set(trainer.logger_connector.callback_metrics) assert expected_metrics == seen_metrics # make sure correct steps were called assert model.validation_step_called assert not model.validation_step_end_called assert not model.validation_epoch_end_called # no early stopping assert len(trainer.dev_debugger.early_stopping_history) == 0 # make sure we logged the exact number of metrics assert len( trainer.dev_debugger.logged_metrics) == epochs * batches + (epochs) assert len( trainer.dev_debugger.pbar_added_metrics) == epochs * batches + (epochs) # make sure we logged the correct epoch metrics for metric_idx in range(0, len(trainer.dev_debugger.logged_metrics), batches + 1): batch_metrics = trainer.dev_debugger.logged_metrics[ metric_idx:metric_idx + batches] epoch_metric = trainer.dev_debugger.logged_metrics[metric_idx + batches] epoch = epoch_metric['epoch'] # make sure the metric was split for batch_metric in batch_metrics: assert f'val_step_log_acc_step/epoch_{epoch}' in batch_metric assert f'val_step_log_pbar_acc_step/epoch_{epoch}' in batch_metric # make sure the epoch split was correct assert 'val_step_log_acc_epoch' in epoch_metric assert 'val_step_log_pbar_acc_epoch' in epoch_metric # make sure we logged the correct pbar metrics for metric_idx in range(0, len(trainer.dev_debugger.pbar_added_metrics), batches + 1): batch_metrics = trainer.dev_debugger.pbar_added_metrics[ metric_idx:metric_idx + batches] epoch_metric = trainer.dev_debugger.pbar_added_metrics[metric_idx + batches] # make sure the metric was split for batch_metric in batch_metrics: assert 'val_step_pbar_acc_step' in batch_metric assert 'val_step_log_pbar_acc_step' in batch_metric # make sure the epoch split was correct assert 'val_step_pbar_acc_epoch' in epoch_metric assert 'val_step_log_pbar_acc_epoch' in epoch_metric # only 1 checkpoint expected since values didn't change after that assert len(trainer.dev_debugger.checkpoint_callback_history) == 1 # make sure the last known metric is correct assert trainer.logger_connector.callback_metrics['checkpoint_on'] == 189
def test_val_step_only_step_metrics(tmpdir): """ Make sure the logged + pbar metrics are allocated accordingly at every step when requested """ # enable internal debugging actions os.environ['PL_DEV_DEBUG'] = '1' model = DeterministicModel() model.training_step = model.training_step_result_log_epoch_and_step_for_callbacks model.training_step_end = None model.training_epoch_end = None model.validation_step = model.validation_step_result_only_step_metrics model.validation_step_end = None model.validation_epoch_end = None batches = 3 epochs = 3 trainer = Trainer( default_root_dir=tmpdir, max_epochs=epochs, row_log_interval=1, limit_train_batches=batches, limit_val_batches=batches, weights_summary=None, ) trainer.fit(model) # make sure correct steps were called assert model.validation_step_called assert not model.validation_step_end_called assert not model.validation_epoch_end_called # no early stopping assert len(trainer.dev_debugger.early_stopping_history) == 0 # make sure we logged the exact number of metrics assert len( trainer.dev_debugger.logged_metrics) == epochs * batches + (epochs) assert len( trainer.dev_debugger.pbar_added_metrics) == epochs * batches + (epochs) # make sure we logged the correct epoch metrics total_empty_epoch_metrics = 0 epoch = 0 for metric in trainer.dev_debugger.logged_metrics: if 'epoch' in metric: epoch += 1 if len(metric) > 2: assert 'no_val_no_pbar' not in metric assert 'val_step_pbar_acc' not in metric assert metric[f'val_step_log_acc/epoch_{epoch}'] assert metric[f'val_step_log_pbar_acc/epoch_{epoch}'] else: total_empty_epoch_metrics += 1 assert total_empty_epoch_metrics == 3 # make sure we logged the correct epoch pbar metrics total_empty_epoch_metrics = 0 for metric in trainer.dev_debugger.pbar_added_metrics: if 'epoch' in metric: epoch += 1 if len(metric) > 2: assert 'no_val_no_pbar' not in metric assert 'val_step_log_acc' not in metric assert metric['val_step_log_pbar_acc'] assert metric['val_step_pbar_acc'] else: total_empty_epoch_metrics += 1 assert total_empty_epoch_metrics == 3 # only 1 checkpoint expected since values didn't change after that assert len(trainer.dev_debugger.checkpoint_callback_history) == 1 # make sure the last known metric is correct assert trainer.logger_connector.callback_metrics['checkpoint_on'] == 189