def test_metric_gathering() -> None: """ Confirm that metrics are gathered from the trial the way that we expect. """ experiment_id = exp.run_basic_test( conf.fixtures_path("metric_maker/const.yaml"), conf.fixtures_path("metric_maker"), 1) trials = exp.experiment_trials(experiment_id) assert len(trials) == 1 # Read the structure of the metrics directly from the config file config = conf.load_config(conf.fixtures_path("metric_maker/const.yaml")) base_value = config["hyperparameters"]["starting_base_value"] gain_per_batch = config["hyperparameters"]["gain_per_batch"] training_structure = config["hyperparameters"]["training_structure"]["val"] validation_structure = config["hyperparameters"]["validation_structure"][ "val"] scheduling_unit = 100 # Check training metrics. full_trial_metrics = exp.trial_metrics(trials[0].trial.id) batches_trained = 0 for step in full_trial_metrics["steps"]: metrics = step["metrics"] actual = metrics["batch_metrics"] assert len(actual) == scheduling_unit first_base_value = base_value + batches_trained batch_values = first_base_value + gain_per_batch * np.arange( scheduling_unit) expected = [ structure_to_metrics(value, training_structure) for value in batch_values ] assert structure_equal(expected, actual) batches_trained = step["total_batches"] # Check validation metrics. validation_workloads = exp.workloads_with_validation(trials[0].workloads) for validation in validation_workloads: actual = validation.metrics batches_trained = validation.totalBatches value = base_value + batches_trained expected = structure_to_metrics(value, validation_structure) assert structure_equal(expected, actual)
def test_metric_gathering() -> None: """ Confirm that metrics are gathered from the trial the way that we expect. """ experiment_id = exp.run_basic_test( conf.fixtures_path("metric_maker/const.yaml"), conf.fixtures_path("metric_maker"), 1) trials = exp.experiment_trials(experiment_id) assert len(trials) == 1 # Read the structure of the metrics directly from the config file config = conf.load_config(conf.fixtures_path("metric_maker/const.yaml")) base_value = config["hyperparameters"]["starting_base_value"] gain_per_batch = config["hyperparameters"]["gain_per_batch"] training_structure = config["hyperparameters"]["training_structure"]["val"] validation_structure = config["hyperparameters"]["validation_structure"][ "val"] scheduling_unit = 100 # Check training metrics. full_trial_metrics = exp.trial_metrics(trials[0]["id"]) for step in full_trial_metrics["steps"]: metrics = step["metrics"] assert metrics["num_inputs"] == scheduling_unit actual = metrics["batch_metrics"] assert len(actual) == scheduling_unit first_base_value = base_value + (step["id"] - 1) * scheduling_unit batch_values = first_base_value + gain_per_batch * np.arange( scheduling_unit) expected = [ structure_to_metrics(value, training_structure) for value in batch_values ] assert structure_equal(expected, actual) # Check validation metrics. for step in trials[0]["steps"]: validation = step["validation"] metrics = validation["metrics"] actual = metrics["validation_metrics"] value = base_value + step["id"] * scheduling_unit expected = structure_to_metrics(value, validation_structure) assert structure_equal(expected, actual)
def test_nan_metrics() -> None: """ Confirm that NaN and Infinity metrics are gathered from the trial. """ exp_id = exp.run_basic_test(conf.fixtures_path("metric_maker/nans.yaml"), conf.fixtures_path("metric_maker"), 1) trials = exp.experiment_trials(exp_id) config = conf.load_config(conf.fixtures_path("metric_maker/nans.yaml")) base_value = config["hyperparameters"]["starting_base_value"] gain_per_batch = config["hyperparameters"]["gain_per_batch"] # Infinity and NaN cannot be processed in the YAML->JSON deserializer # Add them to expected values here training_structure = config["hyperparameters"]["training_structure"]["val"] training_structure["inf"] = "Infinity" training_structure["nan"] = "NaN" training_structure["nanarray"] = ["NaN", "NaN"] validation_structure = config["hyperparameters"]["validation_structure"][ "val"] validation_structure["neg_inf"] = "-Infinity" # Check training metrics. full_trial_metrics = exp.trial_metrics(trials[0].trial.id) batches_trained = 0 for step in full_trial_metrics["steps"]: metrics = step["metrics"] actual = metrics["batch_metrics"] first_base_value = base_value + batches_trained batch_values = first_base_value + gain_per_batch * np.arange(5) expected = [ structure_to_metrics(value, training_structure) for value in batch_values ] assert structure_equal(expected, actual) batches_trained = step["total_batches"] # Check validation metrics. validation_workloads = exp.workloads_with_validation(trials[0].workloads) for validation in validation_workloads: actual = validation.metrics batches_trained = validation.totalBatches expected = structure_to_metrics(base_value, validation_structure) assert structure_equal(expected, actual)