def test_metric_gathering() -> None:
"""
Confirm that metrics are gathered from the trial the way that we expect.
"""
experiment_id = exp.run_basic_test(
conf.fixtures_path("metric_maker/const.yaml"),
conf.fixtures_path("metric_maker"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
# Read the structure of the metrics directly from the config file
config = conf.load_config(conf.fixtures_path("metric_maker/const.yaml"))
base_value = config["hyperparameters"]["starting_base_value"]
gain_per_batch = config["hyperparameters"]["gain_per_batch"]
training_structure = config["hyperparameters"]["training_structure"]["val"]
validation_structure = config["hyperparameters"]["validation_structure"][
"val"]
scheduling_unit = 100
# Check training metrics.
full_trial_metrics = exp.trial_metrics(trials[0].trial.id)
batches_trained = 0
for step in full_trial_metrics["steps"]:
metrics = step["metrics"]
actual = metrics["batch_metrics"]
assert len(actual) == scheduling_unit
first_base_value = base_value + batches_trained
batch_values = first_base_value + gain_per_batch * np.arange(
scheduling_unit)
expected = [
structure_to_metrics(value, training_structure)
for value in batch_values
]
assert structure_equal(expected, actual)
batches_trained = step["total_batches"]
# Check validation metrics.
validation_workloads = exp.workloads_with_validation(trials[0].workloads)
for validation in validation_workloads:
actual = validation.metrics
batches_trained = validation.totalBatches
value = base_value + batches_trained
expected = structure_to_metrics(value, validation_structure)
assert structure_equal(expected, actual)
def test_metric_gathering() -> None:
"""
Confirm that metrics are gathered from the trial the way that we expect.
"""
experiment_id = exp.run_basic_test(
conf.fixtures_path("metric_maker/const.yaml"),
conf.fixtures_path("metric_maker"), 1)
trials = exp.experiment_trials(experiment_id)
assert len(trials) == 1
# Read the structure of the metrics directly from the config file
config = conf.load_config(conf.fixtures_path("metric_maker/const.yaml"))
base_value = config["hyperparameters"]["starting_base_value"]
gain_per_batch = config["hyperparameters"]["gain_per_batch"]
training_structure = config["hyperparameters"]["training_structure"]["val"]
validation_structure = config["hyperparameters"]["validation_structure"][
"val"]
scheduling_unit = 100
# Check training metrics.
full_trial_metrics = exp.trial_metrics(trials[0]["id"])
for step in full_trial_metrics["steps"]:
metrics = step["metrics"]
assert metrics["num_inputs"] == scheduling_unit
actual = metrics["batch_metrics"]
assert len(actual) == scheduling_unit
first_base_value = base_value + (step["id"] - 1) * scheduling_unit
batch_values = first_base_value + gain_per_batch * np.arange(
scheduling_unit)
expected = [
structure_to_metrics(value, training_structure)
for value in batch_values
]
assert structure_equal(expected, actual)
# Check validation metrics.
for step in trials[0]["steps"]:
validation = step["validation"]
metrics = validation["metrics"]
actual = metrics["validation_metrics"]
value = base_value + step["id"] * scheduling_unit
expected = structure_to_metrics(value, validation_structure)
assert structure_equal(expected, actual)
def test_nan_metrics() -> None:
"""
Confirm that NaN and Infinity metrics are gathered from the trial.
"""
exp_id = exp.run_basic_test(conf.fixtures_path("metric_maker/nans.yaml"),
conf.fixtures_path("metric_maker"), 1)
trials = exp.experiment_trials(exp_id)
config = conf.load_config(conf.fixtures_path("metric_maker/nans.yaml"))
base_value = config["hyperparameters"]["starting_base_value"]
gain_per_batch = config["hyperparameters"]["gain_per_batch"]
# Infinity and NaN cannot be processed in the YAML->JSON deserializer
# Add them to expected values here
training_structure = config["hyperparameters"]["training_structure"]["val"]
training_structure["inf"] = "Infinity"
training_structure["nan"] = "NaN"
training_structure["nanarray"] = ["NaN", "NaN"]
validation_structure = config["hyperparameters"]["validation_structure"][
"val"]
validation_structure["neg_inf"] = "-Infinity"
# Check training metrics.
full_trial_metrics = exp.trial_metrics(trials[0].trial.id)
batches_trained = 0
for step in full_trial_metrics["steps"]:
metrics = step["metrics"]
actual = metrics["batch_metrics"]
first_base_value = base_value + batches_trained
batch_values = first_base_value + gain_per_batch * np.arange(5)
expected = [
structure_to_metrics(value, training_structure)
for value in batch_values
]
assert structure_equal(expected, actual)
batches_trained = step["total_batches"]
# Check validation metrics.
validation_workloads = exp.workloads_with_validation(trials[0].workloads)
for validation in validation_workloads:
actual = validation.metrics
batches_trained = validation.totalBatches
expected = structure_to_metrics(base_value, validation_structure)
assert structure_equal(expected, actual)
