def test_get_scalars_with_pivot_table_with_missing_value(self):
mock_api_client = mock.Mock()
def stream_experiment_data(request, **kwargs):
self.assertEqual(request.experiment_id, "789")
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
response = export_service_pb2.StreamExperimentDataResponse()
response.run_name = "train"
response.tag_name = "batch_loss"
response.points.steps.append(0)
response.points.values.append(0.5)
response.points.wall_times.add(seconds=0, nanos=0)
response.points.steps.append(1)
response.points.values.append(0.25)
response.points.wall_times.add(seconds=1, nanos=0)
yield response
response = export_service_pb2.StreamExperimentDataResponse()
response.run_name = "train"
response.tag_name = "epoch_loss"
response.points.steps.append(0)
response.points.values.append(0.375)
response.points.wall_times.add(seconds=2, nanos=0)
yield response
mock_api_client.StreamExperimentData = mock.Mock(
wraps=stream_experiment_data
)
with mock.patch.object(
experiment_from_dev,
"get_api_client",
lambda api_endpoint: mock_api_client,
):
experiment = experiment_from_dev.ExperimentFromDev("789")
with self.assertRaisesRegex(
ValueError,
r"contains missing value\(s\).*different sets of "
r"steps.*pivot=False",
):
experiment.get_scalars(pivot=True)
def test_get_scalars_with_actual_inf_and_nan(self):
"""Test for get_scalars() call that involve inf and nan in user data."""
mock_api_client = mock.Mock()
def stream_experiment_data(request, **kwargs):
self.assertEqual(request.experiment_id, "789")
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
response = export_service_pb2.StreamExperimentDataResponse()
response.run_name = "train"
response.tag_name = "batch_loss"
response.points.steps.append(0)
response.points.values.append(np.nan)
response.points.wall_times.add(seconds=0, nanos=0)
response.points.steps.append(1)
response.points.values.append(np.inf)
response.points.wall_times.add(seconds=10, nanos=0)
yield response
mock_api_client.StreamExperimentData = mock.Mock(
wraps=stream_experiment_data)
with mock.patch.object(
experiment_from_dev,
"get_api_client",
lambda api_endpoint: mock_api_client,
):
experiment = experiment_from_dev.ExperimentFromDev("789")
dataframe = experiment.get_scalars()
expected = pandas.DataFrame({
"run": ["train"] * 2,
"tag": ["batch_loss"] * 2,
"step": [0, 1],
"value": [np.nan, np.inf],
"wall_time": [0.0, 10.0],
}).pivot_table(["value", "wall_time"], ["run", "step"], "tag")
pandas.testing.assert_frame_equal(dataframe,
expected,
check_names=True)
def test_get_scalars_works(self):
mock_api_client = mock.Mock()
def stream_experiment_data(request, **kwargs):
self.assertEqual(request.experiment_id, "789")
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
for run in ("train", "test"):
for tag in ("accuracy", "loss"):
response = export_service_pb2.StreamExperimentDataResponse(
)
response.run_name = run
response.tag_name = tag
display_name = "%s:%s" % (request.experiment_id, tag)
response.tag_metadata.CopyFrom(
test_util.scalar_metadata(display_name))
for step in range(10):
response.points.steps.append(step)
if tag == "loss":
if run == "train":
value = 1.0 / (step + 1)
seconds = step
else:
value = -1.0 / (step + 1)
seconds = 600 + step
else: # "accuracy"
if run == "train":
value = 1.0 / (10 - step)
seconds = step * 2
else:
value = -1.0 / (10 - step)
seconds = 600 + step * 2
response.points.values.append(value)
response.points.wall_times.add(seconds=seconds,
nanos=0)
yield response
mock_api_client.StreamExperimentData = mock.Mock(
wraps=stream_experiment_data)
with mock.patch.object(
experiment_from_dev,
"get_api_client",
lambda api_endpoint: mock_api_client,
):
experiment = experiment_from_dev.ExperimentFromDev("789")
for pivot in (False, True):
for include_wall_time in (False, True):
with self.subTest("pivot=%s; include_wall_time=%s" %
(pivot, include_wall_time)):
dataframe = experiment.get_scalars(
pivot=pivot, include_wall_time=include_wall_time)
if pivot:
run_key = (("run",
"") if include_wall_time else "run")
step_key = (("step",
"") if include_wall_time else "step")
accuracy_value_key = (("value", "accuracy")
if include_wall_time else
"accuracy")
loss_value_key = (("value", "loss")
if include_wall_time else "loss")
data = {
run_key: ["test"] * 10 + ["train"] * 10,
step_key:
np.concatenate(
[np.arange(0, 10),
np.arange(0, 10)]),
accuracy_value_key:
np.concatenate([
-1.0 / (10.0 - np.arange(0, 10)),
1.0 / (10.0 - np.arange(0, 10)),
], ),
loss_value_key:
np.concatenate([
-1.0 / (1.0 + np.arange(0, 10)),
1.0 / (1.0 + np.arange(0, 10)),
], ),
}
if include_wall_time:
data[("wall_time",
"accuracy")] = np.concatenate([
600.0 + 2.0 * np.arange(0, 10),
2.0 * np.arange(0, 10),
])
data[("wall_time", "loss")] = np.concatenate([
600.0 + np.arange(0, 10),
1.0 * np.arange(0, 10),
])
expected = pandas.DataFrame(data)
else: # No pivot_table.
data = {
"run": ["train"] * 20 + ["test"] * 20,
"tag": (["accuracy"] * 10 + ["loss"] * 10) * 2,
"step":
list(np.arange(0, 10)) * 4,
"value":
np.concatenate([
1.0 / (10.0 - np.arange(0, 10)),
1.0 / (1.0 + np.arange(0, 10)),
-1.0 / (10.0 - np.arange(0, 10)),
-1.0 / (1.0 + np.arange(0, 10)),
]),
}
if include_wall_time:
data["wall_time"] = np.concatenate([
2.0 * np.arange(0, 10),
1.0 * np.arange(0, 10),
600.0 + 2.0 * np.arange(0, 10),
600.0 + np.arange(0, 10),
])
expected = pandas.DataFrame(data)
pandas.testing.assert_frame_equal(
dataframe,
expected,
check_names=True,
)
def test_get_scalars_works(self):
mock_api_client = mock.Mock()
def stream_experiment_data(request, **kwargs):
self.assertEqual(request.experiment_id, "789")
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
for run in ("train", "test"):
for tag in ("accuracy", "loss"):
response = export_service_pb2.StreamExperimentDataResponse(
)
response.run_name = run
response.tag_name = tag
display_name = "%s:%s" % (request.experiment_id, tag)
response.tag_metadata.CopyFrom(
test_util.scalar_metadata(display_name))
for step in range(10):
response.points.steps.append(step)
if tag == "loss":
if run == "train":
value = 1.0 / (step + 1)
seconds = step
else:
value = -1.0 / (step + 1)
seconds = 600 + step
else: # "accuracy"
if run == "train":
value = 1.0 / (10 - step)
seconds = step * 2
else:
value = -1.0 / (10 - step)
seconds = 600 + step * 2
response.points.values.append(value)
response.points.wall_times.add(seconds=seconds,
nanos=0)
yield response
mock_api_client.StreamExperimentData = mock.Mock(
wraps=stream_experiment_data)
with mock.patch.object(
experiment_from_dev,
"get_api_client",
lambda api_endpoint: mock_api_client,
):
experiment = experiment_from_dev.ExperimentFromDev("789")
for pivot in (None, False):
with self.subTest("pivot=%s" % pivot):
dataframe = experiment.get_scalars(pivot=pivot)
expected = pandas.DataFrame({
"run": ["train"] * 20 + ["test"] * 20,
"tag": (["accuracy"] * 10 + ["loss"] * 10) * 2,
"step":
list(np.arange(0, 10)) * 4,
"wall_time":
np.concatenate([
2.0 * np.arange(0, 10),
1.0 * np.arange(0, 10),
600.0 + 2.0 * np.arange(0, 10),
600.0 + np.arange(0, 10),
]),
"value":
np.concatenate([
1.0 / (10.0 - np.arange(0, 10)),
1.0 / (1.0 + np.arange(0, 10)),
-1.0 / (10.0 - np.arange(0, 10)),
-1.0 / (1.0 + np.arange(0, 10)),
]),
})
if pivot is None: # Default behavior: pivot_table.
pandas.testing.assert_frame_equal(
dataframe,
expected.pivot_table(["value", "wall_time"],
["run", "step"], "tag"),
check_names=True,
)
else: # pivot == False
pandas.testing.assert_frame_equal(dataframe,
expected,
check_names=True)
def main(args):
experiment = experiment_from_dev.ExperimentFromDev(
args.experiment_id, api_endpoint=args.api_endpoint
)
dataframe = experiment.get_scalars()
print(dataframe)
def main(args):
experiment = experiment_from_dev.ExperimentFromDev(
args.experiment_id, api_endpoint=args.api_endpoint)
dataframe = experiment.get_scalars(
pivot=args.pivot, include_wall_time=args.include_wall_time)
print(dataframe)