def stream_experiments(request, **kwargs):
del request # unused
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["123", "456"])
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["789"])
def stream_experiments(request, **kwargs):
del request # unused
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["123", "456"]
)
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["789"]
)
def stream_experiments(request, **kwargs):
del request # unused
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
response = export_service_pb2.StreamExperimentsResponse()
response.experiments.add(experiment_id="123")
yield response
response = export_service_pb2.StreamExperimentsResponse()
response.experiments.add(experiment_id="456")
yield response
def stream_experiments(request, **kwargs):
del request # unused
self.assertEqual(kwargs["metadata"], grpc_util.version_metadata())
response = export_service_pb2.StreamExperimentsResponse()
response.experiments.add(experiment_id="123")
response.experiments.add(experiment_id="456")
yield response
response = export_service_pb2.StreamExperimentsResponse()
experiment = response.experiments.add()
experiment.experiment_id = "789"
experiment.name = "bert"
experiment.description = "ernie"
util.set_timestamp(experiment.create_time, 981173106)
util.set_timestamp(experiment.update_time, 1015218367)
yield response
def stream_experiments(request, **kwargs):
del request # unused
# Should ignore `experiment_ids` in the presence of `experiments`.
response = export_service_pb2.StreamExperimentsResponse()
response.experiment_ids.append("999") # will be omitted
response.experiments.add(experiment_id="789")
response.experiments.add(experiment_id="012")
yield response
def stream_experiments(request, **kwargs):
del request # unused
# Should include `experiment_ids` when no `experiments` given.
response = export_service_pb2.StreamExperimentsResponse()
response.experiment_ids.append("123")
response.experiment_ids.append("456")
yield response
# Should ignore `experiment_ids` in the presence of `experiments`.
response = export_service_pb2.StreamExperimentsResponse()
response.experiment_ids.append("999") # will be omitted
response.experiments.add(experiment_id="789")
response.experiments.add(experiment_id="012")
yield response
# Should include `experiments` even when no `experiment_ids` are given.
response = export_service_pb2.StreamExperimentsResponse()
response.experiments.add(experiment_id="345")
response.experiments.add(experiment_id="678")
yield response
def test_handles_outdir_with_no_slash(self):
oldcwd = os.getcwd()
try:
os.chdir(self.get_temp_dir())
mock_api_client = self._create_mock_api_client()
mock_api_client.StreamExperiments.return_value = iter([
export_service_pb2.StreamExperimentsResponse(experiment_ids=["123"]),
])
mock_api_client.StreamExperimentData.return_value = iter([
export_service_pb2.StreamExperimentDataResponse()
])
exporter = exporter_lib.TensorBoardExporter(mock_api_client, "outdir")
generator = exporter.export()
self.assertEqual(list(generator), ["123"])
self.assertTrue(os.path.isdir("outdir"))
finally:
os.chdir(oldcwd)
def stream_experiments(request, **kwargs):
del request # unused
response = export_service_pb2.StreamExperimentsResponse()
response.experiments.add(experiment_id="789", name="one")
response.experiments.add(experiment_id="012", description="two")
yield response
def _make_experiments_response(eids):
"""Make a `StreamExperimentsResponse` with experiments with only IDs."""
response = export_service_pb2.StreamExperimentsResponse()
for eid in eids:
response.experiments.add(experiment_id=eid)
return response
def test_e2e_success_case(self):
mock_api_client = self._create_mock_api_client()
mock_api_client.StreamExperiments.return_value = iter([
export_service_pb2.StreamExperimentsResponse(
experiment_ids=["789"]),
])
def stream_experiments(request):
del request # unused
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["123", "456"])
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["789"])
def stream_experiment_data(request):
for run in ("train", "test"):
for tag in ("accuracy", "loss"):
response = export_service_pb2.StreamExperimentDataResponse(
)
response.run_name = run
response.tag_name = tag
display_name = "%s:%s" % (request.experiment_id, tag)
response.tag_metadata.CopyFrom(
test_util.scalar_metadata(display_name))
for step in range(10):
response.points.steps.append(step)
response.points.values.append(2.0 * step)
response.points.wall_times.add(seconds=1571084520 +
step,
nanos=862939144)
yield response
mock_api_client.StreamExperiments = mock.Mock(wraps=stream_experiments)
mock_api_client.StreamExperimentData = mock.Mock(
wraps=stream_experiment_data)
outdir = os.path.join(self.get_temp_dir(), "outdir")
exporter = exporter_lib.TensorBoardExporter(mock_api_client, outdir)
start_time = 1571084846.25
start_time_pb = test_util.timestamp_pb(1571084846250000000)
generator = exporter.export(read_time=start_time)
expected_files = []
self.assertTrue(os.path.isdir(outdir))
self.assertCountEqual(expected_files, os.listdir(outdir))
mock_api_client.StreamExperiments.assert_not_called()
mock_api_client.StreamExperimentData.assert_not_called()
# The first iteration should request the list of experiments and
# data for one of them.
self.assertEqual(next(generator), "123")
expected_files.append("scalars_123.json")
self.assertCountEqual(expected_files, os.listdir(outdir))
expected_eids_request = export_service_pb2.StreamExperimentsRequest()
expected_eids_request.read_timestamp.CopyFrom(start_time_pb)
expected_eids_request.limit = 2**63 - 1
mock_api_client.StreamExperiments.assert_called_once_with(
expected_eids_request)
expected_data_request = export_service_pb2.StreamExperimentDataRequest(
)
expected_data_request.experiment_id = "123"
expected_data_request.read_timestamp.CopyFrom(start_time_pb)
mock_api_client.StreamExperimentData.assert_called_once_with(
expected_data_request)
# The next iteration should just request data for the next experiment.
mock_api_client.StreamExperiments.reset_mock()
mock_api_client.StreamExperimentData.reset_mock()
self.assertEqual(next(generator), "456")
expected_files.append("scalars_456.json")
self.assertCountEqual(expected_files, os.listdir(outdir))
mock_api_client.StreamExperiments.assert_not_called()
expected_data_request.experiment_id = "456"
mock_api_client.StreamExperimentData.assert_called_once_with(
expected_data_request)
# Again, request data for the next experiment; this experiment ID
# was in the second response batch in the list of IDs.
expected_files.append("scalars_789.json")
mock_api_client.StreamExperiments.reset_mock()
mock_api_client.StreamExperimentData.reset_mock()
self.assertEqual(next(generator), "789")
self.assertCountEqual(expected_files, os.listdir(outdir))
mock_api_client.StreamExperiments.assert_not_called()
expected_data_request.experiment_id = "789"
mock_api_client.StreamExperimentData.assert_called_once_with(
expected_data_request)
# The final continuation shouldn't need to send any RPCs.
mock_api_client.StreamExperiments.reset_mock()
mock_api_client.StreamExperimentData.reset_mock()
self.assertEqual(list(generator), [])
self.assertCountEqual(expected_files, os.listdir(outdir))
mock_api_client.StreamExperiments.assert_not_called()
mock_api_client.StreamExperimentData.assert_not_called()
# Spot-check one of the files.
with open(os.path.join(outdir, "scalars_456.json")) as infile:
jsons = [json.loads(line) for line in infile]
self.assertLen(jsons, 4)
datum = jsons[2]
self.assertEqual(datum.pop("run"), "test")
self.assertEqual(datum.pop("tag"), "accuracy")
summary_metadata = summary_pb2.SummaryMetadata.FromString(
base64.b64decode(datum.pop("summary_metadata")))
expected_summary_metadata = test_util.scalar_metadata("456:accuracy")
self.assertEqual(summary_metadata, expected_summary_metadata)
points = datum.pop("points")
expected_steps = [x for x in range(10)]
expected_values = [2.0 * x for x in range(10)]
expected_wall_times = [1571084520.862939144 + x for x in range(10)]
self.assertEqual(points.pop("steps"), expected_steps)
self.assertEqual(points.pop("values"), expected_values)
self.assertEqual(points.pop("wall_times"), expected_wall_times)
self.assertEqual(points, {})
self.assertEqual(datum, {})
def _make_experiments_response(self, eids):
return export_service_pb2.StreamExperimentsResponse(
experiment_ids=eids)
def stream_experiments(request):
del request # unused
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["123"])
def stream_experiments(request):
del request # unused
yield export_service_pb2.StreamExperimentsResponse(
experiment_ids=["../authorized_keys"])
