def test_break_at_scalar_point_boundary(self):
point_count = 2000 # comfortably saturates a single 1024-byte request
events = []
for step in range(point_count):
summary = scalar_v2.scalar_pb("loss", -2.0 * step)
if step > 0:
summary.value[0].ClearField("metadata")
events.append(event_pb2.Event(summary=summary, step=step))
run_to_events = {"train": events}
builder = uploader_lib._RequestBuilder("123")
requests = list(builder.build_requests(run_to_events))
for request in requests:
_clear_wall_times(request)
self.assertGreater(len(requests), 1)
self.assertLess(len(requests), point_count)
total_points_in_result = 0
for request in requests:
self.assertLen(request.runs, 1)
run = request.runs[0]
self.assertEqual(run.name, "train")
self.assertLen(run.tags, 1)
tag = run.tags[0]
self.assertEqual(tag.name, "loss")
for point in tag.points:
self.assertEqual(point.step, total_points_in_result)
self.assertEqual(point.value, -2.0 * point.step)
total_points_in_result += 1
self.assertLessEqual(request.ByteSize(),
uploader_lib._MAX_REQUEST_LENGTH_BYTES)
self.assertEqual(total_points_in_result, point_count)
def test_break_at_tag_boundary(self):
# Choose tag name sizes such that one tag fits, but not two. Note
# that tag names appear in both `Tag.name` and the summary metadata.
long_tag_1 = "a" * 384
long_tag_2 = "b" * 384
event = event_pb2.Event(step=1)
event.summary.value.add(tag=long_tag_1, simple_value=1.0)
event.summary.value.add(tag=long_tag_2, simple_value=2.0)
run_to_events = {"train": [event]}
builder = uploader_lib._RequestBuilder("123")
requests = list(builder.build_requests(run_to_events))
for request in requests:
_clear_wall_times(request)
expected = [
write_service_pb2.WriteScalarRequest(experiment_id="123"),
write_service_pb2.WriteScalarRequest(experiment_id="123"),
]
(expected[0].runs.add(name="train").tags.add(
name=long_tag_1,
metadata=test_util.scalar_metadata(long_tag_1)).points.add(
step=1, value=1.0))
(expected[1].runs.add(name="train").tags.add(
name=long_tag_2,
metadata=test_util.scalar_metadata(long_tag_2)).points.add(
step=1, value=2.0))
self.assertEqual(requests, expected)
def test_break_at_run_boundary(self):
# Choose run name sizes such that one run fits, but not two.
long_run_1 = "A" * 768
long_run_2 = "B" * 768
event_1 = event_pb2.Event(step=1)
event_1.summary.value.add(tag="foo", simple_value=1.0)
event_2 = event_pb2.Event(step=2)
event_2.summary.value.add(tag="bar", simple_value=-2.0)
run_to_events = collections.OrderedDict([
(long_run_1, [event_1]),
(long_run_2, [event_2]),
])
builder = uploader_lib._RequestBuilder("123")
requests = list(builder.build_requests(run_to_events))
for request in requests:
_clear_wall_times(request)
expected = [
write_service_pb2.WriteScalarRequest(experiment_id="123"),
write_service_pb2.WriteScalarRequest(experiment_id="123"),
]
(expected[0].runs.add(name=long_run_1).tags.add(
name="foo",
metadata=test_util.scalar_metadata("foo")).points.add(step=1,
value=1.0))
(expected[1].runs.add(name=long_run_2).tags.add(
name="bar",
metadata=test_util.scalar_metadata("bar")).points.add(step=2,
value=-2.0))
self.assertEqual(requests, expected)
def _populate_run_from_events(self, run_proto, events):
builder = uploader_lib._RequestBuilder(experiment_id="123")
requests = builder.build_requests({"": events})
request = next(requests, None)
if request is not None:
self.assertLen(request.runs, 1)
run_proto.MergeFrom(request.runs[0])
self.assertIsNone(next(requests, None))
def test_no_room_for_single_point(self):
event = event_pb2.Event(step=1, wall_time=123.456)
event.summary.value.add(tag="foo", simple_value=1.0)
long_run_name = "A" * uploader_lib._MAX_REQUEST_LENGTH_BYTES
run_to_events = {long_run_name: [event]}
with self.assertRaises(RuntimeError) as cm:
builder = uploader_lib._RequestBuilder("123")
list(builder.build_requests(run_to_events))
self.assertEqual(str(cm.exception),
"Could not make progress uploading data")
def test_no_budget_for_experiment_id(self):
event = event_pb2.Event(step=1, wall_time=123.456)
event.summary.value.add(tag="foo", simple_value=1.0)
run_to_events = {"run_name": [event]}
long_experiment_id = "A" * uploader_lib._MAX_REQUEST_LENGTH_BYTES
with self.assertRaises(RuntimeError) as cm:
builder = uploader_lib._RequestBuilder(long_experiment_id)
list(builder.build_requests(run_to_events))
self.assertEqual(str(cm.exception),
"Byte budget too small for experiment ID")
def test_prunes_tags_and_runs(self):
event_1 = event_pb2.Event(step=1)
event_1.summary.value.add(tag="foo", simple_value=1.0)
event_2 = event_pb2.Event(step=2)
event_2.summary.value.add(tag="bar", simple_value=-2.0)
run_to_events = collections.OrderedDict([
("train", [event_1]),
("test", [event_2]),
])
real_create_point = uploader_lib._RequestBuilder._create_point
create_point_call_count_box = [0]
def mock_create_point(uploader_self, *args, **kwargs):
# Simulate out-of-space error the first time that we try to store
# the second point.
create_point_call_count_box[0] += 1
if create_point_call_count_box[0] == 2:
raise uploader_lib._OutOfSpaceError()
return real_create_point(uploader_self, *args, **kwargs)
with mock.patch.object(uploader_lib._RequestBuilder, "_create_point",
mock_create_point):
builder = uploader_lib._RequestBuilder("123")
requests = list(builder.build_requests(run_to_events))
for request in requests:
_clear_wall_times(request)
expected = [
write_service_pb2.WriteScalarRequest(experiment_id="123"),
write_service_pb2.WriteScalarRequest(experiment_id="123"),
]
(expected[0].runs.add(name="train").tags.add(
name="foo",
metadata=test_util.scalar_metadata("foo")).points.add(step=1,
value=1.0))
(expected[1].runs.add(name="test").tags.add(
name="bar",
metadata=test_util.scalar_metadata("bar")).points.add(step=2,
value=-2.0))
self.assertEqual(expected, requests)
