def _build_session_groups(self):
"""Returns a list of SessionGroups protobuffers from the summary data."""
# Algorithm: We keep a dict 'groups_by_name' mapping a SessionGroup name
# (str) to a SessionGroup protobuffer. We traverse the runs associated with
# the plugin--each representing a single session. We form a Session
# protobuffer from each run and add it to the relevant SessionGroup object
# in the 'groups_by_name' dict. We create the SessionGroup object, if this
# is the first session of that group we encounter.
groups_by_name = {}
run_to_tag_to_content = self._context.multiplexer.PluginRunToTagToContent(
metadata.PLUGIN_NAME)
for (run, tag_to_content) in six.iteritems(run_to_tag_to_content):
if metadata.SESSION_START_INFO_TAG not in tag_to_content:
continue
start_info = metadata.parse_session_start_info_plugin_data(
tag_to_content[metadata.SESSION_START_INFO_TAG])
end_info = None
if metadata.SESSION_END_INFO_TAG in tag_to_content:
end_info = metadata.parse_session_end_info_plugin_data(
tag_to_content[metadata.SESSION_END_INFO_TAG])
session = self._build_session(run, start_info, end_info)
if session.status in self._request.allowed_statuses:
self._add_session(session, start_info, groups_by_name)
# Compute the session group's aggregated metrics for each group.
groups = groups_by_name.values()
for group in groups:
# We sort the sessions in a group so that the order is deterministic.
group.sessions.sort(key=operator.attrgetter('name'))
self._aggregate_metrics(group)
return groups
def _build_session_groups(self):
"""Returns a list of SessionGroups protobuffers from the summary
data."""
# Algorithm: We keep a dict 'groups_by_name' mapping a SessionGroup name
# (str) to a SessionGroup protobuffer. We traverse the runs associated with
# the plugin--each representing a single session. We form a Session
# protobuffer from each run and add it to the relevant SessionGroup object
# in the 'groups_by_name' dict. We create the SessionGroup object, if this
# is the first session of that group we encounter.
groups_by_name = {}
run_to_tag_to_content = self._context.hparams_metadata(
self._experiment_id,
run_tag_filter=provider.RunTagFilter(tags=[
metadata.SESSION_START_INFO_TAG,
metadata.SESSION_END_INFO_TAG,
]),
)
# The TensorBoard runs with session start info are the
# "sessions", which are not necessarily the runs that actually
# contain metrics (may be in subdirectories).
session_names = [
run for (run, tags) in run_to_tag_to_content.items()
if metadata.SESSION_START_INFO_TAG in tags
]
metric_runs = set()
metric_tags = set()
for session_name in session_names:
for metric in self._experiment.metric_infos:
metric_name = metric.name
(run, tag) = metrics.run_tag_from_session_and_metric(
session_name, metric_name)
metric_runs.add(run)
metric_tags.add(tag)
all_metric_evals = self._context.read_last_scalars(
self._experiment_id,
run_tag_filter=provider.RunTagFilter(runs=metric_runs,
tags=metric_tags),
)
for (session_name, tag_to_content) in run_to_tag_to_content.items():
if metadata.SESSION_START_INFO_TAG not in tag_to_content:
continue
start_info = metadata.parse_session_start_info_plugin_data(
tag_to_content[metadata.SESSION_START_INFO_TAG])
end_info = None
if metadata.SESSION_END_INFO_TAG in tag_to_content:
end_info = metadata.parse_session_end_info_plugin_data(
tag_to_content[metadata.SESSION_END_INFO_TAG])
session = self._build_session(session_name, start_info, end_info,
all_metric_evals)
if session.status in self._request.allowed_statuses:
self._add_session(session, start_info, groups_by_name)
# Compute the session group's aggregated metrics for each group.
groups = groups_by_name.values()
for group in groups:
# We sort the sessions in a group so that the order is deterministic.
group.sessions.sort(key=operator.attrgetter("name"))
self._aggregate_metrics(group)
return groups
def _build_session_infos_by_name(self):
run_to_tag_to_content = self._context.multiplexer.PluginRunToTagToContent(
metadata.PLUGIN_NAME)
result = {}
for (run, tag_to_content) in six.iteritems(run_to_tag_to_content):
if metadata.SESSION_START_INFO_TAG not in tag_to_content:
continue
start_info = metadata.parse_session_start_info_plugin_data(
tag_to_content[metadata.SESSION_START_INFO_TAG])
# end_info will be None if the corresponding tag doesn't exist.
end_info = None
if metadata.SESSION_END_INFO_TAG in tag_to_content:
end_info = metadata.parse_session_end_info_plugin_data(
tag_to_content[metadata.SESSION_END_INFO_TAG])
result[run] = self._SessionInfoTuple(start_info=start_info,
end_info=end_info)
return result
def _build_session_infos_by_name(self):
run_to_tag_to_content = self._context.multiplexer.PluginRunToTagToContent(
metadata.PLUGIN_NAME)
result = {}
for (run, tag_to_content) in six.iteritems(run_to_tag_to_content):
if metadata.SESSION_START_INFO_TAG not in tag_to_content:
continue
start_info = metadata.parse_session_start_info_plugin_data(
tag_to_content[metadata.SESSION_START_INFO_TAG])
# end_info will be None if the corresponding tag doesn't exist.
end_info = None
if metadata.SESSION_END_INFO_TAG in tag_to_content:
end_info = metadata.parse_session_end_info_plugin_data(
tag_to_content[metadata.SESSION_END_INFO_TAG])
result[run] = self._SessionInfoTuple(start_info=start_info,
end_info=end_info)
return result
def test_eager(self):
def mock_time():
mock_time.time += 1
return mock_time.time
mock_time.time = 1556227801.875
initial_time = mock_time.time
with mock.patch("time.time", mock_time):
self._initialize_model(writer=self.logdir)
self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback])
final_time = mock_time.time
files = os.listdir(self.logdir)
self.assertEqual(len(files), 1, files)
events_file = os.path.join(self.logdir, files[0])
plugin_data = []
for event in tf.compat.v1.train.summary_iterator(events_file):
if event.WhichOneof("what") != "summary":
continue
self.assertEqual(len(event.summary.value), 1, event.summary.value)
value = event.summary.value[0]
self.assertEqual(
value.metadata.plugin_data.plugin_name,
metadata.PLUGIN_NAME,
)
plugin_data.append(value.metadata.plugin_data.content)
self.assertEqual(len(plugin_data), 2, plugin_data)
(start_plugin_data, end_plugin_data) = plugin_data
start_pb = metadata.parse_session_start_info_plugin_data(start_plugin_data)
end_pb = metadata.parse_session_end_info_plugin_data(end_plugin_data)
# We're not the only callers of `time.time`; Keras calls it
# internally an unspecified number of times, so we're not guaranteed
# to know the exact values. Instead, we perform relative checks...
self.assertGreater(start_pb.start_time_secs, initial_time)
self.assertLess(start_pb.start_time_secs, end_pb.end_time_secs)
self.assertLessEqual(start_pb.start_time_secs, final_time)
# ...and then stub out the times for proto equality checks below.
start_pb.start_time_secs = 1234.5
end_pb.end_time_secs = 6789.0
expected_start_pb = plugin_data_pb2.SessionStartInfo()
text_format.Merge(
"""
start_time_secs: 1234.5
group_name: "my_trial"
hparams {
key: "optimizer"
value {
string_value: "adam"
}
}
hparams {
key: "dense_neurons"
value {
number_value: 8.0
}
}
""",
expected_start_pb,
)
self.assertEqual(start_pb, expected_start_pb)
expected_end_pb = plugin_data_pb2.SessionEndInfo()
text_format.Merge(
"""
end_time_secs: 6789.0
status: STATUS_SUCCESS
""",
expected_end_pb,
)
self.assertEqual(end_pb, expected_end_pb)
