def graph_impl(
self,
run,
tag,
is_conceptual,
experiment=None,
limit_attr_size=None,
large_attrs_key=None,
):
"""Result of the form `(body, mime_type)`, or `None` if no graph
exists."""
if self._data_provider:
graph_blob_sequences = self._data_provider.read_blob_sequences(
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
blob_datum_list = graph_blob_sequences.get(run, {}).get(tag, ())
try:
blob_ref = blob_datum_list[0].values[0]
except IndexError:
return None
# Always use the blob_key approach for now, even if there is a direct url.
graph_raw = self._data_provider.read_blob(blob_ref.blob_key)
# This method ultimately returns pbtxt, but we have to deserialize and
# later reserialize this anyway, because a) this way we accept binary
# protobufs too, and b) below we run `prepare_graph_for_ui` on the graph.
graph = graph_pb2.GraphDef.FromString(graph_raw)
elif is_conceptual:
tensor_events = self._multiplexer.Tensors(run, tag)
# Take the first event if there are multiple events written from different
# steps.
keras_model_config = json.loads(
tensor_events[0].tensor_proto.string_val[0]
)
graph = keras_util.keras_model_to_graph_def(keras_model_config)
elif tag:
tensor_events = self._multiplexer.Tensors(run, tag)
# Take the first event if there are multiple events written from different
# steps.
run_metadata = config_pb2.RunMetadata.FromString(
tensor_events[0].tensor_proto.string_val[0]
)
graph = graph_pb2.GraphDef()
for func_graph in run_metadata.function_graphs:
graph_util.combine_graph_defs(
graph, func_graph.pre_optimization_graph
)
else:
graph = self._multiplexer.Graph(run)
# This next line might raise a ValueError if the limit parameters
# are invalid (size is negative, size present but key absent, etc.).
process_graph.prepare_graph_for_ui(
graph, limit_attr_size, large_attrs_key
)
return (str(graph), "text/x-protobuf") # pbtxt
def _migrate_graph_event(old_event, experimental_filter_graph=False):
result = event_pb2.Event()
result.wall_time = old_event.wall_time
result.step = old_event.step
value = result.summary.value.add(tag=graphs_metadata.RUN_GRAPH_NAME)
graph_bytes = old_event.graph_def
# TODO(@davidsoergel): Move this stopgap to a more appropriate place.
if experimental_filter_graph:
try:
graph_def = graph_pb2.GraphDef().FromString(graph_bytes)
# The reason for the RuntimeWarning catch here is b/27494216, whereby
# some proto parsers incorrectly raise that instead of DecodeError
# on certain kinds of malformed input. Triggering this seems to require
# a combination of mysterious circumstances.
except (message.DecodeError, RuntimeWarning):
logger.warning(
"Could not parse GraphDef of size %d. Skipping.",
len(graph_bytes),
)
return (old_event, )
# Use the default filter parameters:
# limit_attr_size=1024, large_attrs_key="_too_large_attrs"
process_graph.prepare_graph_for_ui(graph_def)
graph_bytes = graph_def.SerializeToString()
value.tensor.CopyFrom(tensor_util.make_tensor_proto([graph_bytes]))
value.metadata.plugin_data.plugin_name = graphs_metadata.PLUGIN_NAME
# `value.metadata.plugin_data.content` left as the empty proto
value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE
# In the short term, keep both the old event and the new event to
# maintain compatibility.
return (old_event, result)
def graph_impl(self,
run,
tag,
is_conceptual,
limit_attr_size=None,
large_attrs_key=None):
"""Result of the form `(body, mime_type)`, or `None` if no graph exists."""
if is_conceptual:
tensor_events = self._multiplexer.Tensors(run, tag)
# Take the first event if there are multiple events written from different
# steps.
keras_model_config = json.loads(
tensor_events[0].tensor_proto.string_val[0])
graph = keras_util.keras_model_to_graph_def(keras_model_config)
elif tag:
tensor_events = self._multiplexer.Tensors(run, tag)
# Take the first event if there are multiple events written from different
# steps.
run_metadata = config_pb2.RunMetadata.FromString(
tensor_events[0].tensor_proto.string_val[0])
graph = graph_pb2.GraphDef()
for func_graph in run_metadata.function_graphs:
graph_util.combine_graph_defs(
graph, func_graph.pre_optimization_graph)
else:
graph = self._multiplexer.Graph(run)
self.graph = graph
# This next line might raise a ValueError if the limit parameters
# are invalid (size is negative, size present but key absent, etc.).
process_graph.prepare_graph_for_ui(graph, limit_attr_size,
large_attrs_key)
return (str(graph), 'text/x-protobuf') # pbtxt
def _migrate_graph_event(old_event, experimental_filter_graph=False):
result = event_pb2.Event()
result.wall_time = old_event.wall_time
result.step = old_event.step
value = result.summary.value.add(tag=graphs_metadata.RUN_GRAPH_NAME)
graph_bytes = old_event.graph_def
# TODO(@davidsoergel): Move this stopgap to a more appropriate place.
if experimental_filter_graph:
try:
graph_def = graph_pb2.GraphDef().FromString(graph_bytes)
except message.DecodeError:
logger.warning(
"Could not parse GraphDef of size %d. Skipping.",
len(graph_bytes),
)
return (old_event, )
# Use the default filter parameters:
# limit_attr_size=1024, large_attrs_key="_too_large_attrs"
process_graph.prepare_graph_for_ui(graph_def)
graph_bytes = graph_def.SerializeToString()
value.tensor.CopyFrom(tensor_util.make_tensor_proto([graph_bytes]))
value.metadata.plugin_data.plugin_name = graphs_metadata.PLUGIN_NAME
# `value.metadata.plugin_data.content` left as the empty proto
value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE
# In the short term, keep both the old event and the new event to
# maintain compatibility.
return (old_event, result)
def graph_impl(self, run, limit_attr_size=None, large_attrs_key=None):
"""Result of the form `(body, mime_type)`, or `None` if no graph exists."""
try:
graph = self._multiplexer.Graph(run)
except ValueError:
return None
# This next line might raise a ValueError if the limit parameters
# are invalid (size is negative, size present but key absent, etc.).
process_graph.prepare_graph_for_ui(graph, limit_attr_size, large_attrs_key)
return (str(graph), 'text/x-protobuf') # pbtxt
def graph_impl(self, run, limit_attr_size=None, large_attrs_key=None):
"""Result of the form `(body, mime_type)`, or `None` if no graph exists."""
try:
graph = self._multiplexer.Graph(run)
except ValueError:
return None
# This next line might raise a ValueError if the limit parameters
# are invalid (size is negative, size present but key absent, etc.).
process_graph.prepare_graph_for_ui(graph, limit_attr_size,
large_attrs_key)
return (str(graph), 'text/x-protobuf') # pbtxt
def graph_impl(
self,
ctx,
run,
tag,
is_conceptual,
experiment=None,
limit_attr_size=None,
large_attrs_key=None,
):
"""Result of the form `(body, mime_type)`; may raise `NotFound`."""
if is_conceptual:
keras_model_config = json.loads(
self._read_blob(
ctx,
experiment,
[metadata.PLUGIN_NAME_KERAS_MODEL],
run,
tag,
))
graph = keras_util.keras_model_to_graph_def(keras_model_config)
elif tag is None:
graph_raw = self._read_blob(
ctx,
experiment,
[metadata.PLUGIN_NAME],
run,
metadata.RUN_GRAPH_NAME,
)
graph = graph_pb2.GraphDef.FromString(graph_raw)
else:
# Op graph: could be either of two plugins. (Cf. `info_impl`.)
plugins = [
metadata.PLUGIN_NAME_RUN_METADATA,
metadata.PLUGIN_NAME_RUN_METADATA_WITH_GRAPH,
]
raw_run_metadata = self._read_blob(ctx, experiment, plugins, run,
tag)
run_metadata = config_pb2.RunMetadata.FromString(raw_run_metadata)
graph = graph_util.merge_graph_defs([
func_graph.pre_optimization_graph
for func_graph in run_metadata.function_graphs
])
# This next line might raise a ValueError if the limit parameters
# are invalid (size is negative, size present but key absent, etc.).
process_graph.prepare_graph_for_ui(graph, limit_attr_size,
large_attrs_key)
return (str(graph), "text/x-protobuf") # pbtxt
def _filtered_graph_bytes(graph_bytes):
try:
graph_def = graph_pb2.GraphDef().FromString(graph_bytes)
# The reason for the RuntimeWarning catch here is b/27494216, whereby
# some proto parsers incorrectly raise that instead of DecodeError
# on certain kinds of malformed input. Triggering this seems to require
# a combination of mysterious circumstances.
except (message.DecodeError, RuntimeWarning):
logger.warning(
"Could not parse GraphDef of size %d. Skipping.", len(graph_bytes),
)
return None
# Use the default filter parameters:
# limit_attr_size=1024, large_attrs_key="_too_large_attrs"
process_graph.prepare_graph_for_ui(graph_def)
return graph_def.SerializeToString()
