def graph_impl( self, run, tag, is_conceptual, experiment=None, limit_attr_size=None, large_attrs_key=None, ): """Result of the form `(body, mime_type)`, or `None` if no graph exists.""" if self._data_provider: graph_blob_sequences = self._data_provider.read_blob_sequences( experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) blob_datum_list = graph_blob_sequences.get(run, {}).get(tag, ()) try: blob_ref = blob_datum_list[0].values[0] except IndexError: return None # Always use the blob_key approach for now, even if there is a direct url. graph_raw = self._data_provider.read_blob(blob_ref.blob_key) # This method ultimately returns pbtxt, but we have to deserialize and # later reserialize this anyway, because a) this way we accept binary # protobufs too, and b) below we run `prepare_graph_for_ui` on the graph. graph = graph_pb2.GraphDef.FromString(graph_raw) elif is_conceptual: tensor_events = self._multiplexer.Tensors(run, tag) # Take the first event if there are multiple events written from different # steps. keras_model_config = json.loads( tensor_events[0].tensor_proto.string_val[0] ) graph = keras_util.keras_model_to_graph_def(keras_model_config) elif tag: tensor_events = self._multiplexer.Tensors(run, tag) # Take the first event if there are multiple events written from different # steps. run_metadata = config_pb2.RunMetadata.FromString( tensor_events[0].tensor_proto.string_val[0] ) graph = graph_pb2.GraphDef() for func_graph in run_metadata.function_graphs: graph_util.combine_graph_defs( graph, func_graph.pre_optimization_graph ) else: graph = self._multiplexer.Graph(run) # This next line might raise a ValueError if the limit parameters # are invalid (size is negative, size present but key absent, etc.). process_graph.prepare_graph_for_ui( graph, limit_attr_size, large_attrs_key ) return (str(graph), "text/x-protobuf") # pbtxt
def _migrate_graph_event(old_event, experimental_filter_graph=False): result = event_pb2.Event() result.wall_time = old_event.wall_time result.step = old_event.step value = result.summary.value.add(tag=graphs_metadata.RUN_GRAPH_NAME) graph_bytes = old_event.graph_def # TODO(@davidsoergel): Move this stopgap to a more appropriate place. if experimental_filter_graph: try: graph_def = graph_pb2.GraphDef().FromString(graph_bytes) # The reason for the RuntimeWarning catch here is b/27494216, whereby # some proto parsers incorrectly raise that instead of DecodeError # on certain kinds of malformed input. Triggering this seems to require # a combination of mysterious circumstances. except (message.DecodeError, RuntimeWarning): logger.warning( "Could not parse GraphDef of size %d. Skipping.", len(graph_bytes), ) return (old_event, ) # Use the default filter parameters: # limit_attr_size=1024, large_attrs_key="_too_large_attrs" process_graph.prepare_graph_for_ui(graph_def) graph_bytes = graph_def.SerializeToString() value.tensor.CopyFrom(tensor_util.make_tensor_proto([graph_bytes])) value.metadata.plugin_data.plugin_name = graphs_metadata.PLUGIN_NAME # `value.metadata.plugin_data.content` left as the empty proto value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE # In the short term, keep both the old event and the new event to # maintain compatibility. return (old_event, result)
def graph_impl(self, run, tag, is_conceptual, limit_attr_size=None, large_attrs_key=None): """Result of the form `(body, mime_type)`, or `None` if no graph exists.""" if is_conceptual: tensor_events = self._multiplexer.Tensors(run, tag) # Take the first event if there are multiple events written from different # steps. keras_model_config = json.loads( tensor_events[0].tensor_proto.string_val[0]) graph = keras_util.keras_model_to_graph_def(keras_model_config) elif tag: tensor_events = self._multiplexer.Tensors(run, tag) # Take the first event if there are multiple events written from different # steps. run_metadata = config_pb2.RunMetadata.FromString( tensor_events[0].tensor_proto.string_val[0]) graph = graph_pb2.GraphDef() for func_graph in run_metadata.function_graphs: graph_util.combine_graph_defs( graph, func_graph.pre_optimization_graph) else: graph = self._multiplexer.Graph(run) self.graph = graph # This next line might raise a ValueError if the limit parameters # are invalid (size is negative, size present but key absent, etc.). process_graph.prepare_graph_for_ui(graph, limit_attr_size, large_attrs_key) return (str(graph), 'text/x-protobuf') # pbtxt
def _migrate_graph_event(old_event, experimental_filter_graph=False): result = event_pb2.Event() result.wall_time = old_event.wall_time result.step = old_event.step value = result.summary.value.add(tag=graphs_metadata.RUN_GRAPH_NAME) graph_bytes = old_event.graph_def # TODO(@davidsoergel): Move this stopgap to a more appropriate place. if experimental_filter_graph: try: graph_def = graph_pb2.GraphDef().FromString(graph_bytes) except message.DecodeError: logger.warning( "Could not parse GraphDef of size %d. Skipping.", len(graph_bytes), ) return (old_event, ) # Use the default filter parameters: # limit_attr_size=1024, large_attrs_key="_too_large_attrs" process_graph.prepare_graph_for_ui(graph_def) graph_bytes = graph_def.SerializeToString() value.tensor.CopyFrom(tensor_util.make_tensor_proto([graph_bytes])) value.metadata.plugin_data.plugin_name = graphs_metadata.PLUGIN_NAME # `value.metadata.plugin_data.content` left as the empty proto value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE # In the short term, keep both the old event and the new event to # maintain compatibility. return (old_event, result)
def graph_impl(self, run, limit_attr_size=None, large_attrs_key=None): """Result of the form `(body, mime_type)`, or `None` if no graph exists.""" try: graph = self._multiplexer.Graph(run) except ValueError: return None # This next line might raise a ValueError if the limit parameters # are invalid (size is negative, size present but key absent, etc.). process_graph.prepare_graph_for_ui(graph, limit_attr_size, large_attrs_key) return (str(graph), 'text/x-protobuf') # pbtxt
def graph_impl( self, ctx, run, tag, is_conceptual, experiment=None, limit_attr_size=None, large_attrs_key=None, ): """Result of the form `(body, mime_type)`; may raise `NotFound`.""" if is_conceptual: keras_model_config = json.loads( self._read_blob( ctx, experiment, [metadata.PLUGIN_NAME_KERAS_MODEL], run, tag, )) graph = keras_util.keras_model_to_graph_def(keras_model_config) elif tag is None: graph_raw = self._read_blob( ctx, experiment, [metadata.PLUGIN_NAME], run, metadata.RUN_GRAPH_NAME, ) graph = graph_pb2.GraphDef.FromString(graph_raw) else: # Op graph: could be either of two plugins. (Cf. `info_impl`.) plugins = [ metadata.PLUGIN_NAME_RUN_METADATA, metadata.PLUGIN_NAME_RUN_METADATA_WITH_GRAPH, ] raw_run_metadata = self._read_blob(ctx, experiment, plugins, run, tag) run_metadata = config_pb2.RunMetadata.FromString(raw_run_metadata) graph = graph_util.merge_graph_defs([ func_graph.pre_optimization_graph for func_graph in run_metadata.function_graphs ]) # This next line might raise a ValueError if the limit parameters # are invalid (size is negative, size present but key absent, etc.). process_graph.prepare_graph_for_ui(graph, limit_attr_size, large_attrs_key) return (str(graph), "text/x-protobuf") # pbtxt
def _filtered_graph_bytes(graph_bytes): try: graph_def = graph_pb2.GraphDef().FromString(graph_bytes) # The reason for the RuntimeWarning catch here is b/27494216, whereby # some proto parsers incorrectly raise that instead of DecodeError # on certain kinds of malformed input. Triggering this seems to require # a combination of mysterious circumstances. except (message.DecodeError, RuntimeWarning): logger.warning( "Could not parse GraphDef of size %d. Skipping.", len(graph_bytes), ) return None # Use the default filter parameters: # limit_attr_size=1024, large_attrs_key="_too_large_attrs" process_graph.prepare_graph_for_ui(graph_def) return graph_def.SerializeToString()