def _event_parse(event_str, latest_file_name): """ Transform `Event` data to tensor_event and update it to EventsData. This method is static to avoid sending unnecessary objects to other processes. Args: event_str (str): Message event string in summary proto, data read from file handler. latest_file_name (str): Latest file name. """ plugins = { 'scalar_value': PluginNameEnum.SCALAR, 'image': PluginNameEnum.IMAGE, 'histogram': PluginNameEnum.HISTOGRAM, 'tensor': PluginNameEnum.TENSOR } logger.debug("Start to parse event string. Event string len: %s.", len(event_str)) event = summary_pb2.Event.FromString(event_str) logger.debug("Deserialize event string completed.") ret_tensor_events = [] if event.HasField('summary'): for value in event.summary.value: for plugin in plugins: if not value.HasField(plugin): continue plugin_name_enum = plugins[plugin] logger.debug("Processing plugin value: %s.", plugin_name_enum) tensor_event_value = _SummaryParser._parse_summary_value( value, plugin) if tensor_event_value is None: continue tensor_event = TensorEvent( wall_time=event.wall_time, step=event.step, tag='{}/{}'.format(value.tag, plugin_name_enum.value), plugin_name=plugin_name_enum.value, value=tensor_event_value, filename=latest_file_name) logger.debug( "Tensor event generated, plugin is %s, tag is %s, step is %s.", plugin_name_enum, value.tag, event.step) ret_tensor_events.append(tensor_event) elif event.HasField('graph_def'): graph = MSGraph() graph.build_graph(event.graph_def) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=latest_file_name, plugin_name=PluginNameEnum.GRAPH.value, value=graph, filename=latest_file_name) ret_tensor_events.append(tensor_event) return ret_tensor_events
def _event_parse(self, event): """ Transform `Event` data to tensor_event and update it to EventsData. Args: event (Event): Message event in summary proto, data read from file handler. """ if event.HasField('summary'): for value in event.summary.value: if value.HasField('scalar_value'): tag = '{}/{}'.format(value.tag, PluginNameEnum.SCALAR.value) tensor_event = TensorEvent( wall_time=event.wall_time, step=event.step, tag=tag, plugin_name=PluginNameEnum.SCALAR.value, value=value.scalar_value) self._events_data.add_tensor_event(tensor_event) if value.HasField('image'): tag = '{}/{}'.format(value.tag, PluginNameEnum.IMAGE.value) tensor_event = TensorEvent( wall_time=event.wall_time, step=event.step, tag=tag, plugin_name=PluginNameEnum.IMAGE.value, value=value.image) self._events_data.add_tensor_event(tensor_event) if event.HasField('graph_def'): graph_proto = event.graph_def graph = MSGraph() graph.build_graph(graph_proto) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=self._latest_summary_filename, plugin_name=PluginNameEnum.GRAPH.value, value=graph) try: graph_tags = self._events_data.list_tags_by_plugin( PluginNameEnum.GRAPH.value) except KeyError: graph_tags = [] summary_tags = self._filter_summary_files(graph_tags) for tag in summary_tags: self._events_data.delete_tensor_event(tag) self._events_data.add_tensor_event(tensor_event)
def _parse_pb_file(self, filename): """ Parse pb file and write content to `EventsData`. Args: filename (str): The file path of pb file. """ file_path = FileHandler.join(self._summary_dir, filename) logger.info("Start to load graph from pb file, file path: %s.", file_path) filehandler = FileHandler(file_path) model_proto = anf_ir_pb2.ModelProto() try: model_proto.ParseFromString(filehandler.read()) except ParseError: logger.warning( "The given file is not a valid pb file, file path: %s.", file_path) return graph = MSGraph() graph.build_graph(model_proto.graph) tensor_event = TensorEvent(wall_time=FileHandler.file_stat(file_path), step=0, tag=filename, plugin_name=PluginNameEnum.GRAPH.value, value=graph) self._events_data.add_tensor_event(tensor_event)
def test_add_tensor_event_success(self): """Test add_tensor_event success.""" ev_data = self.get_ev_data() t_event = TensorEvent(wall_time=1, step=4, tag='new_tag', plugin_name='plugin_name1', value='value1') ev_data.add_tensor_event(t_event) assert 'tag0' not in ev_data._tags assert ev_data._tags[-1] == 'new_tag' assert 'tag0' not in ev_data._tags_by_plugin['plugin_name1'] assert 'tag0' not in ev_data._reservoir_by_tag assert 'new_tag' in ev_data._tags_by_plugin['plugin_name1'] assert ev_data._reservoir_by_tag['new_tag'].samples()[-1] == _Tensor(t_event.wall_time, t_event.step, t_event.value)
def _parse_pb_file(summary_dir, filename): """ Parse pb file and write content to `EventsData`. Args: filename (str): The file path of pb file. Returns: TensorEvent, if load pb file and build graph success, will return tensor event, else return None. """ file_path = FileHandler.join(summary_dir, filename) logger.info("Start to load graph from pb file, file path: %s.", file_path) filehandler = FileHandler(file_path) model_proto = anf_ir_pb2.ModelProto() try: model_proto.ParseFromString(filehandler.read()) except ParseError: logger.warning( "The given file is not a valid pb file, file path: %s.", file_path) return None graph = MSGraph() try: graph.build_graph(model_proto.graph) except Exception as ex: # Normally, there are no exceptions, and it is only possible for users on the MindSpore side # to dump other non-default graphs. logger.error("Build graph failed, file path: %s.", file_path) logger.exception(ex) raise UnknownError(str(ex)) tensor_event = TensorEvent( wall_time=FileHandler.file_stat(file_path).mtime, step=0, tag=filename, plugin_name=PluginNameEnum.GRAPH.value, value=graph, filename=filename) logger.info("Build graph success, file path: %s.", file_path) return tensor_event
def _event_parse(self, event): """ Transform `Event` data to tensor_event and update it to EventsData. Args: event (Event): Message event in summary proto, data read from file handler. """ if event.HasField('summary'): for value in event.summary.value: if value.HasField('scalar_value'): tag = '{}/{}'.format(value.tag, PluginNameEnum.SCALAR.value) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=tag, plugin_name=PluginNameEnum.SCALAR.value, value=value.scalar_value, filename=self._latest_filename) self._events_data.add_tensor_event(tensor_event) if value.HasField('image'): tag = '{}/{}'.format(value.tag, PluginNameEnum.IMAGE.value) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=tag, plugin_name=PluginNameEnum.IMAGE.value, value=value.image, filename=self._latest_filename) self._events_data.add_tensor_event(tensor_event) if value.HasField('histogram'): histogram_msg = HistogramContainer(value.histogram) # Drop steps if original_buckets_count exceeds HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT # to avoid time-consuming re-sample process. if histogram_msg.original_buckets_count > HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT: logger.warning('original_buckets_count exceeds HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT') else: tag = '{}/{}'.format(value.tag, PluginNameEnum.HISTOGRAM.value) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=tag, plugin_name=PluginNameEnum.HISTOGRAM.value, value=histogram_msg, filename=self._latest_filename) self._events_data.add_tensor_event(tensor_event) if event.HasField('graph_def'): graph_proto = event.graph_def graph = MSGraph() graph.build_graph(graph_proto) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=self._latest_filename, plugin_name=PluginNameEnum.GRAPH.value, value=graph, filename=self._latest_filename) try: graph_tags = self._events_data.list_tags_by_plugin(PluginNameEnum.GRAPH.value) except KeyError: graph_tags = [] summary_tags = self.filter_files(graph_tags) for tag in summary_tags: self._events_data.delete_tensor_event(tag) self._events_data.add_tensor_event(tensor_event)
def _event_parse(self, event): """ Transform `Event` data to tensor_event and update it to EventsData. Args: event (Event): Message event in summary proto, data read from file handler. """ plugins = { 'scalar_value': PluginNameEnum.SCALAR, 'image': PluginNameEnum.IMAGE, 'histogram': PluginNameEnum.HISTOGRAM, } if event.HasField('summary'): for value in event.summary.value: for plugin in plugins: if not value.HasField(plugin): continue plugin_name_enum = plugins[plugin] tensor_event_value = getattr(value, plugin) if plugin == 'histogram': tensor_event_value = HistogramContainer( tensor_event_value) # Drop steps if original_buckets_count exceeds HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT # to avoid time-consuming re-sample process. if tensor_event_value.original_buckets_count > HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT: logger.warning( 'original_buckets_count exceeds ' 'HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT' ) continue tensor_event = TensorEvent( wall_time=event.wall_time, step=event.step, tag='{}/{}'.format(value.tag, plugin_name_enum.value), plugin_name=plugin_name_enum.value, value=tensor_event_value, filename=self._latest_filename) self._events_data.add_tensor_event(tensor_event) elif event.HasField('graph_def'): graph = MSGraph() graph.build_graph(event.graph_def) tensor_event = TensorEvent(wall_time=event.wall_time, step=event.step, tag=self._latest_filename, plugin_name=PluginNameEnum.GRAPH.value, value=graph, filename=self._latest_filename) try: graph_tags = self._events_data.list_tags_by_plugin( PluginNameEnum.GRAPH.value) except KeyError: graph_tags = [] summary_tags = self.filter_files(graph_tags) for tag in summary_tags: self._events_data.delete_tensor_event(tag) self._events_data.add_tensor_event(tensor_event)
def test_add_tensor_event_out_of_order(self): """Test add_tensor_event success for out_of_order summaries.""" wall_time = 1 value = '1' tag = 'tag' plugin_name = 'scalar' file1 = 'file1' ev_data = EventsData() steps = [i for i in range(2, 10)] for step in steps: t_event = TensorEvent(wall_time=1, step=step, tag=tag, plugin_name=plugin_name, value=value, filename=file1) ev_data.add_tensor_event(t_event) t_event = TensorEvent(wall_time=1, step=1, tag=tag, plugin_name=plugin_name, value=value, filename=file1) ev_data.add_tensor_event(t_event) # Current steps should be: [1, 2, 3, 4, 5, 6, 7, 8, 9] assert len(ev_data._reservoir_by_tag[tag].samples()) == len(steps) + 1 file2 = 'file2' new_steps_1 = [5, 10] for step in new_steps_1: t_event = TensorEvent(wall_time=1, step=step, tag=tag, plugin_name=plugin_name, value=value, filename=file2) ev_data.add_tensor_event(t_event) assert ev_data._reservoir_by_tag[tag].samples()[-1] == _Tensor( wall_time, step, value, file2) # Current steps should be: [1, 2, 3, 4, 5, 10] steps = [1, 2, 3, 4, 5, 10] samples = ev_data._reservoir_by_tag[tag].samples() for step, sample in zip(steps, samples): filename = file1 if sample.step < 5 else file2 assert sample == _Tensor(wall_time, step, value, filename) new_steps_2 = [7, 11, 3] for step in new_steps_2: t_event = TensorEvent(wall_time=1, step=step, tag=tag, plugin_name=plugin_name, value=value, filename=file2) ev_data.add_tensor_event(t_event) # Current steps should be: [1, 2, 3, 5, 7, 10, 11], file2: [3, 5, 7, 10, 11] steps = [1, 2, 3, 5, 7, 10, 11] new_steps_2.extend(new_steps_1) samples = ev_data._reservoir_by_tag[tag].samples() for step, sample in zip(steps, samples): filename = file2 if sample.step in new_steps_2 else file1 assert sample == _Tensor(wall_time, step, value, filename)