def _parse_event_proto_str(proto_str): """ Converts the serialized Event protobufs from `tf.data.TFRecordDataset` into a `Message` object. """ event_msg = Event() event_msg.ParseFromString(proto_str) return event_msg
def _add_event(self, step, summary): t = time.time() e = Event(wall_time=t, step=step, summary=summary) self.queue.append(e) if t - self.last_append >= self.tick or len(self.queue) >= self.max_queue_len: self.flush() self.last_append = t
def rename_events(input_path, output_path, old_tags, new_tag): # Make a record writer with tf.io.TFRecordWriter(str(output_path)) as writer: # Iterate event records for rec in tf.data.TFRecordDataset([str(input_path)]): # Read event ev = Event() ev.MergeFromString(rec.numpy()) # Check if it is a summary if ev.summary: # Iterate summary values for v in ev.summary.value: # Check if the tag should be renamed if v.tag in old_tags: # Rename with new tag name v.tag = new_tag writer.write(ev.SerializeToString())
def write_summary(dpath, aggregations_per_key): writer = tf.summary.FileWriter(dpath) for key, (steps, wall_times, aggregations) in aggregations_per_key.items(): for step, wall_time, aggregation in zip(steps, wall_times, aggregations): summary = tf.Summary(value=[tf.Summary.Value(tag=key, simple_value=aggregation)]) scalar_event = Event(wall_time=wall_time, step=step, summary=summary) writer.add_event(scalar_event) writer.flush()
def iter_log_events(tf_event_file): import tensorflow as tf from tensorflow.core.util.event_pb2 import Event for event in tf.data.TFRecordDataset([tf_event_file]): event = Event.FromString(event.numpy()) if event.summary.value: assert len(event.summary.value ) == 1, "Unexpected length for event summary" value = event.summary.value[0] yield value.tag, value.simple_value
def remove_events(input_file, groups_to_remove): new_file = input_file + ".new" # Make a record writer with tf.io.TFRecordWriter(new_file) as writer: # Iterate event records for rec in tf.data.TFRecordDataset([input_file]): # Read event ev = Event() ev.MergeFromString(rec.numpy()) # Check if it is a summary event if ev.summary: orig_values = [v for v in ev.summary.value] filtered_values = [v for v in orig_values if not is_tag_matching_group(v.tag, groups_to_remove)] #print(f"filtered_values={len(filtered_values)}, orig_values={len(orig_values)}") if len(filtered_values) != len(orig_values): # for v in orig_values: # print(v) del ev.summary.value[:] ev.summary.value.extend(filtered_values) writer.write(ev.SerializeToString()) os.rename(new_file, input_file)
def rename_events(input_file, old_tags, new_tag): new_file = input_file + ".new" # Make a record writer with tf.io.TFRecordWriter(new_file) as writer: # Iterate event records for rec in tf.data.TFRecordDataset([input_file]): # Read event ev = Event() ev.MergeFromString(rec.numpy()) # Check if it is a summary #print(ev) if ev.summary: # Iterate summary values for v in ev.summary.value: #print(v) # Check if the tag should be renamed if v.tag in old_tags: # Rename with new tag name v.tag = new_tag writer.write(ev.SerializeToString()) os.rename(new_file, input_file)
def write_summary(dpath, extracts): dpath = dpath / FOLDER_NAME / dpath.name writer = tf.summary.FileWriter(dpath) for key, (steps, wall_times, values) in extracts.items(): for episode, step, wall_time, value in zip(steps, wall_times, values): summary = tf.Summary( value=[tf.Summary.Value(tag=key, simple_value=value)]) scalar_event = Event(episode=episode, wall_time=wall_time, step=step, summary=summary) writer.add_event(scalar_event) writer.flush()
def write_summary(dpath, dname, fname, aggregations_per_key, steps, wall_times): fpath = os.path.join(dpath, dname) fpath = os.path.abspath(fpath) writer = tf.summary.FileWriter(fpath) for key, aggregations in aggregations_per_key.items(): for step, wall_time, aggregation in zip(steps, wall_times, aggregations): summary = tf.Summary( value=[tf.Summary.Value(tag=key, simple_value=aggregation)]) scalar_event = Event(wall_time=wall_time, step=step, summary=summary) writer.add_event(scalar_event) writer.flush()
def read_events(stream): ''' Read and return as a generator a sequence of Event protos from file-like object `stream`. ''' header_size = struct.calcsize('<QI') len_size = struct.calcsize('<Q') footer_size = struct.calcsize('<I') while True: header = stream.read(header_size) if len(header) == 0: break elif len(header) < header_size: raise SummaryReaderException( 'unexpected EOF (expected a %d-byte header, ' 'got %d bytes)' % (header_size, len(header))) data_len, len_crc = struct.unpack('<QI', header) len_crc_actual = masked_crc(header[:len_size]) if len_crc_actual != len_crc: raise SummaryReaderException('incorrect length CRC (%d != %d)' % (len_crc_actual, len_crc)) data = stream.read(data_len) if len(data) < data_len: raise SummaryReaderException( 'unexpected EOF (expected %d bytes, got %d)' % (data_len, len(data))) yield Event.FromString(data) footer = stream.read(footer_size) if len(footer) < footer_size: raise SummaryReaderException( 'unexpected EOF (expected a %d-byte footer, ' 'got %d bytes)' % (footer_size, len(footer))) data_crc, = struct.unpack('<I', footer) data_crc_actual = masked_crc(data) if data_crc_actual != data_crc: raise SummaryReaderException('incorrect data CRC (%d != %d)' % (data_crc_actual, data_crc))
def _add_event(self, event: event_pb2.Event, step: Optional[int]) -> None: # _add_event is vendored from TensorFlow: tensorflow/python/summary/writer/writer.py event.wall_time = time.time() if step is not None: event.step = int(step) self.writer.add_event(event)
def _write_event(self, summary_value, step): self.writer.add_event( Event(wall_time=round(time.time()), step=step, summary=Summary(value=[summary_value])))