def _get_file_writer(self): """Returns the default FileWriter instance. Recreates it if closed.""" if self.rank > 0: return TTDummyFileWriter() if self.all_writers is None or self.file_writer is None: if self.purge_step is not None: most_recent_step = self.purge_step self.file_writer = FileWriter(self.log_dir, self.max_queue, self.flush_secs, self.filename_suffix) self.file_writer.debug = self.debug self.file_writer.rank = self.rank self.file_writer.add_event( Event(step=most_recent_step, file_version='brain.Event:2')) self.file_writer.add_event( Event(step=most_recent_step, session_log=SessionLog(status=SessionLog.START))) else: self.file_writer = FileWriter(self.log_dir, self.max_queue, self.flush_secs, self.filename_suffix) self.all_writers = { self.file_writer.get_logdir(): self.file_writer } return self.file_writer
def _get_file_writer(self): """Returns the default FileWriter instance. Recreates it if closed.""" if self.all_writers is None or self.file_writer is None: if 'purge_step' in self.kwargs.keys(): most_recent_step = self.kwargs.pop('purge_step') self.file_writer = FileWriter(logdir=self.log_dir, **self.kwargs) self.file_writer.add_event( Event(step=most_recent_step, file_version='brain.Event:2')) self.file_writer.add_event( Event(step=most_recent_step, session_log=SessionLog(status=SessionLog.START))) else: self.file_writer = FileWriter(logdir=self.log_dir, **self.kwargs) self.all_writers = {self.file_writer.get_logdir(): self.file_writer} return self.file_writer
def __iter__(self) -> Event: """ Iterates over events in the current events file :return: An Event object :except: NotImplementedError if the stream is in non-blocking mode. :except: EventReadingError on reading error. """ while True: header_size = struct.calcsize('Q') header = self._read_and_check(header_size) if header is None: break event_size = struct.unpack('Q', header)[0] event_raw = self._read_and_check(event_size) if event_raw is None: raise EventReadingError('Unexpected end of events file') event = Event() event.ParseFromString(event_raw) yield event
def event_reader(filepointer): while True: # So, first read a Q eightbytes = filepointer.read(8) if len(eightbytes) != 8: break header, = struct.unpack("Q", eightbytes) fourbytes = filepointer.read(4) purported_header_crc32, = struct.unpack("I", fourbytes) data = filepointer.read(header) fourbytes = filepointer.read(4) purported_data_crc32, = struct.unpack("I", fourbytes) empiric_header_crc32 = masked_crc32c(struct.pack("Q", header)) empiric_data_crc32 = masked_crc32c(data) assert empiric_header_crc32 == purported_header_crc32 assert empiric_data_crc32 == purported_data_crc32 evt = Event() evt.ParseFromString(data) yield evt
def event(step, values): s = Summary() scalar = [ Summary.Value(tag="{}/{}".format(name, field), simple_value=v) for name, value in zip(names, values) for field, v in value._asdict().items() ] hist = [ Summary.Value(tag="{}/inferred_normal_hist".format(name), histo=inferred_histo(value)) for name, value in zip(names, values) ] s.value.extend(scalar + hist) return Event(wall_time=int(step), step=step, summary=s)
def truncate(log_dir, trunc_anchor): log_dir = Path(log_dir) for event_fpath in log_dir.glob('events*'): with event_fpath.open('rb+') as f: trunc_length = 0 while True: length = f.read(8) if len(length) == 0: break length = struct.unpack('Q', length)[0] f.read(4) event = f.read(length) event = Event.FromString(event) f.read(4) trunc_length += 16 + length try: tag = event.summary.value[0].tag value = event.summary.value[0].simple_value if tag == 'hogwarts/progress' and value == trunc_anchor: break except Exception: pass f.seek(0) f.truncate(trunc_length)
def graph_def_to_event(step, graph_def): return Event(wall_time=step, step=step, graph_def=graph_def.SerializeToString())