def _get_file_writer(self):
"""Returns the default FileWriter instance. Recreates it if closed."""
if self.rank > 0:
return TTDummyFileWriter()
if self.all_writers is None or self.file_writer is None:
if self.purge_step is not None:
most_recent_step = self.purge_step
self.file_writer = FileWriter(self.log_dir, self.max_queue,
self.flush_secs,
self.filename_suffix)
self.file_writer.debug = self.debug
self.file_writer.rank = self.rank
self.file_writer.add_event(
Event(step=most_recent_step, file_version='brain.Event:2'))
self.file_writer.add_event(
Event(step=most_recent_step,
session_log=SessionLog(status=SessionLog.START)))
else:
self.file_writer = FileWriter(self.log_dir, self.max_queue,
self.flush_secs,
self.filename_suffix)
self.all_writers = {
self.file_writer.get_logdir(): self.file_writer
}
return self.file_writer
def _get_file_writer(self):
"""Returns the default FileWriter instance. Recreates it if closed."""
if self.all_writers is None or self.file_writer is None:
if 'purge_step' in self.kwargs.keys():
most_recent_step = self.kwargs.pop('purge_step')
self.file_writer = FileWriter(logdir=self.log_dir, **self.kwargs)
self.file_writer.add_event(
Event(step=most_recent_step, file_version='brain.Event:2'))
self.file_writer.add_event(
Event(step=most_recent_step, session_log=SessionLog(status=SessionLog.START)))
else:
self.file_writer = FileWriter(logdir=self.log_dir, **self.kwargs)
self.all_writers = {self.file_writer.get_logdir(): self.file_writer}
return self.file_writer
def __iter__(self) -> Event:
"""
Iterates over events in the current events file
:return: An Event object
:except: NotImplementedError if the stream is in non-blocking mode.
:except: EventReadingError on reading error.
"""
while True:
header_size = struct.calcsize('Q')
header = self._read_and_check(header_size)
if header is None:
break
event_size = struct.unpack('Q', header)[0]
event_raw = self._read_and_check(event_size)
if event_raw is None:
raise EventReadingError('Unexpected end of events file')
event = Event()
event.ParseFromString(event_raw)
yield event
def event_reader(filepointer):
while True:
# So, first read a Q
eightbytes = filepointer.read(8)
if len(eightbytes) != 8:
break
header, = struct.unpack("Q", eightbytes)
fourbytes = filepointer.read(4)
purported_header_crc32, = struct.unpack("I", fourbytes)
data = filepointer.read(header)
fourbytes = filepointer.read(4)
purported_data_crc32, = struct.unpack("I", fourbytes)
empiric_header_crc32 = masked_crc32c(struct.pack("Q", header))
empiric_data_crc32 = masked_crc32c(data)
assert empiric_header_crc32 == purported_header_crc32
assert empiric_data_crc32 == purported_data_crc32
evt = Event()
evt.ParseFromString(data)
yield evt
def event(step, values):
s = Summary()
scalar = [
Summary.Value(tag="{}/{}".format(name, field), simple_value=v)
for name, value in zip(names, values)
for field, v in value._asdict().items()
]
hist = [
Summary.Value(tag="{}/inferred_normal_hist".format(name),
histo=inferred_histo(value))
for name, value in zip(names, values)
]
s.value.extend(scalar + hist)
return Event(wall_time=int(step), step=step, summary=s)
def truncate(log_dir, trunc_anchor):
log_dir = Path(log_dir)
for event_fpath in log_dir.glob('events*'):
with event_fpath.open('rb+') as f:
trunc_length = 0
while True:
length = f.read(8)
if len(length) == 0: break
length = struct.unpack('Q', length)[0]
f.read(4)
event = f.read(length)
event = Event.FromString(event)
f.read(4)
trunc_length += 16 + length
try:
tag = event.summary.value[0].tag
value = event.summary.value[0].simple_value
if tag == 'hogwarts/progress' and value == trunc_anchor:
break
except Exception:
pass
f.seek(0)
f.truncate(trunc_length)
def graph_def_to_event(step, graph_def):
return Event(wall_time=step,
step=step,
graph_def=graph_def.SerializeToString())