def on_epoch_end(self, data: Data) -> None:
if self.system.mode == 'train' and self.histogram_freq.freq and not self.histogram_freq.is_step and \
self.system.epoch_idx % self.histogram_freq.freq == 0:
self.writer.write_weights(mode=self.system.mode,
models=self.system.network.models,
step=self.system.global_step,
visualize=self.paint_weights)
if self.update_freq.freq and (self.update_freq.is_step
or self.system.epoch_idx %
self.update_freq.freq == 0):
self.writer.write_scalars(mode=self.system.mode,
step=self.system.global_step,
scalars=filter(lambda x: is_number(x[1]),
data.items()))
self.writer.write_images(mode=self.system.mode,
step=self.system.global_step,
images=filter(
lambda x: x[1] is not None,
map(lambda y: (y, data.get(y)),
self.write_images)))
self.writer.write_embeddings(
mode=self.system.mode,
step=self.system.global_step,
embeddings=filter(
lambda x: x[1] is not None,
map(
lambda t:
(t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])),
self.write_embeddings)))
def on_batch_end(self, data: Data) -> None:
if self.write_graph and self.system.network.epoch_models.symmetric_difference(
self.painted_graphs):
self.writer.write_epoch_models(mode=self.system.mode)
self.painted_graphs = self.system.network.epoch_models
if self.system.mode != 'train':
return
if self.histogram_freq.freq and self.histogram_freq.is_step and \
self.system.global_step % self.histogram_freq.freq == 0:
self.writer.write_weights(mode=self.system.mode,
models=self.system.network.models,
step=self.system.global_step,
visualize=self.paint_weights)
if self.update_freq.freq and self.update_freq.is_step and self.system.global_step % self.update_freq.freq == 0:
self.writer.write_scalars(mode=self.system.mode,
step=self.system.global_step,
scalars=filter(lambda x: is_number(x[1]),
data.items()))
self.writer.write_images(mode=self.system.mode,
step=self.system.global_step,
images=filter(
lambda x: x[1] is not None,
map(lambda y: (y, data.get(y)),
self.write_images)))
self.writer.write_embeddings(
mode=self.system.mode,
step=self.system.global_step,
embeddings=filter(
lambda x: x[1] is not None,
map(
lambda t:
(t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])),
self.write_embeddings)))
def on_epoch_end(self, data: Data) -> None:
if self.system.mode == 'train' and self.histogram_freq.freq and not self.histogram_freq.is_step and \
self.system.epoch_idx % self.histogram_freq.freq == 0:
self.writer.write_weights(mode=self.system.mode,
models=self.system.network.models,
step=self.system.global_step,
visualize=self.paint_weights)
# Write out any embeddings which were aggregated over batches
for name, val_list in self.collected_embeddings.items():
embeddings = None if any(
x[0] is None
for x in val_list) else concat([x[0] for x in val_list])
labels = None if any(
x[1] is None
for x in val_list) else concat([x[1] for x in val_list])
imgs = None if any(
x[2] is None
for x in val_list) else concat([x[2] for x in val_list])
self.writer.write_embeddings(mode=self.system.mode,
step=self.system.global_step,
embeddings=[(name, embeddings, labels,
imgs)])
self.collected_embeddings.clear()
# Get any embeddings which were generated externally on epoch end
if self.embedding_freq.freq and (self.embedding_freq.is_step
or self.system.epoch_idx %
self.embedding_freq.freq == 0):
self.writer.write_embeddings(
mode=self.system.mode,
step=self.system.global_step,
embeddings=filter(
lambda x: x[1] is not None,
map(
lambda t:
(t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])),
self.write_embeddings)))
if self.update_freq.freq and (self.update_freq.is_step
or self.system.epoch_idx %
self.update_freq.freq == 0):
self.writer.write_scalars(mode=self.system.mode,
step=self.system.global_step,
scalars=filter(lambda x: is_number(x[1]),
data.items()))
self.writer.write_images(mode=self.system.mode,
step=self.system.global_step,
images=filter(
lambda x: x[1] is not None,
map(lambda y: (y, data.get(y)),
self.write_images)))
def on_batch_end(self, data: Data) -> None:
if self.write_graph and self.system.network.epoch_models.symmetric_difference(
self.painted_graphs):
self.writer.write_epoch_models(mode=self.system.mode,
epoch=self.system.epoch_idx)
self.painted_graphs = self.system.network.epoch_models
# Collect embeddings if present in batch but viewing per epoch. Don't aggregate during training though
if self.system.mode != 'train' and self.embedding_freq.freq and not self.embedding_freq.is_step and self.system.epoch_idx % self.embedding_freq.freq == 0:
for elem in self.write_embeddings:
name, lbl, img = elem
if name in data:
self.collected_embeddings[name].append(
(data.get(name), data.get(lbl), data.get(img)))
# Handle embeddings if viewing per step
if self.embedding_freq.freq and self.embedding_freq.is_step and self.system.global_step % self.embedding_freq.freq == 0:
self.writer.write_embeddings(
mode=self.system.mode,
step=self.system.global_step,
embeddings=filter(
lambda x: x[1] is not None,
map(
lambda t:
(t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])),
self.write_embeddings)))
if self.system.mode != 'train':
return
if self.histogram_freq.freq and self.histogram_freq.is_step and \
self.system.global_step % self.histogram_freq.freq == 0:
self.writer.write_weights(mode=self.system.mode,
models=self.system.network.models,
step=self.system.global_step,
visualize=self.paint_weights)
if self.update_freq.freq and self.update_freq.is_step and self.system.global_step % self.update_freq.freq == 0:
self.writer.write_scalars(mode=self.system.mode,
step=self.system.global_step,
scalars=filter(lambda x: is_number(x[1]),
data.items()))
self.writer.write_images(mode=self.system.mode,
step=self.system.global_step,
images=filter(
lambda x: x[1] is not None,
map(lambda y: (y, data.get(y)),
self.write_images)))