def on_epoch_end(self, data: Data) -> None: if self.system.mode == 'train' and self.histogram_freq.freq and not self.histogram_freq.is_step and \ self.system.epoch_idx % self.histogram_freq.freq == 0: self.writer.write_weights(mode=self.system.mode, models=self.system.network.models, step=self.system.global_step, visualize=self.paint_weights) if self.update_freq.freq and (self.update_freq.is_step or self.system.epoch_idx % self.update_freq.freq == 0): self.writer.write_scalars(mode=self.system.mode, step=self.system.global_step, scalars=filter(lambda x: is_number(x[1]), data.items())) self.writer.write_images(mode=self.system.mode, step=self.system.global_step, images=filter( lambda x: x[1] is not None, map(lambda y: (y, data.get(y)), self.write_images))) self.writer.write_embeddings( mode=self.system.mode, step=self.system.global_step, embeddings=filter( lambda x: x[1] is not None, map( lambda t: (t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])), self.write_embeddings)))
def on_batch_end(self, data: Data) -> None: if self.write_graph and self.system.network.epoch_models.symmetric_difference( self.painted_graphs): self.writer.write_epoch_models(mode=self.system.mode) self.painted_graphs = self.system.network.epoch_models if self.system.mode != 'train': return if self.histogram_freq.freq and self.histogram_freq.is_step and \ self.system.global_step % self.histogram_freq.freq == 0: self.writer.write_weights(mode=self.system.mode, models=self.system.network.models, step=self.system.global_step, visualize=self.paint_weights) if self.update_freq.freq and self.update_freq.is_step and self.system.global_step % self.update_freq.freq == 0: self.writer.write_scalars(mode=self.system.mode, step=self.system.global_step, scalars=filter(lambda x: is_number(x[1]), data.items())) self.writer.write_images(mode=self.system.mode, step=self.system.global_step, images=filter( lambda x: x[1] is not None, map(lambda y: (y, data.get(y)), self.write_images))) self.writer.write_embeddings( mode=self.system.mode, step=self.system.global_step, embeddings=filter( lambda x: x[1] is not None, map( lambda t: (t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])), self.write_embeddings)))
def on_epoch_end(self, data: Data) -> None: if self.system.mode == 'train' and self.histogram_freq.freq and not self.histogram_freq.is_step and \ self.system.epoch_idx % self.histogram_freq.freq == 0: self.writer.write_weights(mode=self.system.mode, models=self.system.network.models, step=self.system.global_step, visualize=self.paint_weights) # Write out any embeddings which were aggregated over batches for name, val_list in self.collected_embeddings.items(): embeddings = None if any( x[0] is None for x in val_list) else concat([x[0] for x in val_list]) labels = None if any( x[1] is None for x in val_list) else concat([x[1] for x in val_list]) imgs = None if any( x[2] is None for x in val_list) else concat([x[2] for x in val_list]) self.writer.write_embeddings(mode=self.system.mode, step=self.system.global_step, embeddings=[(name, embeddings, labels, imgs)]) self.collected_embeddings.clear() # Get any embeddings which were generated externally on epoch end if self.embedding_freq.freq and (self.embedding_freq.is_step or self.system.epoch_idx % self.embedding_freq.freq == 0): self.writer.write_embeddings( mode=self.system.mode, step=self.system.global_step, embeddings=filter( lambda x: x[1] is not None, map( lambda t: (t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])), self.write_embeddings))) if self.update_freq.freq and (self.update_freq.is_step or self.system.epoch_idx % self.update_freq.freq == 0): self.writer.write_scalars(mode=self.system.mode, step=self.system.global_step, scalars=filter(lambda x: is_number(x[1]), data.items())) self.writer.write_images(mode=self.system.mode, step=self.system.global_step, images=filter( lambda x: x[1] is not None, map(lambda y: (y, data.get(y)), self.write_images)))
def on_batch_end(self, data: Data) -> None: if self.write_graph and self.system.network.epoch_models.symmetric_difference( self.painted_graphs): self.writer.write_epoch_models(mode=self.system.mode, epoch=self.system.epoch_idx) self.painted_graphs = self.system.network.epoch_models # Collect embeddings if present in batch but viewing per epoch. Don't aggregate during training though if self.system.mode != 'train' and self.embedding_freq.freq and not self.embedding_freq.is_step and self.system.epoch_idx % self.embedding_freq.freq == 0: for elem in self.write_embeddings: name, lbl, img = elem if name in data: self.collected_embeddings[name].append( (data.get(name), data.get(lbl), data.get(img))) # Handle embeddings if viewing per step if self.embedding_freq.freq and self.embedding_freq.is_step and self.system.global_step % self.embedding_freq.freq == 0: self.writer.write_embeddings( mode=self.system.mode, step=self.system.global_step, embeddings=filter( lambda x: x[1] is not None, map( lambda t: (t[0], data.get(t[0]), data.get(t[1]), data.get(t[2])), self.write_embeddings))) if self.system.mode != 'train': return if self.histogram_freq.freq and self.histogram_freq.is_step and \ self.system.global_step % self.histogram_freq.freq == 0: self.writer.write_weights(mode=self.system.mode, models=self.system.network.models, step=self.system.global_step, visualize=self.paint_weights) if self.update_freq.freq and self.update_freq.is_step and self.system.global_step % self.update_freq.freq == 0: self.writer.write_scalars(mode=self.system.mode, step=self.system.global_step, scalars=filter(lambda x: is_number(x[1]), data.items())) self.writer.write_images(mode=self.system.mode, step=self.system.global_step, images=filter( lambda x: x[1] is not None, map(lambda y: (y, data.get(y)), self.write_images)))