def custom_exc(shell, etype, evalue, tb, tb_offset=None):
# still show the error within the notebook, don't just swallow it
shell.showtraceback((etype, evalue, tb), tb_offset=tb_offset)
# grab the traceback and make it into a list of strings
stb = itb.structured_traceback(etype, evalue, tb)
sstb = itb.stb2text(stb)
hostname = socket.gethostname()
sstb = f"An exception occurred in {hostname=}\n\n" + sstb
slack.notify(sstb)
),
)
axs[0].set_yscale("log")
axs[-1].set_yscale("log")
viz.mark_min_values(hist_display.axs_metrics_[0], hist_display.plots_["loss"][0])
viz.mark_min_values(hist_display.axs_metrics_[0], hist_display.plots_["val_loss"][0], txt_kwargs=dict(rotation=0))
hist_display.fig_.savefig(
t2s_model.model_path / (hist_display.title + ".png"),
format='png',
)
history_cb.dataframe.to_csv(history_cb.csv_path, index=True)
model.save(t2s_model.model_path)
# # End
slack.notify(f"script `{args.script_name}` on `{args.host.hostname}` finished!")
except Exception as ex:
slack.notify_exception(ex, args.host.hostname)
logger.info("rebuilding volume - end")
# # Save
logger.debug(
f"Writing probabilities on disk at `{estimation_volume.probabilities_path}`"
)
np.save(estimation_volume.probabilities_path, proba_volume)
if args.opts.save_probas_by_class:
for klass_idx in volume.metadata.labels:
logger.debug(
f"Writing probabilities of class `{klass_idx}` on disk at `{(str_path := str(estimation_volume.get_class_probability_path(klass_idx)))=}`"
)
file_utils.HST_write(proba_volume[:, :, :, klass_idx], str_path)
pred_volume.size * pred_volume.itemsize / 1024**2
logger.debug(
f"Writing predictions on disk at `{(str_path := str(estimation_volume.predictions_path))}`"
)
file_utils.HST_write(pred_volume, str_path)
# # Slack (:
slack.notify("process-volume finished!")
except Exception as ex:
slack.notify_exception(ex, args.host.hostname)