class CometLogger(): def __init__(self): global comet_installed self._logging = False if comet_installed: try: self._experiment = Experiment(auto_metric_logging=False, display_summary_level=0) self._experiment.log_other("Created from", "sweetviz!") self._logging = True except: print( "ERROR: comet_ml is installed, but not configured properly (e.g. check API key setup). HTML reports will not be uploaded." ) def log_html(self, html_content): if self._logging: try: self._experiment.log_html(html_content) except: print( "comet_ml.log_html(): error occurred during call to log_html()." ) else: print( "comet_ml.log_html(): comet_ml is not installed or otherwise ready for logging." ) def end(self): if self._logging: try: self._experiment.end() except: print("comet_ml.end(): error occurred during call to end().") else: print( "comet_ml.end(): comet_ml is not installed or otherwise ready." )
with open(os.path.join(DATADIR, "level1taxon_labels_index.json"), 'r') as f: labels_index = json.load(f, object_hook=lambda d: {int(k): [int(i) for i in v] if isinstance(v, list) else v for k, v in d.items()}) print('creating plotting_metrics dataframe') dev_f1s = pd.Series(f1_score(y_dev, y_pred_dev, average=None, sample_weight=None)) plotting_metrics = pd.concat([pd.concat([pd.DataFrame(np.sum(y_train, axis=0)), pd.DataFrame(np.sum(y_dev, axis=0))]).transpose(), taxon_codes, dev_f1s], axis=1) plotting_metrics.columns = ['train_support', 'dev_support', 'taxon_code', 'dev_f1'] plotting_metrics['taxon_label'] = plotting_metrics['taxon_code'].map(labels_index) plotting_metrics = plotting_metrics.sort_values('dev_f1', ascending=False) print('logging plotting_metrics dataframe to html table') experiment.log_html(plotting_metrics.to_html()) # dev_f1 = plotting_metrics.sort_values('dev_f1', ascending=False).plot(x='taxon_label', # y='dev_f1', # kind = 'barh', # figsize=(10,30), # legend=False, # title='F1 score by taxon', # color='#2B8CC4') # dev_f1, ax = plt.subplots(figsize=(10, 30)) # ax.barh(plotting_metrics['taxon_label'].values, plotting_metrics['dev_f1'].values, color='#2B8CC4') # experiment.log_figure(figure_name='dev_f1_scores', figure=dev_f1) # plt.close() # train_support = plotting_metrics.sort_values('dev_f1', ascending=False).plot(x='taxon_label', # y='train_support',
matrix=confmat_mean, labels=["Cannot", "Must", "May"], row_label="Predicted", column_label="Ground truth", ) exp.log_confusion_matrix( file_name="confusion_matrix_std.json", title="confusion_matrix_std.json", matrix=confmat_std, labels=["Cannot", "Must", "May"], row_label="Predicted", column_label="Ground truth", ) exp.log_metrics(dict(means)) exp.log_table("metrics.csv", df) exp.log_html(df.to_html(col_space="80px")) exp.log_parameters(vars(args)) exp.log_parameter("eval_path", str(eval_path)) exp.add_tag("eval_masker") if args.tags: exp.add_tags(args.tags) exp.log_parameter("model_id", Path(eval_path).name) # Close comet exp.end() # -------------------------------- # ----- END OF MODElS LOOP ----- # -------------------------------- # Compare models
class Logger: def __init__(self, args, path='logs/', name=''): print("logging experiment " + name) self.log_tb = args.log_tb self.log_comet = args.log_comet # Setup Comet if self.log_comet: if not os.path.isfile("settings.json"): sys.exit("You need a 'settings.json' file to use Comet") with open('settings.json') as f: data = json.load(f) api_key = data["comet"]["api_key"] project_name = data["comet"]["project_name"] workspace = data["comet"]["workspace"] self.comet = Experiment(api_key=api_key, project_name=project_name, workspace=workspace) self.comet.set_name(args.namestr) # Setup tensorboad if args.log_tb: # Prep for local logging if not os.path.isdir(path): os.mkdir(path) log_file = os.path.join(path, name) # Check if exists if os.path.isdir(log_file): ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d_%H:%M:%S') name += "_" + st log_file = os.path.join(path, name) self.tblogger = SummaryWriter(log_dir=log_file) self.log_text('logfile location', str(log_file)) # COMET has "notes/markdown" option on website, but can't figure out which call to use # Initial log information self.log_text('comment', args.comment) self.log_text('run command', " ".join(sys.argv)) flags = pprint.pformat(vars(args)) self.log_text('Flags', flags) self.log_text('chkpt', 'chkpt/model_path') self.log_time('start time', 0) def log_text(self, tag, msg): if self.log_comet: self.comet.log_html("<p>{}: {}</p>".format(tag, msg)) if self.log_tb: self.tblogger.add_text(tag, msg, global_step=0) def log_time(self, msg, step): """ Log time only for tensorboard, Comet does it auto """ if self.log_tb: cur = datetime.datetime.now() msg = msg + " " + str(cur) self.tblogger.add_text('time', msg, step) def log(self, tag, scalar, step): if self.log_comet: self.comet.log_metric(tag, scalar, step=step) if self.log_tb: self.tblogger.add_scalar(tag, scalar, step) def log_image(self, tag, image, epoch): """ Save image to tensorboard. Tb expects image with dimensions: CHW, so must reshuffle current format of HWC """ if self.log_tb: image = image.transpose(2, 0, 1) self.tblogger.add_image(tag, image, epoch) if self.log_comet: #TODO: not sure about comet. Looks like need to save file first # then call experiment.log_image(file_path) pass def log_histogram(self, tag, param, epoch): if self.log_tb: self.tblogger.add_histogram(tag, param, epoch)