class CometLogger():
def __init__(self):
global comet_installed
self._logging = False
if comet_installed:
try:
self._experiment = Experiment(auto_metric_logging=False,
display_summary_level=0)
self._experiment.log_other("Created from", "sweetviz!")
self._logging = True
except:
print(
"ERROR: comet_ml is installed, but not configured properly (e.g. check API key setup). HTML reports will not be uploaded."
)
def log_html(self, html_content):
if self._logging:
try:
self._experiment.log_html(html_content)
except:
print(
"comet_ml.log_html(): error occurred during call to log_html()."
)
else:
print(
"comet_ml.log_html(): comet_ml is not installed or otherwise ready for logging."
)
def end(self):
if self._logging:
try:
self._experiment.end()
except:
print("comet_ml.end(): error occurred during call to end().")
else:
print(
"comet_ml.end(): comet_ml is not installed or otherwise ready."
)
with open(os.path.join(DATADIR, "level1taxon_labels_index.json"), 'r') as f:
labels_index = json.load(f, object_hook=lambda d: {int(k): [int(i) for i in v] if isinstance(v, list) else v for k, v in d.items()})
print('creating plotting_metrics dataframe')
dev_f1s = pd.Series(f1_score(y_dev, y_pred_dev, average=None, sample_weight=None))
plotting_metrics = pd.concat([pd.concat([pd.DataFrame(np.sum(y_train, axis=0)),
pd.DataFrame(np.sum(y_dev, axis=0))]).transpose(),
taxon_codes,
dev_f1s], axis=1)
plotting_metrics.columns = ['train_support', 'dev_support', 'taxon_code', 'dev_f1']
plotting_metrics['taxon_label'] = plotting_metrics['taxon_code'].map(labels_index)
plotting_metrics = plotting_metrics.sort_values('dev_f1', ascending=False)
print('logging plotting_metrics dataframe to html table')
experiment.log_html(plotting_metrics.to_html())
# dev_f1 = plotting_metrics.sort_values('dev_f1', ascending=False).plot(x='taxon_label',
# y='dev_f1',
# kind = 'barh',
# figsize=(10,30),
# legend=False,
# title='F1 score by taxon',
# color='#2B8CC4')
# dev_f1, ax = plt.subplots(figsize=(10, 30))
# ax.barh(plotting_metrics['taxon_label'].values, plotting_metrics['dev_f1'].values, color='#2B8CC4')
# experiment.log_figure(figure_name='dev_f1_scores', figure=dev_f1)
# plt.close()
# train_support = plotting_metrics.sort_values('dev_f1', ascending=False).plot(x='taxon_label',
# y='train_support',
matrix=confmat_mean,
labels=["Cannot", "Must", "May"],
row_label="Predicted",
column_label="Ground truth",
)
exp.log_confusion_matrix(
file_name="confusion_matrix_std.json",
title="confusion_matrix_std.json",
matrix=confmat_std,
labels=["Cannot", "Must", "May"],
row_label="Predicted",
column_label="Ground truth",
)
exp.log_metrics(dict(means))
exp.log_table("metrics.csv", df)
exp.log_html(df.to_html(col_space="80px"))
exp.log_parameters(vars(args))
exp.log_parameter("eval_path", str(eval_path))
exp.add_tag("eval_masker")
if args.tags:
exp.add_tags(args.tags)
exp.log_parameter("model_id", Path(eval_path).name)
# Close comet
exp.end()
# --------------------------------
# ----- END OF MODElS LOOP -----
# --------------------------------
# Compare models
class Logger:
def __init__(self, args, path='logs/', name=''):
print("logging experiment " + name)
self.log_tb = args.log_tb
self.log_comet = args.log_comet
# Setup Comet
if self.log_comet:
if not os.path.isfile("settings.json"):
sys.exit("You need a 'settings.json' file to use Comet")
with open('settings.json') as f:
data = json.load(f)
api_key = data["comet"]["api_key"]
project_name = data["comet"]["project_name"]
workspace = data["comet"]["workspace"]
self.comet = Experiment(api_key=api_key,
project_name=project_name,
workspace=workspace)
self.comet.set_name(args.namestr)
# Setup tensorboad
if args.log_tb:
# Prep for local logging
if not os.path.isdir(path):
os.mkdir(path)
log_file = os.path.join(path, name)
# Check if exists
if os.path.isdir(log_file):
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime(
'%Y-%m-%d_%H:%M:%S')
name += "_" + st
log_file = os.path.join(path, name)
self.tblogger = SummaryWriter(log_dir=log_file)
self.log_text('logfile location', str(log_file))
# COMET has "notes/markdown" option on website, but can't figure out which call to use
# Initial log information
self.log_text('comment', args.comment)
self.log_text('run command', " ".join(sys.argv))
flags = pprint.pformat(vars(args))
self.log_text('Flags', flags)
self.log_text('chkpt', 'chkpt/model_path')
self.log_time('start time', 0)
def log_text(self, tag, msg):
if self.log_comet:
self.comet.log_html("<p>{}: {}</p>".format(tag, msg))
if self.log_tb:
self.tblogger.add_text(tag, msg, global_step=0)
def log_time(self, msg, step):
"""
Log time only for tensorboard, Comet does it auto
"""
if self.log_tb:
cur = datetime.datetime.now()
msg = msg + " " + str(cur)
self.tblogger.add_text('time', msg, step)
def log(self, tag, scalar, step):
if self.log_comet:
self.comet.log_metric(tag, scalar, step=step)
if self.log_tb:
self.tblogger.add_scalar(tag, scalar, step)
def log_image(self, tag, image, epoch):
"""
Save image to tensorboard. Tb expects image with dimensions: CHW, so
must reshuffle current format of HWC
"""
if self.log_tb:
image = image.transpose(2, 0, 1)
self.tblogger.add_image(tag, image, epoch)
if self.log_comet:
#TODO: not sure about comet. Looks like need to save file first
# then call experiment.log_image(file_path)
pass
def log_histogram(self, tag, param, epoch):
if self.log_tb:
self.tblogger.add_histogram(tag, param, epoch)
