class CometWriter(LoggerWriter):
def __init__(self, project_name, enable_log, cfg):
from comet_ml import Experiment
super().__init__(project_name, enable_log, cfg)
self.exp = Experiment(api_key=cfg.comet_api_key,
project_name=project_name,
disabled=not enable_log)
def log_params(self, args):
self.exp.log_parameters(vars(args))
def log_metric(self, name, value, step):
self.exp.log_metric(name, value, step)
def log_confusion_matrix(self, matrix, step):
self.exp.log_confusion_matrix(
matrix=matrix,
step=step,
file_name="confusion_matrix_{}".format(step))
def eval(
model: LiSSModel,
dataset: UnalignedDataset,
exp: Experiment,
total_iters: int = 0,
nb_ims: int = 30,
):
liss = model.opt.model == "liss"
metrics = {}
print(f"----------- Evaluation {total_iters} ----------")
with torch.no_grad():
data = {
"translation": {
"A": {
"rec": None,
"idt": None,
"real": None,
"fake": None
},
"B": {
"rec": None,
"idt": None,
"real": None,
"fake": None
},
}
}
force = set(["identity", "translation"])
if liss:
for t in model.tasks:
tmp = {}
if t.eval_visuals_pred or t.log_type == "acc":
tmp["pred"] = None
if t.eval_visuals_target or t.log_type == "acc":
tmp["target"] = None
data[t.key] = {domain: deepcopy(tmp) for domain in "AB"}
force |= set(model.tasks.keys)
losses = {
k: []
for k in dir(model) if k.startswith("loss_")
and isinstance(getattr(model, k), torch.Tensor)
}
for i, b in enumerate(dataset):
# print(f"\rEval batch {i}", end="")
model.set_input(b)
model.forward(force=force)
model.backward_G(losses_only=True, force=force)
for k in dir(model):
if k.startswith("loss_") and isinstance(
getattr(model, k), torch.Tensor):
if k not in losses:
losses[k] = []
losses[k].append(getattr(model, k).detach().cpu().item())
if liss:
for t in model.tasks:
for domain in "AB":
for dtype in data[t.key][domain]:
if (t.log_type != "acc"
and data[t.key][domain][dtype] is not None
and
len(data[t.key][domain][dtype]) >= nb_ims):
continue
v = model.get(
f"{domain}_{t.key}_{dtype}").detach().cpu()
if data[t.key][domain][dtype] is None:
data[t.key][domain][dtype] = v
else:
data[t.key][domain][dtype] = torch.cat(
[data[t.key][domain][dtype], v],
dim=0,
)
# -------------------------
# ----- Translation -----
# -------------------------
if (data["translation"]["A"]["real"] is None
or len(data["translation"]["A"]["real"]) < nb_ims):
for domain in "AB":
for dtype in ["real", "fake", "rec", "idt"]:
dom = domain
if dtype in {"fake", "idt"}:
dom = swap_domain(domain)
v = model.get(f"{dom}_{dtype}").detach().cpu()
if data["translation"][domain][dtype] is None:
data["translation"][domain][dtype] = v
else:
data["translation"][domain][dtype] = torch.cat(
[data["translation"][domain][dtype], v], dim=0)
# print(
# f"{domain} {dtype} {len(data['translation'][domain][dtype])}"
# )
for task in data:
if task != "translation" and model.tasks[task].log_type != "vis":
continue
for domain in data[task]:
for i, v in data[task][domain].items():
data[task][domain][i] = torch.cat(list(v[:nb_ims].permute(
0, 2, 3, 1)),
axis=1)
log_images = int(data["translation"]["A"]["real"].shape[1] /
data["translation"]["A"]["real"].shape[0])
im_size = data["translation"]["A"]["real"].shape[0]
ims = {"A": None, "B": None}
data_keys = ["translation"]
translation_keys = ["real", "fake", "rec", "idt"]
data_keys += [task for task in data if task not in data_keys]
for task in data_keys:
if task != "translation" and model.tasks[task].log_type != "vis":
continue
for domain in "AB":
im_types = (translation_keys if task == "translation" else list(
data[task][domain].keys()))
for im_type in im_types:
v = data[task][domain][im_type].float()
if task == "depth":
v = to_min1_1(v)
v = v.repeat((1, 1, 3))
v = v + 1
v = v / 2
if ims[domain] is None:
ims[domain] = v
else:
ims[domain] = torch.cat([ims[domain], v], dim=0)
# ------------------------
# ----- Comet Logs -----
# ------------------------
for i in range(0, log_images, 5):
k = i + 5
exp.log_image(
ims["A"][:, i * im_size:k * im_size, :].numpy(),
"test_A_{}_{}_rfcidg".format(i * 5, (i + 1) * 5 - 1),
step=total_iters,
)
exp.log_image(
ims["B"][:, i * im_size:k * im_size, :].numpy(),
"test_B_{}_{}_rfcidg".format(i * 5, (i + 1) * 5 - 1),
step=total_iters,
)
if liss:
test_losses = {
"test_" + ln: np.mean(losses["loss_" + ln])
for t in model.tasks for ln in t.loss_names
}
test_accs = {
f"test_G_{domain}_{t.key}_acc":
np.mean(data[t.key][domain]["pred"].max(-1)[1].numpy() == data[
t.key][domain]["target"].numpy())
for domain in "AB" for t in model.tasks if t.log_type == "acc"
}
if liss:
exp.log_metrics(test_losses, step=total_iters)
exp.log_metrics(test_accs, step=total_iters)
for t in model.tasks:
if t.log_type != "acc":
continue
for domain in "AB":
target = data[t.key][domain]["target"].numpy()
pred = data[t.key][domain]["pred"].numpy()
exp.log_confusion_matrix(
get_one_hot(target, t.output_dim),
pred,
file_name=
f"confusion_{domain}_{t.key}_{total_iters}.json",
title=f"confusion_{domain}_{t.key}_{total_iters}.json",
)
metrics = {k + "_loss": v for k, v in test_losses.items()}
metrics.update(test_accs)
print("----------- End Evaluation----------")
return metrics
# Log accuracy to Comet.ml
experiment.log_metric("accuracy", correct / total, step=step)
step += 1
if (i + 1) % 100 == 0:
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f' %
(epoch + 1, hyper_params['num_epochs'], i + 1,
len(train_dataset) // hyper_params['batch_size'],
loss.data.item()))
# At end of epoch:
print("Computing confusion matrix and uploading samples...")
confusion_matrix.compute_matrix(epoch_targets, epoch_predictions)
experiment.log_confusion_matrix(
matrix=confusion_matrix,
title="Train Confusion Matrix, Epoch #%s" % (epoch + 1, ),
file_name="train-confusion-matrix-%03d.json" % (epoch + 1),
)
print("Logging weights as histogram...")
# Log model weights
weights = []
for name in rnn.named_parameters():
if 'weight' in name[0]:
weights.extend(name[1].detach().numpy().tolist())
experiment.log_histogram_3d(weights, step=epoch + 1)
with experiment.test():
# Test the Model
correct = 0
total = 0
experiment.log_metrics(metrics)
# Optionally save per-label recall, precision and F1
# label_metrics = pd.DataFrame.from_dict(cls_report_dct, orient='index')
# label_metrics_file = os.path.join(output_dir, experiment_name + "_label_metrics.csv")
# label_metrics.to_csv(label_metrics_file)
# experiment.log_asset(label_metrics_file)
conf_matrix_fig, confusion_matrix = plot_confusion_matrix(
true_labels, predicted_labels, labels)
confusion_matrix_file = os.path.join(
output_dir, experiment_name + "_confusion_matrix.png")
conf_matrix_fig.savefig(confusion_matrix_file)
experiment.log_image(confusion_matrix_file)
experiment.log_confusion_matrix(
matrix=confusion_matrix.tolist(),
labels=labels[:len(confusion_matrix)])
end_time = time.time()
print("Testing took " + str(('%.3f' % (end_time - start_time))) +
" seconds for " + str(test_steps) + " steps")
# Close the current session
sess.close()
tf.keras.backend.clear_session()
# Log results to csv
if training and testing:
experiment_file = os.path.join(
task_name, task_name + "_" + experiment_type + ".csv")
save_experiment(experiment_file, experiment_params,
# Comet-ML log data
for i in history.epoch:
experiment.log_metric("accuracy",
history.history['accuracy'][i],
epoch=i + 1)
experiment.log_metric("loss", history.history['loss'][i], epoch=i + 1)
## Confusion matrix
y_test_hat = model.predict(X_test, batch_size=4)
y_test_hat = np.argmax(y_test_hat, axis=1)
y_test = np.argmax(y_test, axis=1)
conf_m = confusion_matrix(y_test, y_test_hat)
# Comet-ML log data
experiment.log_confusion_matrix(y_test, y_test_hat)
# Plot matrix
plt.figure(figsize=(5, 3))
sb.set(font_scale=1.2)
ax = sb.heatmap(conf_m,
annot=True,
xticklabels=['N', 'P'],
yticklabels=['N', 'P'],
cbar=False,
cmap='Blues',
linewidths=1,
linecolor='black',
fmt='.0f')
plt.yticks(rotation=0)
plt.xlabel('Predicted labels')
model.compile(loss='categorical_crossentropy', optimizer=_OPTIMIZER)
model.fit(oDataSet.attributes[oData.Training_indexes],
binarizer(oDataSet.labels[oData.Training_indexes]),
batch_size=1,
epochs=epochs,
verbose=1)
y_pred = model.predict(
oDataSet.attributes[oData.Testing_indexes]).argmax(axis=1)
y_true = oDataSet.labels[oData.Testing_indexes]
experiment.log_metric("test_accuracy", accuracy_score(y_true, y_pred))
experiment.log_metric("beta", best_b)
experiment.log_metric("neurons", best_p)
experiment.log_confusion_matrix(matrix=confusion_matrix(y_true,
y_pred).tolist(),
labels=oDataSet.labelsNames)
# model.save('model.h5')
# experiment.log_asset("model.h5")
model.save_weights('model.weights')
experiment.log_asset("model.weights")
print(accuracy_score(y_true, y_pred))
print(confusion_matrix(y_true, y_pred))
oData.confusion_matrix = confusion_matrix(y_true, y_pred)
oData.model = model
oData.params = {
"k_fold": K_FOLD,
"GRID_RESULT": grid_result,
"GRID_VALUES_NEURON": GRID_NEURON,
"GRID_VALUES_BETA": GRID_B,
ap, f1_max, precision, recall, f1_max_th, fig_pre_rec, fig_th_pre_rec = precision_recall(y_test,
score_test,
limit=1000,
label_anomaly=labels[0])
#fig_score_train = plot_score(score_train, 'train')
#fig_score_val = plot_score(score_val, 'validation')
fig_score_test = plot_score(score_test, 'test', 10, labels=y_test, th=f1_max_th)
fig_cumsum = pca.plot_cumsum()
y_pred, conf_matrix = predict(score_test, f1_max_th, y_test, labels)
experiment.add_tags([data, metric])
parameters = {'var': var, 'pc': pca.pcs_, 'metric': metric}
experiment.log_parameters(parameters)
experiment.log_metric('ap', ap)
experiment.log_metric('f1', f1_max)
experiment.log_metric('precision', precision)
experiment.log_metric('recall', recall)
experiment.log_metric('train_time', pca.time_)
experiment.log_parameter('th_f1', f1_max_th)
experiment.log_figure('cumsum', fig_cumsum)
experiment.log_figure('score_test',fig_score_test)
experiment.log_figure('precision_recall',fig_pre_rec)
experiment.log_figure('th_pre_rec_f1', fig_th_pre_rec)
experiment.log_confusion_matrix(matrix=conf_matrix, labels=labels)
experiment.end()
def train_and_evaluate(self, train_gen, val_gen, epochs):
"""
"""
experiment = Experiment(
api_key="VNQSdbR1pw33EkuHbUsGUSZWr",
project_name="piratesofthecaribbean",
workspace="florpi",
)
model = self.build()
with experiment.train():
model_path = os.path.join(self.directory,
"cnn_{epoch:02d}-{val_loss:.2f}.hdf5")
callbacks = [
ModelCheckpoint(model_path, monitor="val_loss", mode="min"),
# EarlyStopping(
# monitor="val_loss",
# mode="min",
# min_delta=0.1,
# patience=1,
# restore_best_weights=True,
# ),
]
model.fit(
train_gen,
epochs=epochs,
validation_data=val_gen,
callbacks=callbacks,
class_weight=CLASS_WEIGHTS,
)
model.save(os.path.join(self.directory, "cnn_final.h5"))
# Run validation
with experiment.test():
probabilities = []
y_val_all = []
# reset generator
val_gen.reset()
for idx, (X_val, y_val) in tqdm(enumerate(val_gen),
desc="valset",
total=val_gen._num_examples):
y_val_all += y_val.tolist()
probs = model.predict(X_val)
probabilities += probs.tolist()
if idx > val_gen._num_examples:
break
y_true = np.argmax(y_val_all, axis=-1)
y_pred = np.argmax(probabilities, axis=-1)
visualize.plot_confusion_matrix(y_true,
y_pred,
classes=LABELS,
normalize=True,
experiment=experiment)
visualize.plot_confusion_matrix(y_true,
y_pred,
classes=LABELS,
normalize=False,
experiment=experiment)
experiment.log_confusion_matrix(y_true=y_true,
y_predicted=y_pred,
labels=LABELS)
return model
#Get Alpha score for the weighted spectral/spatial average. Higher alpha favors spatial network.
if model.config["train"]["weighted_sum"]:
estimate_a = model.model.layers[-1].get_weights()
experiment.log_metric(name="spatial-spectral weight", value=estimate_a[0][0])
##Evaluate
#Evaluation scores, see config.yml for tfrecords path
y_pred, y_true = model.evaluate(model.val_split)
#Evaluation accuracy
eval_acc = keras_metrics.CategoricalAccuracy()
eval_acc.update_state(y_true, y_pred)
experiment.log_metric("Evaluation Accuracy",eval_acc.result().numpy())
macro, micro = metrics.f1_scores(y_true, y_pred)
experiment.log_metric("MicroF1",micro)
experiment.log_metric("MacroF1",macro)
print("Unique labels in ytrue {}, unique labels in y_pred {}".format(np.unique(np.argmax(y_true,1)),np.unique(np.argmax(y_pred,1))))
#Read class labels
labeldf = pd.read_csv(model.classes_file)
experiment.log_confusion_matrix(y_true = y_true, y_predicted = y_pred, labels=list(labeldf.taxonID.values), title="Confusion Matrix")
#Save model
model.model.save("{}/{}.h5".format(save_dir,timestamp))
class ModelTrainer:
def __init__(self, model, dataloader, args):
self.model = model
self.args = args
self.data = dataloader
self.metric = args.metric
if (dataloader is not None):
self.frq_log = len(dataloader['train']) // args.frq_log
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
model.to(self.device)
if args.optimizer == 'sgd':
self.optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
elif args.optimizer == 'adam':
self.optimizer = optim.Adam(model.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999),
weight_decay=args.weight_decay)
else:
raise Exception('--optimizer should be one of {sgd, adam}')
if args.scheduler == 'set':
self.scheduler = optim.lr_scheduler.LambdaLR(
self.optimizer,
lambda epoch: 10**(epoch / args.scheduler_factor))
elif args.scheduler == 'auto':
self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(
self.optimizer,
mode='min',
factor=args.scheduler_factor,
patience=5,
verbose=True,
threshold=0.0001,
threshold_mode='rel',
cooldown=0,
min_lr=0,
eps=1e-08)
self.experiment = Experiment(api_key=args.comet_key,
project_name=args.comet_project,
workspace=args.comet_workspace,
auto_weight_logging=True,
auto_metric_logging=False,
auto_param_logging=False)
self.experiment.set_name(args.name)
self.experiment.log_parameters(vars(args))
self.experiment.set_model_graph(str(self.model))
def train_one_epoch(self, epoch):
self.model.train()
train_loader = self.data['train']
train_loss = 0
correct = 0
comet_offset = epoch * len(train_loader)
for batch_idx, (data, target) in tqdm(enumerate(train_loader),
leave=True,
total=len(train_loader)):
data, target = data.to(self.device), target.to(self.device)
self.optimizer.zero_grad()
output = self.model(data)
loss = F.cross_entropy(output, target, reduction='sum')
loss.backward()
self.optimizer.step()
pred = output.argmax(dim=1, keepdim=True)
acc = pred.eq(target.view_as(pred)).sum().item()
train_loss += loss.item()
correct += acc
loss = loss.item() / len(data)
acc = 100. * acc / len(data)
comet_step = comet_offset + batch_idx
self.experiment.log_metric('batch_loss', loss, comet_step, epoch)
self.experiment.log_metric('batch_acc', acc, comet_step, epoch)
if (batch_idx + 1) % self.frq_log == 0:
self.experiment.log_metric('log_loss', loss, comet_step, epoch)
self.experiment.log_metric('log_acc', acc, comet_step, epoch)
print('Epoch: {} [{}/{}]\tLoss: {:.6f}\tAcc: {:.2f}%'.format(
epoch + 1, (batch_idx + 1) * len(data),
len(train_loader.dataset), loss, acc))
train_loss /= len(train_loader.dataset)
acc = 100. * correct / len(train_loader.dataset)
comet_step = comet_offset + len(train_loader) - 1
self.experiment.log_metric('loss', train_loss, comet_step, epoch)
self.experiment.log_metric('acc', acc, comet_step, epoch)
print(
'Epoch: {} [Done]\tLoss: {:.4f}\tAccuracy: {}/{} ({:.2f}%)'.format(
epoch + 1, train_loss, correct, len(train_loader.dataset),
acc))
return {'loss': train_loss, 'acc': acc}
def train(self):
self.log_cmd()
best = -1
history = {'lr': [], 'train_loss': []}
try:
print(">> Training %s" % self.model.name)
for epoch in range(self.args.nepoch):
with self.experiment.train():
train_res = self.train_one_epoch(epoch)
with self.experiment.validate():
print("\nvalidation...")
comet_offset = (epoch + 1) * len(self.data['train']) - 1
res = self.val(self.data['val'], comet_offset, epoch)
if res[self.metric] > best:
best = res[self.metric]
self.save_weights(epoch)
if self.args.scheduler == 'set':
lr = self.optimizer.param_groups[0]['lr']
history['lr'].append(lr)
history['train_loss'].append(train_res['loss'])
self.scheduler.step(epoch + 1)
lr = self.optimizer.param_groups[0]['lr']
print('learning rate changed to: %.10f' % lr)
elif self.args.scheduler == 'auto':
self.scheduler.step(train_res['loss'])
finally:
print(">> Training model %s. [Stopped]" % self.model.name)
self.experiment.log_asset_folder(os.path.join(
self.args.outf, self.args.name, 'weights'),
step=None,
log_file_name=False,
recursive=False)
if self.args.scheduler == 'set':
plt.semilogx(history['lr'], history['train_loss'])
plt.grid(True)
self.experiment.log_figure(figure=plt)
plt.show()
def val(self, val_loader, comet_offset=-1, epoch=-1):
self.model.eval()
test_loss = 0
correct = 0
labels = list(range(self.args.nclass))
cm = np.zeros((len(labels), len(labels)))
with torch.no_grad():
for data, target in tqdm(val_loader,
leave=True,
total=len(val_loader)):
data, target = data.to(self.device), target.to(self.device)
output = self.model(data)
test_loss += F.cross_entropy(output, target,
reduction='sum').item()
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
pred = pred.view_as(target).data.cpu().numpy()
target = target.data.cpu().numpy()
cm += confusion_matrix(target, pred, labels=labels)
test_loss /= len(val_loader.dataset)
accuracy = 100. * correct / len(val_loader.dataset)
print('Evaluation: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)'.
format(test_loss, correct, len(val_loader.dataset), accuracy))
res = {'loss': test_loss, 'acc': accuracy}
self.experiment.log_metrics(res, step=comet_offset, epoch=epoch)
self.experiment.log_confusion_matrix(
matrix=cm,
labels=[ClassDict.getName(x) for x in labels],
title='confusion matrix after epoch %03d' % epoch,
file_name="confusion_matrix_%03d.json" % epoch)
return res
def test(self):
self.load_weights()
with self.experiment.test():
print('\ntesting....')
res = self.val(self.data['test'])
def log_cmd(self):
d = vars(self.args)
cmd = '!python main.py \\\n'
tab = ' '
for k, v in d.items():
if v is None or v == '' or (isinstance(v, bool) and v is False):
continue
if isinstance(v, bool):
arg = '--{} \\\n'.format(k)
else:
arg = '--{} {} \\\n'.format(k, v)
cmd = cmd + tab + arg
# print(cmd);
self.experiment.log_text(cmd)
def save_weights(self, epoch: int):
weight_dir = os.path.join(self.args.outf, self.args.name, 'weights')
if not os.path.exists(weight_dir):
os.makedirs(weight_dir)
torch.save({
'epoch': epoch,
'state_dict': self.model.state_dict()
}, os.path.join(weight_dir, 'model.pth'))
def load_weights(self):
path_g = self.args.weights_path
if path_g is None:
weight_dir = os.path.join(self.args.outf, self.args.name,
'weights')
path_g = os.path.join(weight_dir, 'model.pth')
print('>> Loading weights...')
weights_g = torch.load(path_g, map_location=self.device)['state_dict']
self.model.load_state_dict(weights_g)
print(' Done.')
def predict(self, x):
x = x / 2**15
self.model.eval()
with torch.no_grad():
x = torch.from_numpy(x).float()
x = self.transform(x)
x = x.unsqueeze(0)
x = self.model(x)
x = F.softmax(x, dim=1)
x = x.numpy()
return x
exp.log_metric(f"img_{k}", v, step=idx)
# Confusion matrix
confmat, _ = get_confusion_matrix(
metrics_dict["tpr"],
metrics_dict["tnr"],
metrics_dict["fpr"],
metrics_dict["fnr"],
metrics_dict["mnr"],
metrics_dict["mpr"],
)
confmat = np.around(confmat, decimals=3)
exp.log_confusion_matrix(
file_name=imgs_paths[idx].name + ".json",
title=imgs_paths[idx].name,
matrix=confmat,
labels=["Cannot", "Must", "May"],
row_label="Predicted",
column_label="Ground truth",
)
if args.plot:
# Plot prediction images
fig_filename = plot_dir / imgs_paths[idx].name
plot_images(
fig_filename,
img,
label,
pred,
metrics_dict,
maps_dict,
edge_coherence,
11: "Sidewalks",
12: "Crosswalks",
13: "Major thoroughfares",
14: "Highways",
15: "Railways",
16: "Paved parking lots",
17: "Unpaved parking lots",
18: "Cars",
19: "Trains",
20: "Stadium seat"
}
print("Unique labels in ytrue {}, unique labels in y_pred {}".format(
np.unique(np.argmax(y_true, 1)), np.unique(np.argmax(y_pred, 1))))
experiment.log_confusion_matrix(y_true=y_true,
y_predicted=y_pred,
labels=list(class_labels.values()),
title="Confusion Matrix")
#Predict
predict_tfrecords = glob.glob(
"/orange/ewhite/b.weinstein/Houston2018/tfrecords/predict/*.tfrecord")
results = model.predict_raster(predict_tfrecords, batch_size=512)
#predicted classes
print(results.label.unique())
predicted_raster = visualize.create_raster(results)
print(np.unique(predicted_raster))
experiment.log_image(name="Prediction",
image_data=predicted_raster,
image_colormap=visualize.discrete_cmap(20,
base_cmap="jet"))
def model_eval(
models: list,
x: np.array,
y: np.array,
experiment: Experiment,
postfix: str = "_train",
outdir_models: str = None,
):
"""Peforms a model evaluation on training and test set and dump the predictions with the actual values
into an outout file
Arguments:
models {list} -- list of tuples with model name and model itself
x {np.array} -- feature matrix
y {np.array} -- label vector
experiment {comet_ml.Experiment}
postfix {str} -- string that will be attached to filename
outdir_models {str} -- output directory for models
"""
predictions = []
trainlogger.debug("entered evaluation function")
for name, model in models:
postfix_ = "_".join([postfix, name])
outname_base_models = os.path.join(
outdir_models, "_".join([STARTTIMESTRING, postfix_]))
dump(model, outname_base_models + ".joblib")
experiment.log_asset(outname_base_models + ".joblib")
predict = model.predict(x)
accuracy = accuracy_score(y, predict)
f1_micro = f1_score(y, predict, average="micro")
f1_macro = f1_score(y, predict, average="macro")
balanced_accuracy = balanced_accuracy_score(y, predict)
precision = precision_score(y, predict, average="micro")
recall = recall_score(y, predict, average="micro")
prediction = {
"model": name,
"outname_base_models": outname_base_models,
"accuracy" + postfix_: accuracy,
"f1_micro" + postfix_: f1_micro,
"f1_macro" + postfix_: f1_macro,
"balanced_accuracy" + postfix_: balanced_accuracy,
"precision" + postfix_: precision,
"recall" + postfix_: recall,
"points" + postfix_: len(y),
"n_features" + postfix_: x.shape[0],
}
predictions.append(prediction)
try:
experiment.log_metrics(prediction)
experiment.log_confusion_matrix(
keras.utils.to_categorical(y),
keras.utils.to_categorical(predict),
title=postfix_.strip("_"),
)
except Exception:
pass
return predictions
