class CometWriter(LoggerWriter): def __init__(self, project_name, enable_log, cfg): from comet_ml import Experiment super().__init__(project_name, enable_log, cfg) self.exp = Experiment(api_key=cfg.comet_api_key, project_name=project_name, disabled=not enable_log) def log_params(self, args): self.exp.log_parameters(vars(args)) def log_metric(self, name, value, step): self.exp.log_metric(name, value, step) def log_confusion_matrix(self, matrix, step): self.exp.log_confusion_matrix( matrix=matrix, step=step, file_name="confusion_matrix_{}".format(step))
def eval( model: LiSSModel, dataset: UnalignedDataset, exp: Experiment, total_iters: int = 0, nb_ims: int = 30, ): liss = model.opt.model == "liss" metrics = {} print(f"----------- Evaluation {total_iters} ----------") with torch.no_grad(): data = { "translation": { "A": { "rec": None, "idt": None, "real": None, "fake": None }, "B": { "rec": None, "idt": None, "real": None, "fake": None }, } } force = set(["identity", "translation"]) if liss: for t in model.tasks: tmp = {} if t.eval_visuals_pred or t.log_type == "acc": tmp["pred"] = None if t.eval_visuals_target or t.log_type == "acc": tmp["target"] = None data[t.key] = {domain: deepcopy(tmp) for domain in "AB"} force |= set(model.tasks.keys) losses = { k: [] for k in dir(model) if k.startswith("loss_") and isinstance(getattr(model, k), torch.Tensor) } for i, b in enumerate(dataset): # print(f"\rEval batch {i}", end="") model.set_input(b) model.forward(force=force) model.backward_G(losses_only=True, force=force) for k in dir(model): if k.startswith("loss_") and isinstance( getattr(model, k), torch.Tensor): if k not in losses: losses[k] = [] losses[k].append(getattr(model, k).detach().cpu().item()) if liss: for t in model.tasks: for domain in "AB": for dtype in data[t.key][domain]: if (t.log_type != "acc" and data[t.key][domain][dtype] is not None and len(data[t.key][domain][dtype]) >= nb_ims): continue v = model.get( f"{domain}_{t.key}_{dtype}").detach().cpu() if data[t.key][domain][dtype] is None: data[t.key][domain][dtype] = v else: data[t.key][domain][dtype] = torch.cat( [data[t.key][domain][dtype], v], dim=0, ) # ------------------------- # ----- Translation ----- # ------------------------- if (data["translation"]["A"]["real"] is None or len(data["translation"]["A"]["real"]) < nb_ims): for domain in "AB": for dtype in ["real", "fake", "rec", "idt"]: dom = domain if dtype in {"fake", "idt"}: dom = swap_domain(domain) v = model.get(f"{dom}_{dtype}").detach().cpu() if data["translation"][domain][dtype] is None: data["translation"][domain][dtype] = v else: data["translation"][domain][dtype] = torch.cat( [data["translation"][domain][dtype], v], dim=0) # print( # f"{domain} {dtype} {len(data['translation'][domain][dtype])}" # ) for task in data: if task != "translation" and model.tasks[task].log_type != "vis": continue for domain in data[task]: for i, v in data[task][domain].items(): data[task][domain][i] = torch.cat(list(v[:nb_ims].permute( 0, 2, 3, 1)), axis=1) log_images = int(data["translation"]["A"]["real"].shape[1] / data["translation"]["A"]["real"].shape[0]) im_size = data["translation"]["A"]["real"].shape[0] ims = {"A": None, "B": None} data_keys = ["translation"] translation_keys = ["real", "fake", "rec", "idt"] data_keys += [task for task in data if task not in data_keys] for task in data_keys: if task != "translation" and model.tasks[task].log_type != "vis": continue for domain in "AB": im_types = (translation_keys if task == "translation" else list( data[task][domain].keys())) for im_type in im_types: v = data[task][domain][im_type].float() if task == "depth": v = to_min1_1(v) v = v.repeat((1, 1, 3)) v = v + 1 v = v / 2 if ims[domain] is None: ims[domain] = v else: ims[domain] = torch.cat([ims[domain], v], dim=0) # ------------------------ # ----- Comet Logs ----- # ------------------------ for i in range(0, log_images, 5): k = i + 5 exp.log_image( ims["A"][:, i * im_size:k * im_size, :].numpy(), "test_A_{}_{}_rfcidg".format(i * 5, (i + 1) * 5 - 1), step=total_iters, ) exp.log_image( ims["B"][:, i * im_size:k * im_size, :].numpy(), "test_B_{}_{}_rfcidg".format(i * 5, (i + 1) * 5 - 1), step=total_iters, ) if liss: test_losses = { "test_" + ln: np.mean(losses["loss_" + ln]) for t in model.tasks for ln in t.loss_names } test_accs = { f"test_G_{domain}_{t.key}_acc": np.mean(data[t.key][domain]["pred"].max(-1)[1].numpy() == data[ t.key][domain]["target"].numpy()) for domain in "AB" for t in model.tasks if t.log_type == "acc" } if liss: exp.log_metrics(test_losses, step=total_iters) exp.log_metrics(test_accs, step=total_iters) for t in model.tasks: if t.log_type != "acc": continue for domain in "AB": target = data[t.key][domain]["target"].numpy() pred = data[t.key][domain]["pred"].numpy() exp.log_confusion_matrix( get_one_hot(target, t.output_dim), pred, file_name= f"confusion_{domain}_{t.key}_{total_iters}.json", title=f"confusion_{domain}_{t.key}_{total_iters}.json", ) metrics = {k + "_loss": v for k, v in test_losses.items()} metrics.update(test_accs) print("----------- End Evaluation----------") return metrics
# Log accuracy to Comet.ml experiment.log_metric("accuracy", correct / total, step=step) step += 1 if (i + 1) % 100 == 0: print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f' % (epoch + 1, hyper_params['num_epochs'], i + 1, len(train_dataset) // hyper_params['batch_size'], loss.data.item())) # At end of epoch: print("Computing confusion matrix and uploading samples...") confusion_matrix.compute_matrix(epoch_targets, epoch_predictions) experiment.log_confusion_matrix( matrix=confusion_matrix, title="Train Confusion Matrix, Epoch #%s" % (epoch + 1, ), file_name="train-confusion-matrix-%03d.json" % (epoch + 1), ) print("Logging weights as histogram...") # Log model weights weights = [] for name in rnn.named_parameters(): if 'weight' in name[0]: weights.extend(name[1].detach().numpy().tolist()) experiment.log_histogram_3d(weights, step=epoch + 1) with experiment.test(): # Test the Model correct = 0 total = 0
experiment.log_metrics(metrics) # Optionally save per-label recall, precision and F1 # label_metrics = pd.DataFrame.from_dict(cls_report_dct, orient='index') # label_metrics_file = os.path.join(output_dir, experiment_name + "_label_metrics.csv") # label_metrics.to_csv(label_metrics_file) # experiment.log_asset(label_metrics_file) conf_matrix_fig, confusion_matrix = plot_confusion_matrix( true_labels, predicted_labels, labels) confusion_matrix_file = os.path.join( output_dir, experiment_name + "_confusion_matrix.png") conf_matrix_fig.savefig(confusion_matrix_file) experiment.log_image(confusion_matrix_file) experiment.log_confusion_matrix( matrix=confusion_matrix.tolist(), labels=labels[:len(confusion_matrix)]) end_time = time.time() print("Testing took " + str(('%.3f' % (end_time - start_time))) + " seconds for " + str(test_steps) + " steps") # Close the current session sess.close() tf.keras.backend.clear_session() # Log results to csv if training and testing: experiment_file = os.path.join( task_name, task_name + "_" + experiment_type + ".csv") save_experiment(experiment_file, experiment_params,
# Comet-ML log data for i in history.epoch: experiment.log_metric("accuracy", history.history['accuracy'][i], epoch=i + 1) experiment.log_metric("loss", history.history['loss'][i], epoch=i + 1) ## Confusion matrix y_test_hat = model.predict(X_test, batch_size=4) y_test_hat = np.argmax(y_test_hat, axis=1) y_test = np.argmax(y_test, axis=1) conf_m = confusion_matrix(y_test, y_test_hat) # Comet-ML log data experiment.log_confusion_matrix(y_test, y_test_hat) # Plot matrix plt.figure(figsize=(5, 3)) sb.set(font_scale=1.2) ax = sb.heatmap(conf_m, annot=True, xticklabels=['N', 'P'], yticklabels=['N', 'P'], cbar=False, cmap='Blues', linewidths=1, linecolor='black', fmt='.0f') plt.yticks(rotation=0) plt.xlabel('Predicted labels')
model.compile(loss='categorical_crossentropy', optimizer=_OPTIMIZER) model.fit(oDataSet.attributes[oData.Training_indexes], binarizer(oDataSet.labels[oData.Training_indexes]), batch_size=1, epochs=epochs, verbose=1) y_pred = model.predict( oDataSet.attributes[oData.Testing_indexes]).argmax(axis=1) y_true = oDataSet.labels[oData.Testing_indexes] experiment.log_metric("test_accuracy", accuracy_score(y_true, y_pred)) experiment.log_metric("beta", best_b) experiment.log_metric("neurons", best_p) experiment.log_confusion_matrix(matrix=confusion_matrix(y_true, y_pred).tolist(), labels=oDataSet.labelsNames) # model.save('model.h5') # experiment.log_asset("model.h5") model.save_weights('model.weights') experiment.log_asset("model.weights") print(accuracy_score(y_true, y_pred)) print(confusion_matrix(y_true, y_pred)) oData.confusion_matrix = confusion_matrix(y_true, y_pred) oData.model = model oData.params = { "k_fold": K_FOLD, "GRID_RESULT": grid_result, "GRID_VALUES_NEURON": GRID_NEURON, "GRID_VALUES_BETA": GRID_B,
ap, f1_max, precision, recall, f1_max_th, fig_pre_rec, fig_th_pre_rec = precision_recall(y_test, score_test, limit=1000, label_anomaly=labels[0]) #fig_score_train = plot_score(score_train, 'train') #fig_score_val = plot_score(score_val, 'validation') fig_score_test = plot_score(score_test, 'test', 10, labels=y_test, th=f1_max_th) fig_cumsum = pca.plot_cumsum() y_pred, conf_matrix = predict(score_test, f1_max_th, y_test, labels) experiment.add_tags([data, metric]) parameters = {'var': var, 'pc': pca.pcs_, 'metric': metric} experiment.log_parameters(parameters) experiment.log_metric('ap', ap) experiment.log_metric('f1', f1_max) experiment.log_metric('precision', precision) experiment.log_metric('recall', recall) experiment.log_metric('train_time', pca.time_) experiment.log_parameter('th_f1', f1_max_th) experiment.log_figure('cumsum', fig_cumsum) experiment.log_figure('score_test',fig_score_test) experiment.log_figure('precision_recall',fig_pre_rec) experiment.log_figure('th_pre_rec_f1', fig_th_pre_rec) experiment.log_confusion_matrix(matrix=conf_matrix, labels=labels) experiment.end()
def train_and_evaluate(self, train_gen, val_gen, epochs): """ """ experiment = Experiment( api_key="VNQSdbR1pw33EkuHbUsGUSZWr", project_name="piratesofthecaribbean", workspace="florpi", ) model = self.build() with experiment.train(): model_path = os.path.join(self.directory, "cnn_{epoch:02d}-{val_loss:.2f}.hdf5") callbacks = [ ModelCheckpoint(model_path, monitor="val_loss", mode="min"), # EarlyStopping( # monitor="val_loss", # mode="min", # min_delta=0.1, # patience=1, # restore_best_weights=True, # ), ] model.fit( train_gen, epochs=epochs, validation_data=val_gen, callbacks=callbacks, class_weight=CLASS_WEIGHTS, ) model.save(os.path.join(self.directory, "cnn_final.h5")) # Run validation with experiment.test(): probabilities = [] y_val_all = [] # reset generator val_gen.reset() for idx, (X_val, y_val) in tqdm(enumerate(val_gen), desc="valset", total=val_gen._num_examples): y_val_all += y_val.tolist() probs = model.predict(X_val) probabilities += probs.tolist() if idx > val_gen._num_examples: break y_true = np.argmax(y_val_all, axis=-1) y_pred = np.argmax(probabilities, axis=-1) visualize.plot_confusion_matrix(y_true, y_pred, classes=LABELS, normalize=True, experiment=experiment) visualize.plot_confusion_matrix(y_true, y_pred, classes=LABELS, normalize=False, experiment=experiment) experiment.log_confusion_matrix(y_true=y_true, y_predicted=y_pred, labels=LABELS) return model
#Get Alpha score for the weighted spectral/spatial average. Higher alpha favors spatial network. if model.config["train"]["weighted_sum"]: estimate_a = model.model.layers[-1].get_weights() experiment.log_metric(name="spatial-spectral weight", value=estimate_a[0][0]) ##Evaluate #Evaluation scores, see config.yml for tfrecords path y_pred, y_true = model.evaluate(model.val_split) #Evaluation accuracy eval_acc = keras_metrics.CategoricalAccuracy() eval_acc.update_state(y_true, y_pred) experiment.log_metric("Evaluation Accuracy",eval_acc.result().numpy()) macro, micro = metrics.f1_scores(y_true, y_pred) experiment.log_metric("MicroF1",micro) experiment.log_metric("MacroF1",macro) print("Unique labels in ytrue {}, unique labels in y_pred {}".format(np.unique(np.argmax(y_true,1)),np.unique(np.argmax(y_pred,1)))) #Read class labels labeldf = pd.read_csv(model.classes_file) experiment.log_confusion_matrix(y_true = y_true, y_predicted = y_pred, labels=list(labeldf.taxonID.values), title="Confusion Matrix") #Save model model.model.save("{}/{}.h5".format(save_dir,timestamp))
class ModelTrainer: def __init__(self, model, dataloader, args): self.model = model self.args = args self.data = dataloader self.metric = args.metric if (dataloader is not None): self.frq_log = len(dataloader['train']) // args.frq_log self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') model.to(self.device) if args.optimizer == 'sgd': self.optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) elif args.optimizer == 'adam': self.optimizer = optim.Adam(model.parameters(), lr=args.lr, betas=(args.beta1, 0.999), weight_decay=args.weight_decay) else: raise Exception('--optimizer should be one of {sgd, adam}') if args.scheduler == 'set': self.scheduler = optim.lr_scheduler.LambdaLR( self.optimizer, lambda epoch: 10**(epoch / args.scheduler_factor)) elif args.scheduler == 'auto': self.scheduler = optim.lr_scheduler.ReduceLROnPlateau( self.optimizer, mode='min', factor=args.scheduler_factor, patience=5, verbose=True, threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08) self.experiment = Experiment(api_key=args.comet_key, project_name=args.comet_project, workspace=args.comet_workspace, auto_weight_logging=True, auto_metric_logging=False, auto_param_logging=False) self.experiment.set_name(args.name) self.experiment.log_parameters(vars(args)) self.experiment.set_model_graph(str(self.model)) def train_one_epoch(self, epoch): self.model.train() train_loader = self.data['train'] train_loss = 0 correct = 0 comet_offset = epoch * len(train_loader) for batch_idx, (data, target) in tqdm(enumerate(train_loader), leave=True, total=len(train_loader)): data, target = data.to(self.device), target.to(self.device) self.optimizer.zero_grad() output = self.model(data) loss = F.cross_entropy(output, target, reduction='sum') loss.backward() self.optimizer.step() pred = output.argmax(dim=1, keepdim=True) acc = pred.eq(target.view_as(pred)).sum().item() train_loss += loss.item() correct += acc loss = loss.item() / len(data) acc = 100. * acc / len(data) comet_step = comet_offset + batch_idx self.experiment.log_metric('batch_loss', loss, comet_step, epoch) self.experiment.log_metric('batch_acc', acc, comet_step, epoch) if (batch_idx + 1) % self.frq_log == 0: self.experiment.log_metric('log_loss', loss, comet_step, epoch) self.experiment.log_metric('log_acc', acc, comet_step, epoch) print('Epoch: {} [{}/{}]\tLoss: {:.6f}\tAcc: {:.2f}%'.format( epoch + 1, (batch_idx + 1) * len(data), len(train_loader.dataset), loss, acc)) train_loss /= len(train_loader.dataset) acc = 100. * correct / len(train_loader.dataset) comet_step = comet_offset + len(train_loader) - 1 self.experiment.log_metric('loss', train_loss, comet_step, epoch) self.experiment.log_metric('acc', acc, comet_step, epoch) print( 'Epoch: {} [Done]\tLoss: {:.4f}\tAccuracy: {}/{} ({:.2f}%)'.format( epoch + 1, train_loss, correct, len(train_loader.dataset), acc)) return {'loss': train_loss, 'acc': acc} def train(self): self.log_cmd() best = -1 history = {'lr': [], 'train_loss': []} try: print(">> Training %s" % self.model.name) for epoch in range(self.args.nepoch): with self.experiment.train(): train_res = self.train_one_epoch(epoch) with self.experiment.validate(): print("\nvalidation...") comet_offset = (epoch + 1) * len(self.data['train']) - 1 res = self.val(self.data['val'], comet_offset, epoch) if res[self.metric] > best: best = res[self.metric] self.save_weights(epoch) if self.args.scheduler == 'set': lr = self.optimizer.param_groups[0]['lr'] history['lr'].append(lr) history['train_loss'].append(train_res['loss']) self.scheduler.step(epoch + 1) lr = self.optimizer.param_groups[0]['lr'] print('learning rate changed to: %.10f' % lr) elif self.args.scheduler == 'auto': self.scheduler.step(train_res['loss']) finally: print(">> Training model %s. [Stopped]" % self.model.name) self.experiment.log_asset_folder(os.path.join( self.args.outf, self.args.name, 'weights'), step=None, log_file_name=False, recursive=False) if self.args.scheduler == 'set': plt.semilogx(history['lr'], history['train_loss']) plt.grid(True) self.experiment.log_figure(figure=plt) plt.show() def val(self, val_loader, comet_offset=-1, epoch=-1): self.model.eval() test_loss = 0 correct = 0 labels = list(range(self.args.nclass)) cm = np.zeros((len(labels), len(labels))) with torch.no_grad(): for data, target in tqdm(val_loader, leave=True, total=len(val_loader)): data, target = data.to(self.device), target.to(self.device) output = self.model(data) test_loss += F.cross_entropy(output, target, reduction='sum').item() pred = output.argmax(dim=1, keepdim=True) correct += pred.eq(target.view_as(pred)).sum().item() pred = pred.view_as(target).data.cpu().numpy() target = target.data.cpu().numpy() cm += confusion_matrix(target, pred, labels=labels) test_loss /= len(val_loader.dataset) accuracy = 100. * correct / len(val_loader.dataset) print('Evaluation: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)'. format(test_loss, correct, len(val_loader.dataset), accuracy)) res = {'loss': test_loss, 'acc': accuracy} self.experiment.log_metrics(res, step=comet_offset, epoch=epoch) self.experiment.log_confusion_matrix( matrix=cm, labels=[ClassDict.getName(x) for x in labels], title='confusion matrix after epoch %03d' % epoch, file_name="confusion_matrix_%03d.json" % epoch) return res def test(self): self.load_weights() with self.experiment.test(): print('\ntesting....') res = self.val(self.data['test']) def log_cmd(self): d = vars(self.args) cmd = '!python main.py \\\n' tab = ' ' for k, v in d.items(): if v is None or v == '' or (isinstance(v, bool) and v is False): continue if isinstance(v, bool): arg = '--{} \\\n'.format(k) else: arg = '--{} {} \\\n'.format(k, v) cmd = cmd + tab + arg # print(cmd); self.experiment.log_text(cmd) def save_weights(self, epoch: int): weight_dir = os.path.join(self.args.outf, self.args.name, 'weights') if not os.path.exists(weight_dir): os.makedirs(weight_dir) torch.save({ 'epoch': epoch, 'state_dict': self.model.state_dict() }, os.path.join(weight_dir, 'model.pth')) def load_weights(self): path_g = self.args.weights_path if path_g is None: weight_dir = os.path.join(self.args.outf, self.args.name, 'weights') path_g = os.path.join(weight_dir, 'model.pth') print('>> Loading weights...') weights_g = torch.load(path_g, map_location=self.device)['state_dict'] self.model.load_state_dict(weights_g) print(' Done.') def predict(self, x): x = x / 2**15 self.model.eval() with torch.no_grad(): x = torch.from_numpy(x).float() x = self.transform(x) x = x.unsqueeze(0) x = self.model(x) x = F.softmax(x, dim=1) x = x.numpy() return x
exp.log_metric(f"img_{k}", v, step=idx) # Confusion matrix confmat, _ = get_confusion_matrix( metrics_dict["tpr"], metrics_dict["tnr"], metrics_dict["fpr"], metrics_dict["fnr"], metrics_dict["mnr"], metrics_dict["mpr"], ) confmat = np.around(confmat, decimals=3) exp.log_confusion_matrix( file_name=imgs_paths[idx].name + ".json", title=imgs_paths[idx].name, matrix=confmat, labels=["Cannot", "Must", "May"], row_label="Predicted", column_label="Ground truth", ) if args.plot: # Plot prediction images fig_filename = plot_dir / imgs_paths[idx].name plot_images( fig_filename, img, label, pred, metrics_dict, maps_dict, edge_coherence,
11: "Sidewalks", 12: "Crosswalks", 13: "Major thoroughfares", 14: "Highways", 15: "Railways", 16: "Paved parking lots", 17: "Unpaved parking lots", 18: "Cars", 19: "Trains", 20: "Stadium seat" } print("Unique labels in ytrue {}, unique labels in y_pred {}".format( np.unique(np.argmax(y_true, 1)), np.unique(np.argmax(y_pred, 1)))) experiment.log_confusion_matrix(y_true=y_true, y_predicted=y_pred, labels=list(class_labels.values()), title="Confusion Matrix") #Predict predict_tfrecords = glob.glob( "/orange/ewhite/b.weinstein/Houston2018/tfrecords/predict/*.tfrecord") results = model.predict_raster(predict_tfrecords, batch_size=512) #predicted classes print(results.label.unique()) predicted_raster = visualize.create_raster(results) print(np.unique(predicted_raster)) experiment.log_image(name="Prediction", image_data=predicted_raster, image_colormap=visualize.discrete_cmap(20, base_cmap="jet"))
def model_eval( models: list, x: np.array, y: np.array, experiment: Experiment, postfix: str = "_train", outdir_models: str = None, ): """Peforms a model evaluation on training and test set and dump the predictions with the actual values into an outout file Arguments: models {list} -- list of tuples with model name and model itself x {np.array} -- feature matrix y {np.array} -- label vector experiment {comet_ml.Experiment} postfix {str} -- string that will be attached to filename outdir_models {str} -- output directory for models """ predictions = [] trainlogger.debug("entered evaluation function") for name, model in models: postfix_ = "_".join([postfix, name]) outname_base_models = os.path.join( outdir_models, "_".join([STARTTIMESTRING, postfix_])) dump(model, outname_base_models + ".joblib") experiment.log_asset(outname_base_models + ".joblib") predict = model.predict(x) accuracy = accuracy_score(y, predict) f1_micro = f1_score(y, predict, average="micro") f1_macro = f1_score(y, predict, average="macro") balanced_accuracy = balanced_accuracy_score(y, predict) precision = precision_score(y, predict, average="micro") recall = recall_score(y, predict, average="micro") prediction = { "model": name, "outname_base_models": outname_base_models, "accuracy" + postfix_: accuracy, "f1_micro" + postfix_: f1_micro, "f1_macro" + postfix_: f1_macro, "balanced_accuracy" + postfix_: balanced_accuracy, "precision" + postfix_: precision, "recall" + postfix_: recall, "points" + postfix_: len(y), "n_features" + postfix_: x.shape[0], } predictions.append(prediction) try: experiment.log_metrics(prediction) experiment.log_confusion_matrix( keras.utils.to_categorical(y), keras.utils.to_categorical(predict), title=postfix_.strip("_"), ) except Exception: pass return predictions