def _print_val_results(self, precisions, recalls, dices, epoch, name, classes, logger): # Log the results # We add them to a pd dataframe just for the pretty print output index = ["cls %i" % i for i in classes] val_results = pd.DataFrame( { "precision": precisions, "recall": recalls, "dice": dices, }, index=index) # Transpose the results to have metrics in rows val_results = val_results.T # Add mean and set in first row means = [precisions.mean(), recalls.mean(), dices.mean()] val_results["mean"] = means cols = list(val_results.columns) cols.insert(0, cols.pop(cols.index('mean'))) val_results = val_results.ix[:, cols] # Print the df to screen logger( highlighted(("[%s] Validation Results for " "Epoch %i" % (name, epoch)).lstrip(" "))) logger(val_results.round(self.print_round).to_string() + "\n")
def _print_val_results(self, class_wise_metrics, batch_wise_metrics, epoch, task_name, classes): """ TODO :param class_wise_metrics: :param batch_wise_metrics: :param epoch: :param task_name: :param classes: :return: """ # We add them to a pd dataframe just for the pretty print output index = ["mean"] + ["cls %i" % i for i in classes] columns = list(batch_wise_metrics.keys()) + list( class_wise_metrics.keys()) df = pd.DataFrame(data={c: [np.nan] * len(index) for c in columns}, index=index) # Fill the df with metrics for m_name, value in batch_wise_metrics.items(): df.loc['mean', m_name] = value for m_name, values in class_wise_metrics.items(): values = [np.nanmean(values)] + list(values) df.loc[:, m_name] = values # Print the df to screen s = "Validation Results for epoch %i" % epoch self.logger( highlighted((("[%s]" % task_name) if task_name else "") + s)) print_string = df.round(self.print_round).T.to_string() self.logger(print_string.replace("NaN", " -") + "\n")
def _print_val_results(self, precisions, recalls, dices, metrics, epoch, name, classes): # Log the results # We add them to a pd dataframe just for the pretty print output index = ["cls %i" % i for i in classes] metric_keys, metric_vals = map(list, list(zip(*metrics.items()))) col_order = metric_keys + ["precision", "recall", "dice"] nan_arr = np.empty(shape=len(precisions)) nan_arr[:] = np.nan value_dict = {"precision": precisions, "recall": recalls, "dice": dices} value_dict.update({key: nan_arr for key in metrics}) val_results = pd.DataFrame(value_dict, index=index).loc[:, col_order] # ensure order # Transpose the results to have metrics in rows val_results = val_results.T # Add mean and set in first row means = metric_vals + [precisions.mean(), recalls.mean(), dices.mean()] val_results["mean"] = means cols = list(val_results.columns) cols.insert(0, cols.pop(cols.index('mean'))) val_results = val_results.loc[:, cols] # Print the df to screen self.logger(highlighted(("[%s] Validation Results for " "Epoch %i" % (name, epoch)).lstrip(" "))) print_string = val_results.round(self.print_round).to_string() self.logger(print_string.replace("NaN", "---") + "\n")
def on_epoch_end(self, epoch, logs={}): scores = self.eval() mean_dice = scores.mean() s = "Mean dice for epoch %d: %.4f\nPr. class: %s" % (epoch, mean_dice, scores) self.logger(highlighted(s)) self.scores.append(mean_dice) # Add to log logs["val_dice"] = mean_dice
def log(self): self.logger(highlighted("\nAudit for %i images" % len(self.nii_paths))) self.logger("Total memory GiB: %.3f" % self.total_memory_gib) if self.n_classes is not None: self.logger("Number of classes: %i" % self.n_classes) self.logger("\n2D:\n" "Real space span: %.3f\n" "Sample dim: %.3f" % (self.real_space_span_2D, self.sample_dim_2D)) self.logger( "\n3D:\n" "Sample dim: %i\n" "Real space span: %.3f\n" "Box span: %.3f" % (self.sample_dim_3D, self.real_space_span_3D, self.real_box_span))
def on_epoch_end(self, epoch, logs={}): self.logger("\n") # Predict and get CM TPs, relevant, selected, metrics = self.predict() for name in self.IDs: tp, rel, sel = TPs[name], relevant[name], selected[name] precisions, recalls, dices = self._compute_dice(tp=tp, sel=sel, rel=rel) classes = np.arange(len(dices)) # Add to log n = (name + "_") if len(self.IDs) > 1 else "" logs[f"{n}val_dice"] = dices.mean().round(self.log_round) logs[f"{n}val_precision"] = precisions.mean().round(self.log_round) logs[f"{n}val_recall"] = recalls.mean().round(self.log_round) for m_name, value in metrics[name].items(): logs[f"{n}val_{m_name}"] = value.round(self.log_round) if self.verbose: self._print_val_results(precisions=precisions, recalls=recalls, dices=dices, epoch=epoch, name=name, classes=classes, logger=self.logger) if len(self.IDs) > 1: # Print cross-dataset mean values if self.verbose: self.logger( highlighted(f"[ALL DATASETS] Means Across Classes" f" for Epoch {epoch}")) fetch = ("val_dice", "val_precision", "val_recall") m_fetch = tuple(["val_" + s for s in self.model.metrics_names]) to_print = {} for f in fetch + m_fetch: scores = [logs["%s_%s" % (name, f)] for name in self.IDs] res = np.mean(scores) logs[f] = res.round(self.log_round) # Add to log file to_print[f.split("_")[-1]] = list(scores) + [res] if self.verbose: df = pd.DataFrame(to_print) df.index = self.IDs + ["mean"] print(df.round(self.print_round)) self.logger("")
def on_epoch_end(self, epoch, logs={}): # Predict and get CM class_wise_metrics, mean_batch_wise_metrics = self.evalaute() for n_classes, name in zip(self.n_classes, self.task_names): classes = np.arange(n_classes) n = (name + "_") if len(self.task_names) > 1 else "" # Add batch-wise metrics to log for m_name, value in mean_batch_wise_metrics[name].items(): logs[f"{n}val_{m_name}"] = value.round(self.log_round) # Add mean of class-wise metrics to log for m_name, values in class_wise_metrics[name].items(): logs[f"{n}val_{m_name}"] = np.nanmean(values) if self.verbose: self._print_val_results( class_wise_metrics=class_wise_metrics[name], batch_wise_metrics=mean_batch_wise_metrics[name], epoch=epoch, task_name=name, classes=classes) if len(self.task_names) > 1: # Print cross-dataset mean values if self.verbose: self.logger( highlighted(f"[ALL DATASETS] Means Across Classes" f" for Epoch {epoch}")) fetch = ("val_dice", "val_precision", "val_recall") m_fetch = tuple(["val_" + s for s in self.model.metrics_names]) to_print = {} for f in fetch + m_fetch: scores = [ logs["%s_%s" % (name, f)] for name in self.task_names ] res = np.mean(scores) logs[f] = res.round(self.log_round) # Add to log file to_print[f.split("_")[-1]] = list(scores) + [res] if self.verbose: df = pd.DataFrame(to_print) df.index = self.task_names + ["mean"] self.logger(df.round(self.print_round)) self.logger("")
def _run_fusion_training(sets, logger, hparams, min_val_images, is_validation, views, n_classes, unet, fusion_model, early_stopping, fm_batch_size, epochs, eval_prob, fusion_weights_path): """ TODO """ for _round, _set in enumerate(sets): s = "Set %i/%i:\n%s" % (_round + 1, len(sets), _set) logger("\n%s" % highlighted(s)) # Reload data images = ImagePairLoader(**hparams["val_data"]) if len(images) < min_val_images: images.add_images(ImagePairLoader(**hparams["train_data"])) # Get list of ImagePair objects to run on image_set_dict = { m.identifier: m for m in images if m.identifier in _set } # Set scaler and bg values images.set_scaler_and_bg_values( bg_value=hparams.get_from_anywhere('bg_value'), scaler=hparams.get_from_anywhere('scaler'), compute_now=False) # Init LazyQueue and get its sequencer from mpunet.sequences.utils import get_sequence seq = get_sequence(data_queue=images, is_validation=True, views=views, **hparams["fit"], **hparams["build"]) # Fetch points from the set images points_collection = [] targets_collection = [] N_im = len(image_set_dict) for num_im, image_id in enumerate(list(image_set_dict.keys())): logger("") logger( highlighted("(%i/%i) Running on %s (%s)" % (num_im + 1, N_im, image_id, "val" if is_validation[image_id] else "train"))) with seq.image_pair_queue.get_image_by_id(image_id) as image: # Get voxel grid in real space voxel_grid_real_space = get_voxel_grid_real_space(image) # Get array to store predictions across all views targets = image.labels.reshape(-1, 1) points = np.empty(shape=(len(targets), len(views), n_classes), dtype=np.float32) points.fill(np.nan) # Predict on all views for k, v in enumerate(views): print("\n%s" % "View: %s" % v) points[:, k, :] = predict_and_map( model=unet, seq=seq, image=image, view=v, voxel_grid_real_space=voxel_grid_real_space, n_planes='same+20', targets=targets, eval_prob=eval_prob).reshape(-1, n_classes) # add to collections points_collection.append(points) targets_collection.append(targets) print(image.is_loaded) # Stack points into one matrix logger("Stacking points...") X, y = stack_collections(points_collection, targets_collection) # Shuffle train print("Shuffling points...") X, y = shuffle(X, y) print("Getting validation set...") val_ind = int(0.20 * X.shape[0]) X_val, y_val = X[:val_ind], y[:val_ind] X, y = X[val_ind:], y[val_ind:] # Prepare dice score callback for validation data val_cb = ValDiceScores((X_val, y_val), n_classes, 50000, logger) # Callbacks cbs = [ val_cb, CSVLogger(filename="logs/fusion_training.csv", separator=",", append=True), PrintLayerWeights(fusion_model.layers[-1], every=1, first=1000, per_epoch=True, logger=logger) ] es = EarlyStopping(monitor='val_dice', min_delta=0.0, patience=early_stopping, verbose=1, mode='max') cbs.append(es) # Start training try: fusion_model.fit(X, y, batch_size=fm_batch_size, epochs=epochs, callbacks=cbs, verbose=1) except KeyboardInterrupt: pass fusion_model.save_weights(fusion_weights_path)