def load_from_pkl(self, path: str):
"""
Load attribute dict from pickle file.
Args:
path: Path to pickle file which contains the attribute dictionary.
"""
self.__init__(**load_pkl(path))
return self
def merge_reports(set_path: str, eval_name: str):
"""
Combines all reports from multiple evaluations (e.g. checkpoint evaluations) to a single report.
"""
set_path = os.path.expanduser(set_path)
dirs = os.listdir(set_path)
reports = {}
for di in dirs:
report_path = os.path.join(set_path,
di + f'/{eval_name}/{eval_name}.pkl')
if not os.path.exists(report_path):
continue
report = basics.load_pkl(report_path)
argscont = basics.load_pkl(os.path.join(set_path,
di + '/argscont.pkl'))
report.update(argscont)
reports[di] = report
basics.save2pkl(reports, set_path, name=eval_name)
return os.path.join(set_path, eval_name + '.pkl')
def load_prediction(self, name: str):
self._curr_obj = objects.load_obj(self._datatype,
f'{self._data_path}{name}.pkl')
if self._hybrid_mode and isinstance(self._curr_obj, CloudEnsemble):
self._curr_obj = self._curr_obj.hc
if self._label_remove is not None:
self._curr_obj.remove_nodes(self._label_remove)
self._curr_name = name
if os.path.exists(f'{self._save_path}{name}_preds.pkl'):
preds = basics.load_pkl(f'{self._save_path}{name}_preds.pkl')
self._curr_obj.set_predictions(preds)
def preds2hc(preds: str):
preds = os.path.expanduser(preds)
preds = basics.load_pkl(preds)
obj = objects.load_obj('ce', preds[0])
obj.set_predictions(preds[1])
obj.generate_pred_labels()
if isinstance(obj, CloudEnsemble):
hc = obj.hc
else:
hc = obj
_ = hc.pred_node_labels
return hc
def simple_view(self, name: str, seed: int = 4):
idx = 0
while idx < len(self.files1):
file = self.files1[idx]
slashs = [pos for pos, char in enumerate(file) if char == '/']
filename = file[slashs[-1] + 1:-4]
cloud = basics.load_pkl(file)
res = self.core_next(cloud, save_name=filename, seed=seed)
if res is None:
return
else:
idx += res
def load_obj(
data_type: str, file: str
) -> Union[HybridMesh, HybridCloud, PointCloud, CloudEnsemble]:
if data_type == 'obj':
return basics.load_pkl(file)
if data_type == 'ce':
return ensembles.ensemble_from_pkl(file)
if data_type == 'hc':
hc = HybridCloud()
return hc.load_from_pkl(file)
if data_type == 'hm':
hm = HybridMesh()
return hm.load_from_pkl(file)
else:
pc = PointCloud()
return pc.load_from_pkl(file)
def extract_data_from_reports(reports_path: str,
cell_key: str = 'total',
mode_key: str = 'mv',
class_key: str = 'macro avg',
metric_key: str = 'f1-score'):
"""
Extracts the requested metric from the reports file at reports_path and creates a datacontainer
which can then be transformed into a diagram.
reports structure:
1. directories (e.g. '2020_03_14_50_5000'),
2. cell predictions, total predictions and argument container keywords (e.g. 'total', 'sso_46313345_preds', 'sample_num', ...),
3. evaluation mode (e.g. 'mv' for majority vote results on vertex level or 'mv_skel' for results on node level),
4. classes and meta metric keys (e.g. 'dendrite', 'axon', 'accuracy', macro avg'),
5. actual score (e.g. for 'accuracy') or metric keys (e.g. 'precision', 'recall', 'f1-score'),
6. actual scores
Args:
reports_path: path to reports file.
cell_key: Choose between single cells (e.g. 'sso_491527_poisson') or choose 'total'
mode_key: Choose between mesh (e.g. 'mv') or skeleton (e.g. 'mv_skel') evaluation
class_key: Chosse between classes (e.g. 'dendrite', 'axon', ...) or averages (e.g. 'accuracy', 'macro avg', ...)
metric_key: Choose between metrics (e.g. 'precision', 'f1-score', ...)
Return:
Data container with requested metrics.
"""
reports = basics.load_pkl(reports_path)
epochs = []
scores = []
for key in list(reports.keys()):
report = reports[key]
reports.pop(key)
metric = report[cell_key][mode_key][class_key]
if class_key == 'accuracy':
score = metric
else:
score = metric[metric_key]
epoch_num = int(re.findall(r"epoch_(\d+)", key)[0])
epochs.append(epoch_num)
scores.append(score)
return epochs, scores
def load_cloudset(self, idx: int):
""" Gets executed when cloudset flag is set and visualizes the results from cloudset chunking performed by
the morphx.data.analyser save_cloudset method.
Args:
idx: Index at which file the viewing should start.
"""
while idx < len(self.files1):
file = self.files1[idx]
slashs = [pos for pos, char in enumerate(file) if char == '/']
filename = file[slashs[-1]:-4]
print("Viewing: " + filename)
with open(file, 'rb') as f:
content = pickle.load(f)
hybrid_idx = content[0]
hybrid_file = [
file for file in self.files2
if 'cloud_{}'.format(hybrid_idx) in file
]
hybrid = basics.load_pkl(hybrid_file[0])
local_bfs = content[1]
sample = content[2]
bfs_cloud = visualize.prepare_bfs(hybrid, local_bfs)
hybrid_bfs = clouds.merge_clouds([hybrid, bfs_cloud])
res = self.core_next(hybrid_bfs, sample,
'sample_h{}_i{}'.format(hybrid_idx, idx))
if res is None:
return
else:
idx += res
def visualize_prediction_set(input_path: str,
output_path: str,
gt_path: str,
random_seed: int = 4,
data_type: str = 'ce',
save_to_image: bool = True):
""" Saves images of all predicted files at input_path using the visualize_prediction method for each file. """
files = glob.glob(input_path + 'sso_*.pkl')
gt_files = glob.glob(gt_path + 'sso_*.pkl')
if not os.path.isdir(output_path):
os.makedirs(output_path)
# find corresponding gt file, map predictions to labels and save images
for file in tqdm(files):
for gt_file in gt_files:
slashs = [pos for pos, char in enumerate(gt_file) if char == '/']
name = gt_file[slashs[-1] + 1:-4]
if name in file:
obj = objects.load_obj(data_type, gt_file)
pred = basics.load_pkl(file)
obj.set_predictions(pred[1])
visualize_prediction(obj,
output_path + name + '.png',
random_seed=random_seed,
save_to_image=save_to_image)
def evaluate_cell_predictions(prediction: str,
total_evaluation: dict = None,
evaluation_mode: str = 'mv',
label_names: list = None,
skeleton_smoothing: bool = False,
remove_unpredicted: bool = True,
data_type: str = 'obj',
label_mapping: List[Tuple[int, int]] = None,
label_remove: List[int] = None) -> tuple:
"""
Can be used to evaluation single cell prediction. See `evaluate_prediction_set` for argument descriptions.
"""
reports = {}
reports_txt = ""
prediction = os.path.expanduser(prediction)
# --- load predictions into corresponding cell (predictions contain pointer to original cell file) ---
vertex_predictions = basics.load_pkl(prediction)
obj = objects.load_obj(data_type, vertex_predictions[0])
if label_remove is not None:
obj.remove_nodes(label_remove)
obj.set_predictions(vertex_predictions[1])
reports['pred_num'] = obj.pred_num
if label_mapping is not None:
obj.map_labels(label_mapping)
# --- reduce multiple predictions to single label by either:
# d: taking first prediction as label
# mv: taking majority vote on all predictions as new label
# mvs: taking majority vote on all predictions as label and apply smoothing ---
if evaluation_mode == 'd':
obj.generate_pred_labels(False)
elif evaluation_mode == 'mv':
obj.generate_pred_labels()
elif evaluation_mode == 'mvs':
obj.generate_pred_labels()
obj.hc.prediction_smoothing()
else:
raise ValueError(f"Mode {evaluation_mode} is not known.")
if isinstance(obj, CloudEnsemble):
hc = obj.hc
else:
hc = obj
if len(hc.pred_labels) != len(hc.labels):
raise ValueError(
"Length of predicted label array doesn't match with length of label array."
)
coverage = hc.get_coverage()
reports['cov'] = coverage
# --- vertex evaluation ---
vertex_labels = hc.labels
vertex_predictions = hc.pred_labels
if remove_unpredicted:
mask = np.logical_and(vertex_labels != -1, vertex_predictions != -1)
vertex_labels, vertex_predictions = vertex_labels[
mask], vertex_predictions[mask]
targets = get_target_names(vertex_labels, vertex_predictions, label_names)
reports[evaluation_mode] = sm.classification_report(vertex_labels,
vertex_predictions,
output_dict=True,
target_names=targets)
reports_txt += evaluation_mode + '\n\n' + \
f'Coverage: {coverage[1] - coverage[0]} of {coverage[1]}, ' \
f'{round((1 - coverage[0] / coverage[1]) * 100)} %\n\n' + \
f'Number of predictions: {obj.pred_num}\n\n' + \
sm.classification_report(vertex_labels, vertex_predictions, target_names=targets) + '\n\n'
cm = sm.confusion_matrix(vertex_labels, vertex_predictions)
reports_txt += write_confusion_matrix(cm, targets) + '\n\n'
# --- skeleton evaluation ---
evaluation_mode += '_skel'
if skeleton_smoothing:
hc.node_sliding_window_bfs(neighbor_num=20)
node_labels = hc.node_labels
node_predictions = hc.pred_node_labels
if remove_unpredicted:
mask = np.logical_and(node_labels != -1, node_predictions != -1)
node_labels, node_predictions = node_labels[mask], node_predictions[
mask]
targets = get_target_names(node_labels, node_predictions, label_names)
reports[evaluation_mode] = sm.classification_report(node_labels,
node_predictions,
output_dict=True,
target_names=targets)
reports_txt += evaluation_mode + '\n\n' + sm.classification_report(
node_labels, node_predictions, target_names=targets) + '\n\n'
cm = sm.confusion_matrix(node_labels, node_predictions)
reports_txt += write_confusion_matrix(cm, targets) + '\n\n'
# --- save generated labels for total evaluation ---
if total_evaluation is not None:
total_evaluation['vertex_predictions'] = np.append(
total_evaluation['vertex_predictions'], vertex_predictions)
total_evaluation['node_predictions'] = np.append(
total_evaluation['node_predictions'], node_predictions)
total_evaluation['vertex_labels'] = np.append(
total_evaluation['vertex_labels'], vertex_labels)
total_evaluation['node_labels'] = np.append(
total_evaluation['node_labels'], node_labels)
total_evaluation['coverage'][0] += coverage[0]
total_evaluation['coverage'][1] += coverage[1]
return reports, reports_txt