def main() -> None:
parser = argparse.ArgumentParser(
"Check whether trainings are complete and converged.")
parser.add_argument(
"type",
type=str,
choices=["completeness", "convergence"],
help="Pick whether to check convergence or just completeness.")
parser.add_argument("setup", type=str, help="Network setup to check.")
parser.add_argument(
"--tol_distance",
type=int,
default=40,
nargs="+",
help=
"Tolerance distance to check for with metrics using tolerance distance."
)
parser.add_argument(
"--clip_distance",
type=int,
default=200,
nargs="+",
help="Clip distance to check for which metrics using clip distance.")
parser.add_argument(
"--threshold",
type=int,
default=127,
help="Threshold to have been applied on top of raw predictions.")
parser.add_argument("--training_version",
type=str,
default="v0003.2",
help="Version of training")
parser.add_argument("--gt_version",
type=str,
default="v0003",
help="Version of groundtruth")
parser.add_argument("--check_private_db", action="store_true")
args = parser.parse_args()
db = cosem_db.MongoCosemDB(training_version=args.training_version,
gt_version=args.gt_version,
write_access=args.check_private_db)
metric_params = {
"tol_distance": args.tol_distance,
"clip_distance": args.clip_distance
}
if args.type == "completeness":
print(
check_completeness(db,
args.setup,
metric_params,
threshold=args.threshold))
else:
print(check_convergence(args.setup, args.threshold, db))
def query_score(cropno,
labelname,
threshold=127,
setup=None,
s1=False,
clip_distance=200,
tol_distance=40,
training_version="v0003.2",
gt_version="v0003"):
db = cosem_db.MongoCosemDB(training_version=training_version,
gt_version=gt_version)
c = db.get_crop_by_number(cropno)
labelname, setup, iteration, s1 = get_best_manual(
c["dataset_id"],
labelname,
setup=setup,
s1=s1,
training_version=db.training_version)
path = construct_pred_path(setup,
iteration,
c,
s1,
training_version=db.training_version)
threshold = threshold
metric_params = dict()
metric_params["clip_distance"] = clip_distance
metric_params["tol_distance"] = tol_distance
scores = dict()
for metric in EvaluationMetrics:
specific_params = filter_params(metric_params, metric)
query = {
"path": path,
"dataset": labelname,
"setup": setup,
"iteration": iteration,
"crop": str(cropno),
"threshold": threshold,
"metric": metric,
"metric_params": specific_params
}
doc = db.read_evaluation_result(query)
scores[metric.name] = doc["value"]
return scores
def main(alt_args=None):
parser = argparse.ArgumentParser("Evaluate predictions")
parser.add_argument("--setup", type=str, nargs='+', default=None,
help="network setup from which to evaluate a prediction, e.g. setup01")
parser.add_argument("--iteration", type=int, nargs='+', default=None,
help="network iteration from which to evaluate prediction, e.g. 725000")
parser.add_argument("--label", type=str, nargs='+', default=None,
help="label for which to evaluate prediction, choices: " + ", ".join(list(hierarchy.keys())))
parser.add_argument("--crop", type=int, nargs='+', default=None,
help="number of crop with annotated groundtruth, e.g. 110")
parser.add_argument("--threshold", type=int, default=127, nargs='+',
help="threshold to apply on distances")
parser.add_argument("--pred_path", type=str, default=None, nargs='+',
help="path of n5 file containing predictions")
parser.add_argument("--pred_ds", type=str, default=None, nargs='+',
help="dataset of the n5 file containing predictions")
parser.add_argument("--metric", type=str, default=None, help="metric to evaluate",
choices=list(em.value for em in EvaluationMetrics), nargs="+")
parser.add_argument("--clip_distance", type=int, default=200,
help="Parameter used for clipped false distances. False distances larger than the value of "
"this parameter are reduced to this value.")
parser.add_argument("--tol_distance", type=int, default=40,
help="Parameter used for counting false negatives/positives with a tolerance. Only false "
"predictions that are farther than this value from the closest pixel where they would be "
"correct are counted.")
parser.add_argument("--training_version", type=str, default="v0003.2", help="Version of training from which to "
"evaluate setup.")
parser.add_argument("--gt_version", type=str, default="v0003", help="Version of groundtruth to use for evaluation.")
parser.add_argument("--save", action='store_true',
help="save to database and csv file")
parser.add_argument("--overwrite", action='store_true',
help="overwrite existing entries in database and csv")
parser.add_argument("--s1", action='store_true', help="use s1 standard directory")
parser.add_argument("--refined", action='store_true', help="use refined predictions")
parser.add_argument("--dry-run", action='store_true',
help="show list of evaluations that would be run with given arguments without compute anything")
args = parser.parse_args(alt_args)
db = cosem_db.MongoCosemDB(write_access=True, training_version=args.training_version, gt_version=args.gt_version)
eval_results_csv_folder = os.path.join(config_loader.get_config()["organelles"]["evaluation_path"],
db.training_version, "evaluation_results")
csvhandler = cosem_db.CosemCSV(eval_results_csv_folder)
if args.overwrite and not args.save:
raise ValueError("Overwriting should only be set if save is also set")
if args.crop is None:
crops = db.get_all_validation_crops()
else:
crops = []
for cno in args.crop:
c = db.get_crop_by_number(cno)
if c is None:
raise ValueError("Did not find crop {0:} in database".format(cno))
crops.append(c)
if args.metric is None:
metric = list(em.value for em in EvaluationMetrics)
else:
metric = list(always_iterable(args.metric))
metric_params = dict()
metric_params['clip_distance'] = args.clip_distance
metric_params['tol_distance'] = args.tol_distance
if args.refined:
assert args.setup is None
assert args.iteration is None
else:
assert args.setup is not None
num_validations = max(len(list(always_iterable(args.setup))), len(list(always_iterable(args.iteration))),
len(list(always_iterable(args.label))), len(list(always_iterable(args.pred_path))),
len(list(always_iterable(args.pred_ds))), len(list(always_iterable(args.threshold))), 1)
iterator = itertools.product(zip(range(num_validations), repeat_last(always_iterable(args.setup)),
repeat_last(always_iterable(args.iteration)),
repeat_last(always_iterable(args.label)),
repeat_last(always_iterable(args.pred_path)),
repeat_last(always_iterable(args.pred_ds)),
repeat_last(always_iterable(args.threshold))), always_iterable(crops))
print("\nWill run the following validations:\n")
validations = []
for (valno, setup, iteration, label, pred_path, pred_ds, thr), crop in iterator:
if pred_ds is not None and label is None:
raise ValueError("If pred_ds is specified, label can't be autodetected")
if pred_path is None:
if iteration is None and not args.refined:
raise ValueError("Either pred_path or iteration must be specified")
if args.refined:
pred_path = construct_refined_path(crop)
else:
pred_path = construct_pred_path(setup, iteration, crop, args.s1, training_version=args.training_version)
if not os.path.exists(pred_path):
raise ValueError("{0:} not found".format(pred_path))
if not os.path.exists(os.path.join(pred_path, 'attributes.json')):
raise RuntimeError("N5 is incompatible with zarr due to missing attributes files. Consider running"
" `add_missing_n5_attributes {0:}`".format(pred_path))
if label is None:
labels = autodetect_labelnames(pred_path, crop)
else:
labels = list(always_iterable(label))
for ll in labels:
if ll not in crop_utils.get_all_annotated_labelnames(crop):
raise ValueError("Label {0:} not annotated in crop {1:}".format(ll, crop['number']))
for ll in labels:
if pred_ds is None:
ds = ll
else:
ds = pred_ds
if not os.path.exists(os.path.join(pred_path, ds)):
raise ValueError('{0:} not found'.format(os.path.join(pred_path, ds)))
if iteration is not None:
iter = autodetect_iteration(pred_path, ds)
if iter is not None:
if iteration != iter:
raise ValueError(
"You specified pred_path as well as iteration. The iteration does not match the "
"iteration in the attributes of the prediction."
)
else:
iter = iteration
else:
iter = autodetect_iteration(pred_path, ds)
if iter is None:
raise ValueError(
"Please sepcify iteration, it is not contained in the prediction metadata."
)
if setup is None:
this_setup = autodetect_setup(pred_path, ds)
if this_setup is None:
raise ValueError(
"Please specify setup, it is not contained in the prediction metadata."
)
else:
this_setup = autodetect_setup(pred_path, ds)
if this_setup is not None:
if this_setup != setup:
raise ValueError(
"The specified setup does not match the setup in the attributes of the prediction."
)
else:
this_setup = setup
if not args.refined and pred_path != construct_pred_path(this_setup, iter, crop, args.s1,
training_version=args.training_version):
warnings.warn(
"You specified pred_path as well as setup and the pred_path does not match the standard "
"location."
)
if args.refined and pred_path != construct_refined_path(crop):
warnings.warn(
"You specified pred_path does not match the standard location."
)
if not os.path.exists(os.path.join(pred_path, ds)):
raise ValueError('{0:} not found'.format(os.path.join(pred_path, ds)))
n5 = zarr.open(pred_path, mode="r")
raw_ds = n5[ds].attrs["raw_ds"]
parent_path = n5[ds].attrs["raw_data_path"]
parent_dataset_id = n5[ds].attrs["parent_dataset_id"]
validations.append([pred_path, ds, this_setup, iter, hierarchy[ll], crop, raw_ds, parent_path,
parent_dataset_id, thr])
tabs = [(pp, d, s, i, ll.labelname, c['number'], r_ds, parent, p_id, t, m, filter_params(metric_params, m)) for
(pp, d, s, i, ll, c, r_ds, parent, p_id, t), m in itertools.product(validations, metric)]
print(tabulate.tabulate(tabs, ["Path", "Dataset", "Setup", "Iteration", "Label", "Crop", "Raw Dataset",
"Parent Path", "Parent Id", "Threshold", "Metric", "Metric Params"]))
if not args.dry_run:
print("\nRunning Evaluations:")
for val_params in validations:
pp, d, s, i, ll, c , r_ds, parent, p_id, t = val_params
results = run_validation(pp, d, s, i, ll, c, t, metric, metric_params, db, csvhandler, args.save,
args.overwrite, args.refined, gt_version=args.gt_version)
val_params.append(results)
print("\nResults Summary:")
tabs = [(pp, d, s, i, ll.labelname, c['number'], r_ds, parent, p_id, t, m, filter_params(metric_params, m), v) for
(pp, d, s, i, ll, c, r_ds, parent, p_id, t, r) in validations for m, v in r.items()]
print(tabulate.tabulate(tabs, ["Path", "Dataset", "Setup", "Iteration", "Label", "Crop",
"Raw Dataset", "Parent Path", "Parent Id", "Threshold", "Metric",
"Metric Params", "Value"]))
def get_differences(cropno,
metrics,
domain="setup",
threshold=127,
clip_distance=200,
tol_distance=40,
training_version="v0003.2",
gt_version="v0003"):
if not (isinstance(metrics, tuple) or isinstance(metrics, list)):
metrics = [
metrics,
]
db = cosem_db.MongoCosemDB(training_version=training_version,
gt_version=gt_version)
c = db.get_crop_by_number(str(cropno))
metric_params = dict()
metric_params["clip_distance"] = clip_distance
metric_params["tol_distance"] = tol_distance
csv_folder = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
db.training_version, "manual")
if domain == "setup":
csv_file = os.path.join(csv_folder, c["dataset_id"] + "_setup.csv")
f = open(csv_file, "r")
fieldnames = ["labelname", "setup", "iteration", "s1"]
elif domain == "iteration":
csv_file = os.path.join(csv_folder, c["dataset_id"] + "_iteration.csv")
f = open(csv_file, "r")
fieldnames = ["setup", "labelname", "iteration", "s1"]
else:
raise ValueError("unknown domain")
reader = csv.DictReader(f, fieldnames)
all_manual = []
for row in reader:
manual_result = dict()
manual_result["setup"] = row["setup"]
manual_result["labelname"] = row["labelname"]
manual_result["iteration"] = int(row["iteration"])
manual_result["s1"] = bool(int(row["s1"]))
all_manual.append(manual_result)
f.close()
for manual_result in all_manual:
if domain == "setup":
query_setup = None
else:
query_setup = manual_result["setup"]
for best_by_metric in metrics + ["manual"]:
for eval_metric in metrics:
entry = "{eval_metric:}_by_{best_by_metric:}".format(
eval_metric=eval_metric, best_by_metric=best_by_metric)
if best_by_metric == "manual":
manual_result[entry] = query_score(
cropno,
manual_result["labelname"],
setup=query_setup,
s1=manual_result["s1"],
threshold=threshold,
clip_distance=clip_distance,
tol_distance=tol_distance,
training_version=training_version,
gt_version=gt_version)[eval_metric]
else:
specific_params = filter_params(metric_params,
best_by_metric)
best_result = get_best_automatic(
db,
cropno,
manual_result["labelname"],
best_by_metric,
specific_params,
query_setup,
threshold=threshold,
s1=manual_result["s1"])
specific_params = filter_params(metric_params, eval_metric)
query = {
"path": best_result["path"],
"dataset": best_result["dataset"],
"setup": best_result["setup"],
"iteration": best_result["iteration"],
"label": best_result["label"],
"crop": best_result["crop"],
"threshold": best_result["threshold"],
"metric": eval_metric,
"metric_params": specific_params
}
doc = db.read_evaluation_result(query)
manual_result[entry] = doc["value"]
return all_manual
"label": best_result["label"],
"crop": best_result["crop"],
"threshold": best_result["threshold"],
"metric": eval_metric,
"metric_params": specific_params
}
doc = db.read_evaluation_result(query)
manual_result[entry] = doc["value"]
return all_manual
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("label", type=str)
parser.add_argument("crop", type=int)
parser.add_argument("metric", type=str)
parser.add_argument("--setup", type=str, default=None)
parser.add_argument("--training_version", type=str, default="v0003.2")
parser.add_argument("--gt_version", type=str, default="v0003")
args = parser.parse_args()
db = cosem_db.MongoCosemDB(training_version=args.training_version,
gt_version=args.gt_version)
print(
get_best_automatic(db,
args.crop,
args.label,
args.metric, {},
setup=args.setup))
def main() -> None:
parser = argparse.ArgumentParser(
"Run a pre-defined comparison for evaluation results in the database.")
parser.add_argument("comparison",
type=str,
help="Type of comparison to run",
choices=[
"4nm-vs-8nm", "s1-vs-sub", "raw-vs-refined",
"all-vs-common-vs-single", "metrics",
"generalization", "best-4nm", "best-8nm"
])
parser.add_argument(
"--metric",
nargs="+",
type=str,
choices=list(em.value
for em in segmentation_metrics.EvaluationMetrics) +
["manual"],
help=
"Metric to use for evaluation. For metrics evaluation the first one is used for "
"comparison, the second one is the alternative metric by which to pick the best result"
)
parser.add_argument("--crops",
type=int,
nargs="*",
default=None,
help="Crops on which .")
parser.add_argument("--threshold",
type=int,
default=127,
help="threshold applied on distances for evaluation")
parser.add_argument("--clip_distance",
type=int,
default=200,
help="Parameter used for clipped false distances "
"for relevant metrics.")
parser.add_argument("--tol_distance",
type=int,
default=40,
help="Parameter used for tolerated false distances "
"for relevant metrics.")
parser.add_argument(
"--mode",
type=str,
choices=["across-setups", "per-setup", "all"],
help=
"Mode for some of the comparisons on whether to compare across setups ("
"`across-setups`) or only between equivalent setups (`per-setup`)",
default="across-setups")
parser.add_argument("--test",
action="store_true",
help="use cross validation for automatic evaluations")
parser.add_argument("--raw_ds",
type=str,
help="filter for raw dataset",
default="volumes/raw/s0")
parser.add_argument("--training_version",
type=str,
default="v0003.2",
help="Version of training")
parser.add_argument("--gt_version",
type=str,
default="v0003",
help="Version of groundtruth")
parser.add_argument(
"--save",
type=functools.partial(type_or_bool, type=str),
default="False",
help=
("True to save to a standardized csv file, or path to a custom csv file. False for not "
"saving."))
parser.add_argument("--check_private_db", action="store_true")
args = parser.parse_args()
if args.mode == "all" and args.comparison != "metrics":
raise ValueError("Mode all can only be used for metrics comparison.")
db = cosem_db.MongoCosemDB(training_version=args.training_version,
gt_version=args.gt_version,
write_access=args.check_private_db)
compare(args.comparison,
db,
args.metric,
crops=args.crops,
save=args.save,
tol_distance=args.tol_distance,
clip_distance=args.clip_distance,
threshold=args.threshold,
mode=args.mode,
raw_ds=args.raw_ds,
test=args.test)