def main():
logging.debug("Starting to run partner finding")
samples = ("A", "B", "C", "A+", "B+", "C+")
inp = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post/pre_and_post-v9.0/run01/evaluation/186000/data2016-aligned/cc.msg")
output = os.path.join(os.path.dirname(inp), "partners{0:}.msg")
de = "data2016-aligned"
thr = 127
cc_thr = 42
pre_thr = 42
post_thr = 35
dist_thr = 600
size_thr = 5
for s in samples:
logging.debug("Starting with sample {0:}".format(s))
filename = os.path.join(os.path.dirname(inp), s + ".h5")
syn_file = os.path.join(os.path.dirname(inp), s + ".n5")
cleft_cc_ds = "clefts_cropped_thr{0:}_cc{1:}".format(thr, cc_thr)
pre_ds = "pre_dist_cropped"
post_ds = "post_dist_cropped"
cleft_ds = "clefts_cropped"
seg_file = os.path.join(
config_loader.get_config()["synapses"]["cremieval_path"], de, s + ".n5"
)
seg_ds = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped"
if "unaligned" in de:
aligned = False
else:
aligned = True
off = tuple(np.array(offsets[s][aligned]) * np.array((40, 4, 4)))
mm = Matchmaker(
syn_file,
cleft_cc_ds,
cleft_ds,
pre_ds,
post_ds,
seg_file,
seg_ds,
filename,
offset=off,
safe_mem=True,
dist_thr=dist_thr,
size_thr=size_thr,
pre_thr=pre_thr,
post_thr=post_thr,
)
# mm.prepare_file()
mm.write_partners()
mm.cremi_file.close()
del mm
for s in samples:
done = (output.format(s)).open("w")
done.close()
def __init__(self,
uri: Optional[str] = None,
gt_version: str = "v0003",
training_version: str = "v0003.2",
write_access: bool = False):
if uri is None:
if write_access:
uri = config_loader.get_config(
)["organelles"]["database-private"]
else:
uri = config_loader.get_config(
)["organelles"]["database-public"]
super(MongoCosemDB, self).__init__(uri, gt_version, training_version,
write_access)
def get_unet_setup(setup: str,
training_version: str = "v0003.2") -> ModuleType:
"""
Load specific setup config.
Args:
setup: Setup to load.
training_version: version of trainings associated with the desired setup script.
Returns:
Imported setup config as module.
"""
setup_dirs = config_loader.get_config(
)["organelles"]["training_setups_paths"].split(",")
for setup_dir_root in setup_dirs:
setup_dir = os.path.join(setup_dir_root, training_version, setup)
if os.path.exists(setup_dir):
# setups can have names that are not python compatible (like include '.') so this trickery is necessary
spec = importlib.util.spec_from_file_location(
"unet_template", os.path.join(setup_dir, "unet_template.py"))
config = importlib.util.module_from_spec(spec)
spec.loader.exec_module(config)
return config
raise FileNotFoundError(
"Setup config not found for {setup:}".format(setup=setup))
def main_test_blocks():
samples = ["A+", "B+", "C+"]
filename_src = os.path.join(config_loader.get_config()["synapses"]["cremieval_path"], "{0:}/{1:}.n5")
data_eval = ["data2016-aligned", "data2016-unaligned"]
datasets_srcs = ["volumes/masks/groundtruth", "segmentation/multicut"]
dataset_tgts = [
"volumes/masks/groundtruth_cropped",
"volumes/labels/neuron_ids_constis_cropped",
]
for de in data_eval:
if "unaligned" in de:
aligned = False
else:
aligned = True
for sample in samples:
logging.info("cropping sample {0:}".format(sample))
off = offsets[sample][aligned]
sh = shapes[sample][aligned]
for ds_src, ds_tgt in zip(datasets_srcs, dataset_tgts):
logging.info(" dataset {0:}".format(ds_src))
crop_to_seg(
filename_src.format(de, sample),
ds_src,
filename_src.format(de, sample),
ds_tgt,
off,
sh,
)
def main():
data_sources = ["A", "B", "C"]
cremi_dir = config_loader.get_config()["synapses"]["cremi17_data_path"]
csv_files = [
os.path.join(cremi_dir, "cleft-partners_" + sample + "_2017.csv")
for sample in data_sources
]
hf = h5py.File(
os.path.join(cremi_dir, "sample_B_padded_20170424.aligned.0bg.hdf"), "r"
)
clefts = np.array(hf["volumes/labels/clefts"][50:150, 1400:2400, 1900:2900])
labels = np.array(hf["volumes/labels/neuron_ids"][50:150, 1400:2400, 1900:2900])
raw = np.array(hf["volumes/raw"][50:150, 1400:2400, 1900:2900])
pre_dict, post_dict = make_cleft_to_prepostsyn_neuron_id_dict(csv_files)
print("data loaded")
pre_dist, post_dist = create_prepost_dt(
clefts, labels, (40, 4, 4), pre_dict, post_dict
)
print("dist computed")
hf.close()
tar = h5py.File("righthere.hdf", "w")
tar.create_dataset("pre_dist", data=pre_dist)
tar.create_dataset("post_dist", data=post_dist)
tar.create_dataset("raw", data=raw)
tar.close()
def crop_main(it):
setups_path = os.path.join(
config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post")
samples = ["A+", "B+", "C+", "A", "B", "C"]
filename_src = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5")
dataset_srcs = [
"predictions_it{0:}/cleft_dist".format(it),
"predictions_it{0:}/pre_dist".format(it),
"predictions_it{0:}/post_dist".format(it),
]
filename_tgt = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5")
dataset_tgts = [
"predictions_it{0:}/cleft_dist_cropped".format(it),
"predictions_it{0:}/pre_dist_cropped".format(it),
"predictions_it{0:}/post_dist_cropped".format(it),
]
for sample in samples:
logging.info("cropping sample {0:}".format(sample))
off = offsets[sample]
sh = shapes[sample]
for ds_src, ds_tgt in zip(dataset_srcs, dataset_tgts):
logging.info(" dataset {0:}".format(ds_src))
crop.crop_to_seg(
filename_src.format(sample),
ds_src,
filename_tgt.format(sample),
ds_tgt,
off,
sh,
)
def _best_manual(
db: cosem_db.MongoCosemDB,
label: str,
setups: Sequence[str],
cropno: Union[int, str],
raw_ds: Optional[Sequence[str]] = None
) -> Optional[Dict[str, Union[str, int, bool]]]:
# read csv file containing results of manual evaluation, first for best iteration
c = db.get_crop_by_number(str(cropno))
csv_folder_manual = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
db.training_version, "manual")
csv_file_iterations = open(
os.path.join(csv_folder_manual, c["dataset_id"] + "_iteration.csv"),
"r")
fieldnames = ["setup", "labelname", "iteration", "raw_dataset"]
reader = csv.DictReader(csv_file_iterations, fieldnames)
# look for all possible matches with the given query
best_manuals = []
for row in reader:
if row["labelname"] == label and row["setup"] in setups:
if raw_ds is None or row["raw_dataset"] in raw_ds:
manual_result = {
"setup": row["setup"],
"label": row["labelname"],
"iteration": int(row["iteration"]),
"raw_dataset": row["raw_dataset"],
"crop": str(cropno),
"metric": "manual"
}
best_manuals.append(manual_result)
if len(
best_manuals
) == 0: # no manual evaluations with the given constraints were done
return None
elif len(best_manuals
) == 1: # if there's only one match it has to be the best one
return best_manuals[0]
else: # if there's several matches check the setup results for overall best
# read csv file containing results of manual evaluations, now for best setup per label/crop
csv_file_setups = open(
os.path.join(csv_folder_manual, c["dataset_id"] + "_setup.csv"),
"r")
reader = csv.DictReader(csv_file_setups, fieldnames)
for row in reader:
if row["labelname"] == label and row["setup"] in setups:
if raw_ds is None or row["raw_dataset"] in raw_ds:
manual_result_best = {
"setup": row["setup"],
"label": row["labelname"],
"iteration": int(row["iteration"]),
"raw_dataset": row["raw_dataset"],
"crop": str(cropno),
"metric": "manual",
"refined": False
}
return manual_result_best
return None
def construct_pred_path(setup, iteration, crop, s1, training_version="v0003.2"):
for tsp in config_loader.get_config()["organelles"]["training_setups_paths"].split(","):
setup_path = os.path.join(tsp, training_version, setup)
if os.path.exists(setup_path):
pred_path = os.path.join(setup_path, crop_utils.get_data_path(crop, s1).format(iteration))
return pred_path
raise FileNotFoundError("Have not found location for setup {0:}".format(setup))
def get_best_manual(dataset,
labelname,
setup=None,
s1=False,
training_version="v0003.2"):
csv_folder = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
training_version, "manual")
if setup is None:
csv_file = os.path.join(csv_folder, dataset + "_setup.csv")
f = open(csv_file, "r")
fieldnames = ["labelname", "setup", "iteration", "s1"]
else:
csv_file = os.path.join(csv_folder, dataset + "_iteration.csv")
f = open(csv_file, "r")
fieldnames = ["setup", "labelname", "iteration", "s1"]
reader = csv.DictReader(f, fieldnames)
for row in reader:
if setup is None or row["setup"] == setup:
if row["labelname"] == labelname:
if bool(int(row["s1"])) == s1:
result = row
break
iteration = int(result["iteration"])
setup = row["setup"]
f.close()
return labelname, setup, iteration, s1
def cc_main(it):
thrs_mult = [[127, 42]]
samples = ["A", "B", "C", "A+", "B+", "C+"]
setups_path = os.path.join(
config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post")
filename_src = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5")
dataset_srcs = [
"predictions_it{0:}".format(it) + "/cleft_dist_cropped_thr{0:}"
]
filename_tgt = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5")
dataset_tgts = [
"predictions_it{0:}".format(it) + "/cleft_dist_cropped_thr{0:}_cc{1:}"
]
for sample in samples:
logging.info(
"finding connected components for sample {0:}".format(sample))
for thrs in thrs_mult:
for ds_src, ds_tgt in zip(dataset_srcs, dataset_tgts):
logging.info(" dataset {0:}".format(ds_tgt.format(*thrs)))
connected_components_double_threshold.cc2(
filename_src.format(sample),
ds_src.format(thrs[0]),
ds_src.format(thrs[1]),
filename_tgt.format(sample),
ds_tgt.format(*thrs),
)
def read_gt(crop, label, gt_version="v0003", data_path=None):
if data_path is None:
data_path = config_loader.get_config()["organelles"]["data_path"]
n5file = zarr.open(os.path.join(data_path, crop["parent"]), mode="r")
blueprint_label_ds = "volumes/groundtruth/{version:}/crop{cropno:}/labels/{{label:}}"
label_ds = blueprint_label_ds.format(version=gt_version.lstrip("v"), cropno=crop["number"])
if label.separate_labelset:
label_ds_name = label.labelname
else:
label_ds_name = "all"
gt_seg = n5file[label_ds.format(label=label_ds_name)]
resolution = gt_seg.attrs["resolution"]
# blueprint_labelmask_ds = "volumes/groundtruth/{version:}/crop{cropno:}/masks/{{label:}}"
# labelmask_ds = blueprint_labelmask_ds.format(version=gt_version.lstrip("v"), cropno=crop["number"])
# labelmask_ds.format(label=label_ds_name)
# if labelmask_ds in n5file:
# mask = n5file[labelmask_ds]
# else:
# if label.generic_label is not None:
# specific_labels = list(set(label.labelid) - set(label.generic_label))
# generic_condition = (all(l in get_all_annotated_label_ids(crop) for l in label.generic_label) or
# all(l in get_all_annotated_label_ids(crop) for l in specific_labels))
# else:
# generic_condition = False
# if all(l in get_all_annotated_label_ids(crop) for l in label.labelid) or generic_condition:
# mask = ((gt_seg > 0) * 1).astype(np.bool)
# else:
# mask = ((gt_seg > 0) * 0).astype(np.bool)
return np.array(gt_seg), resolution
def main():
thrs_mult = [[127, 42]]
samples = ["A", "B", "C", "A+", "B+", "C+"]
setups_path = config_loader.get_config(
)["synapses"]["training_setups_path"]
filename_src = os.path.join(
setups_path, "pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5")
dataset_srcs = [
"predictions_it100000/cleft_dist_cropped",
"predictions_it100000/cleft_dist_cropped",
]
filename_tgt = os.path.join(
setups_path, "pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5")
dataset_tgts = [
"predictions_it100000/cleft_dist_cropped_thr{0:}",
"predictions_it100000/cleft_dist_cropped_thr{0:}",
]
for sample in samples:
logging.info("thresholding sample {0:}".format(sample))
for thrs in thrs_mult:
for ds_src, ds_tgt, thr in zip(dataset_srcs, dataset_tgts, thrs):
logging.info(" dataset {0:} at {1:}".format(ds_src, thr))
threshold(
filename_src.format(sample),
ds_src,
filename_tgt.format(sample),
ds_tgt.format(thr),
thr,
)
def main():
samples = ["A", "B", "C"]
filename_src = os.path.join(config_loader.get_config()["synapses"]["cremieval_path"], "{0:}/{1:}.n5")
data_eval = [
"data2016-aligned",
"data2016-unaligned",
"data2017-aligned",
"data2017-unaligned",
]
dataset_srcs = ["volumes/masks/groundtruth"]
# filename_tgt = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5'
dataset_tgts = ["volumes/masks/groundtruth_cropped"]
for de in data_eval:
if "unaligned" in de:
aligned = False
else:
aligned = True
for sample in samples:
logging.info("cropping sample {0:}".format(sample))
off = offsets[sample][aligned]
sh = shapes[sample][aligned]
for ds_src, ds_tgt in zip(dataset_srcs, dataset_tgts):
logging.info(" dataset {0:}".format(ds_src))
crop_to_seg(
filename_src.format(de, sample),
ds_src,
filename_src.format(de, sample),
ds_tgt,
off,
sh,
)
def main(s, mode=0, data=None):
# samples = ['A','B', 'C']
samples = [(s.split("/")[-1]).split("_")[0]]
for sample in samples:
logging.info(
"evaluating synapse predictions for sample {0:}".format(sample))
truth_fn = (os.path.join(
config_loader.get_config()["synapses"]["cremi17_data_path"],
"sample_{0:}_padded_20170424.aligned.hdf".format(sample)))
if data is not None:
logging.info("sample {0:} in mode {1:} using {2:}".format(
sample, mode, data))
if (data == "val" or data == "validation" or data == "VAL"
or data == "VALIDATION"):
assert s.endswith(".hdf")
test = CremiFile(s.replace(".hdf", ".validation.hdf"), "a")
truth = CremiFile(truth_fn.replace(".hdf", ".validation.hdf"), "a")
elif (data == "train" or data == "training" or data == "TRAIN"
or data == "TRAINING"):
assert s.endswith(".hdf")
test = CremiFile(s.replace(".hdf", ".training.hdf"), "a")
truth = CremiFile(truth_fn.replace(".hdf", ".training.hdf"), "a")
else:
test = CremiFile(s, "a")
truth = CremiFile(truth_fn, "a")
if mode == 0:
evaluate(test, truth)
elif mode == 1:
evaluate_multrecgt(test, truth, add_in_file=True)
elif mode == 2:
evaluate_multrecgt(test, truth)
def main_seg():
samples = ["A", "B", "C"] # ['A', 'C', 'B+', 'C+']
setups_path = os.path.join(
config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post")
filename_src = os.path.join(setups_path, "cremi/{0:}.n5")
dataset_src = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped"
filename_mask = os.path.join(setups_path, "cremi/{0:}.n5")
dataset_mask = "volumes/labels/neuron_ids_gt_cropped"
filename_tgt = os.path.join(setups_path, "cremi/{0:}.n5")
dataset_tgt = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped_masked"
for sample in samples:
print(sample)
off = offsets[sample]
sh = shapes[sample]
set_mask_to_zero(
filename_src.format(sample),
dataset_src,
filename_mask.format(sample),
dataset_mask,
filename_tgt.format(sample),
dataset_tgt,
off,
sh,
)
def _get_iteration_queries(cropno: Sequence[Union[int, str]],
db: cosem_db.MongoCosemDB) -> List[Dict[str, str]]:
csv_folder_manual = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
db.training_version, "manual")
csv_result_files = _get_csv_files(csv_folder_manual, "iteration", cropno,
db)
iteration_queries = []
for csv_f in csv_result_files:
f = open(os.path.join(csv_folder_manual, csv_f), "r")
fieldnames = ["setup", "labelname", "iteration", "raw_dataset"]
cell_id = re.split("_(setup|iteration).csv", csv_f)[0]
crop = db.get_validation_crop_by_cell_id(cell_id)
reader = csv.DictReader(f, fieldnames)
for row in reader:
if any(lbl in get_label_ids_by_category(crop, "present_annotated")
for lbl in hierarchy[row["labelname"]].labelid):
query = {
"label": row["labelname"],
"raw_dataset": row["raw_dataset"],
"setups": [row["setup"]],
"crop": crop["number"]
}
iteration_queries.append(query)
return iteration_queries
def run():
filepath = os.path.join(
config_loader.get_config()["synapses"]["training_setups_path"],
"cremi_all/cremi_all_0116_01/prediction_cremi_warped_sampleC+_200000.n5"
)
dataset = "syncleft_dist"
thr = 0.0
dataset_tgt = "syncleft_dist_thr{0:}".format(thr)
threshold(filepath, dataset, filepath, dataset_tgt, thr)
def run(self):
logging.debug("Starting to run partner finding")
progress = 0.0
self.set_progress_percentage(progress)
thr = 127
cc_thr = 42
pre_thr = 42
post_thr = 35
dist_thr = 600
size_thr = 5
for s in self.samples:
logging.debug("Starting with sample {0:}".format(s))
filename = os.path.join(os.path.dirname(self.input().fn), s + ".h5")
syn_file = os.path.join(os.path.dirname(self.input().fn), s + ".n5")
cleft_cc_ds = "clefts_cropped_thr{0:}_cc{1:}".format(thr, cc_thr)
pre_ds = "pre_dist_cropped"
post_ds = "post_dist_cropped"
cleft_ds = "clefts_cropped"
seg_file = os.path.join(
config_loader.get_config()["synapses"]["cremieval_path"],
self.de,
s + ".n5",
)
seg_ds = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped"
if "unaligned" in self.de:
aligned = False
else:
aligned = True
off = tuple(np.array(offsets[s][aligned]) * np.array((40, 4, 4)))
mm = Matchmaker(
syn_file,
cleft_cc_ds,
cleft_ds,
pre_ds,
post_ds,
seg_file,
seg_ds,
filename,
offset=off,
safe_mem=True,
dist_thr=dist_thr,
size_thr=size_thr,
pre_thr=pre_thr,
post_thr=post_thr,
)
# mm.prepare_file()
mm.write_partners()
mm.cremi_file.close()
del mm
progress += 100.0 / len(self.samples)
try:
self.set_progress_percentage(progress)
except:
pass
for o in self.output():
done = o.open("w")
done.close()
def get_offset_and_shape_from_crop(crop, gt_version="v0003"):
n5file = zarr.open(os.path.join(
config_loader.get_config()["organelles"]["data_path"], crop["parent"]),
mode="r")
label_ds = "volumes/groundtruth/{version:}/crop{cropno:}/labels/all".format(
version=gt_version.lstrip("v"), cropno=crop["number"])
offset_wc = n5file[label_ds].attrs["offset"][::-1]
offset = tuple(np.array(offset_wc) / 4.)
shape = tuple(np.array(n5file[label_ds].shape) / 2.)
return offset, shape
def main_seg():
samples = ["A", "B", "C", "A+", "B+", "C+"] # ['A', 'C', 'B+', 'C+']
filename_src = os.path.join(config_loader.get_config()["synapses"]["cremi17_data_path"],
"sample_{0:}_padded_20170424.aligned.0bg.n5")
dataset_src = "volumes/labels/neuron_ids"
filename_tgt = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post/cremi/{0:}.n5")
dataset_tgt = "volumes/labels/neuron_ids_gt_cropped"
for sample in samples:
print(sample)
off = offsets[sample]
sh = shapes[sample]
crop_to_seg(
filename_src.format(sample),
dataset_src,
filename_tgt.format(sample),
dataset_tgt,
off,
sh,
)
def transfer(training_version="v0003.2"):
db = MongoCosemDB(training_version=training_version)
eval_col = db.access("evaluation", db.training_version)
eval_results_csv_folder = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
training_version, "evaluation_results")
csv_d = CosemCSV(eval_results_csv_folder)
for l in hierarchy.keys():
csv_d.erase(l)
for db_entry in eval_col.find():
csv_d.write_evaluation_result(db_entry)
class SingleEvaluation(luigi.WrapperTask):
iteration = luigi.IntParameter(default=186000)
path = luigi.Parameter(default=os.path.join(config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post/pre_and_post-v9.0/run01/"))
data_eval = luigi.TupleParameter(default=("data2016-aligned", "data2016-unaligned"))
samples = luigi.TupleParameter(default=("A", "B", "C", "A+", "B+", "C+"))
def requires(self):
for de in self.data_eval:
if "A+" in self.samples or "B+" in self.samples or "C+" in self.samples:
test_samples = []
for s in self.samples:
if "+" in s:
test_samples.append(s)
test_samples = tuple(test_samples)
yield SplitModi(
self.iteration,
self.path,
de,
"groundtruth",
test_samples,
self.data_eval,
)
if "A" in self.samples or "B" in self.samples or "C" in self.samples:
training_samples = []
for s in self.samples:
if not ("+" in s):
training_samples.append(s)
training_samples = tuple(training_samples)
yield PartnerReport(
self.iteration,
self.path,
de,
"groundtruth",
training_samples,
self.data_eval,
)
yield PartnerReport(
self.iteration,
self.path,
de,
"validation",
training_samples,
self.data_eval,
)
yield PartnerReport(
self.iteration,
self.path,
de,
"training",
training_samples,
self.data_eval,
)
def get_refined_comparisons(
db: cosem_db.MongoCosemDB,
cropno: Union[None, str, int, Sequence[Union[str, int]]] = None
) -> List[Dict[str, Any]]:
"""
Get list of queries for predictions that have been refined (as read from csv file)
Args:
db: Database with crop information.
cropno: Specific crop number or list of crop numbers that should be included in queries.
Returns:
List of queries for which refined predictions exist.
"""
csv_folder_refined = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
db.training_version, "refined")
# get list of csv files relevant for crops
if cropno is None:
csv_result_files = os.listdir(csv_folder_refined)
else:
if isinstance(cropno, str) or isinstance(cropno, int):
cropno = [cropno]
csv_result_files = []
for cno in cropno:
crop = db.get_crop_by_number(cno)
csv_result_files.append(
os.path.join(csv_folder_refined,
crop["dataset_id"] + "_setup.csv"))
# collect entries from those csv files
queries = []
for csv_f in csv_result_files:
f = open(os.path.join(csv_folder_refined, csv_f), "r")
fieldnames = ["setup", "labelname", "iteration", "raw_dataset"]
cell_id = re.split("_setup.csv", csv_f)[0]
crop = db.get_validation_crop_by_cell_id(cell_id)
reader = csv.DictReader(f, fieldnames)
for row in reader:
# only consider results that we can evaluate automatically (actually contained in the crop)
if any(lbl in get_label_ids_by_category(crop, "present_annotated")
for lbl in hierarchy[row["labelname"]].labelid):
query = {
"label": row["labelname"],
"raw_dataset": row["raw_dataset"],
"setup": row["setup"],
"crop": crop["number"],
"iteration": int(row["iteration"])
}
queries.append(query)
return queries
def output(self):
base = os.path.join(
config_loader.get_config()["synapses"]["training_setups_path"],
"data_and_augmentations",
self.dt,
self.aug,
"unet_checkpoint_" + str(self.it),
)
return [
luigi.LocalTarget(base + ".data-00000-of-00001"),
luigi.LocalTarget(base + ".index"),
luigi.LocalTarget(base + ".meta"),
]
def main():
thrs_mult = [[153, 76, 76], [127, 63, 63]]
samples = ["B"] # ['A+', 'B+', 'C+', 'A', 'B', 'C']
filename_src = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5"
)
dataset_srcs = ["predictions_it400000/cleft_dist_cropped_thr{0:}"]
filename_tgt = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"],
"pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5"
)
dataset_tgts = ["predictions_it400000/cleft_dist_cropped_thr{0:}_cc"]
for sample in samples:
logging.info("finding connected components for sample {0:}".format(sample))
for thrs in thrs_mult:
for ds_src, ds_tgt, thr in zip(dataset_srcs, dataset_tgts, thrs):
logging.info(" dataset {0:}".format(ds_src.format(thr)))
cc(
filename_src.format(sample),
ds_src.format(thr),
filename_tgt.format(sample),
ds_tgt.format(thr),
)
def main():
thrs = [127, 63, 63]
samples = ["A+", "B+", "C+"]
# samples = ['B+', 'C+']
slf = 1
offsets = {
"A+": (37 * 40, 1176 * 4, 955 * 4),
"B+": (37 * 40, 1076 * 4, 1284 * 4),
"C+": (37 * 40, 1002 * 4, 1165 * 4),
"A": (38 * 40, 942 * 4, 951 * 4),
"B": (37 * 40, 1165 * 4, 1446 * 4),
"C": (37 * 40, 1032 * 4, 1045 * 4),
}
# segf_name = {'A+': 'sample_A+_85_aff_0.8_cf_hq_dq_dm1_mf0.81',
# 'B+': 'sample_B+_median_aff_0.8_cf_hq_dq_dm1_mf0.87',
# 'C+': 'sample_C+_85_aff_0.8_cf_hq_dq_dm1_mf0.75',
# }
# filename_src = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/{0:}.n5'
# dataset_src = 'main'
# filename_tgt = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/{0:}_sizefiltered750.n5'
# dataset_tgt = 'main'
# dat_file = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/pre_and_post-v3.0/cremi/{0:}_sizefilter750.dat'
# filename_src = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/sample{0}.n5'
# dataset_src = 'segmentations/multicut'
# dataset_src = 'segmentations/mc_glia_global2'
# filename_src = '/groups/saalfeld/saalfeldlab/larissa/data/cremi-2017/sample_{0:}_padded_20170424.aligned.0bg.n5'
# dataset_src = 'volumes/labels/neuron_ids'
# filename_src = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_new/sample{0:}.n5'
# dataset_src = 'segmentation/multicut'
# filename_tgt = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/cremi/{0}.n5'
filename = os.path.join(config_loader.get_config()["synapses"]["cremieval_path"],"data2016-aligned/{0:}.n5")
dataset_src = "volumes/labels/neuron_ids_constis_cropped"
dataset_tgt = "volumes/labels/neuron_ids_constis_slf{0:}_cropped"
# dat_file = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/pre_and_post-v3.0/cremi/{0:}_constis_slicefilter{' \
# '1:}.dat'
for sample in samples:
print(sample)
slicefilter(
filename.format(sample),
dataset_src,
filename.format(sample),
dataset_tgt.format(slf),
slf,
) # , dat_file.format(sample, slf))
def load_result(data_train, augmentation, data_eval, iteration, mode):
result_json = os.path.join(
config_loader.get_config()["synapses"]["training_setups_path"],
"data_and_augmentations",
data_train,
augmentation,
"evaluation",
str(iteration),
data_eval,
"partners.{0:}.json".format(mode),
)
try:
with open(result_json, "r") as f:
resdict = json.load(f)
except IOError:
return None
return resdict
def _get_setup_queries(
cropno: Sequence[Union[int, str]], db: cosem_db.MongoCosemDB
) -> List[Dict[str, Union[str, Sequence[str]]]]:
csv_folder_manual = os.path.join(
config_loader.get_config()["organelles"]["evaluation_path"],
db.training_version, "manual")
csv_result_files = _get_csv_files(csv_folder_manual, "setup", cropno, db)
setup_queries = []
for csv_f in csv_result_files:
f = open(os.path.join(csv_folder_manual, csv_f), "r")
fieldnames = ["setup", "labelname", "iteration", "raw_dataset"]
cell_id = re.split("_(setup|iteration).csv", csv_f)[0]
crop = db.get_validation_crop_by_cell_id(cell_id)
reader = csv.DictReader(f, fieldnames)
for row in reader:
if any(lbl in get_label_ids_by_category(crop, "present_annotated")
for lbl in hierarchy[row["labelname"]].labelid):
# find the csv files with the list of setups compared for each label (4nm or 8nm)
if row["raw_dataset"] == "volumes/raw/s0":
ff = open(
os.path.join(csv_folder_manual,
"compared_4nm_setups.csv"), "r")
elif row["raw_dataset"] == "volumes/subsampled/raw/0" or row[
"raw_dataset"] == "volumes/raw/s1":
ff = open(
os.path.join(csv_folder_manual,
"compared_8nm_setups.csv"), "r")
else:
raise ValueError("The raw_dataset {0:} ".format(
row["raw_dataset"]))
# get that list of compared setups from the csv file
compare_reader = csv.reader(ff)
for compare_row in compare_reader:
if compare_row[0] == row["labelname"]:
setups = compare_row[1:]
break
# collect result
query = {
"label": row["labelname"],
"raw_dataset": row["raw_dataset"],
"setups": setups,
"crop": crop["number"]
}
setup_queries.append(query)
return setup_queries
def run(self):
progress = 0.0
self.set_progress_percentage(progress)
for s in self.samples:
print(s)
filename = os.path.join(os.path.dirname(self.input()[0].fn),
s + ".h5")
mask_filename = os.path.join(
config_loader.get_config()["synapses"]["cremieval_path"],
self.de,
s + ".n5",
)
mask_dataset = "volumes/masks/" + self.m
filename_tgt = filename.replace("h5", self.m + ".h5")
# shutil.copy(filename, filename_tgt)
f = CremiFile(filename, "a")
g = CremiFile(filename_tgt, "a")
maskf = zarr.open(mask_filename, mode="r")
mask = maskf[mask_dataset]
off = mask.attrs["offset"]
res = mask.attrs["resolution"]
mask = np.array(mask[:])
ann = f.read_annotations()
shift = sub(ann.offset, off)
ids = ann.ids()
rmids = []
for i in ids:
t, loc = ann.get_annotation(i)
vx_idx = (np.array(add(loc, shift)) / res).astype(np.int)
if not mask[tuple(vx_idx)]:
rmids.append(i)
print(rmids)
for i in rmids:
print("removing {0:}".format(i))
ann.remove_annotation(i)
print(ann.comments.keys())
print(ann.pre_post_partners)
g.write_annotations(ann)
progress += 100.0 / len(self.samples)
try:
self.set_progress_percentage(progress)
except:
pass
for o in self.output():
done = o.open("w")
done.close()
def run(self):
progress = 0.0
self.set_progress_percentage(progress)
results = dict()
for s in self.samples:
thr = 127
testfile = os.path.join(os.path.dirname(self.input().fn),
s + ".n5")
truthfile = os.path.join(
config_loader.get_config()["synapses"]["cremieval_path"],
self.de,
s + ".n5",
)
test = np.array(
zarr.open(testfile,
mode="r")["clefts_cropped_thr" + str(thr)][:])
truth = np.array(
zarr.open(truthfile,
mode="r")["volumes/labels/clefts_cropped"][:])
mask = np.array(
zarr.open(truthfile,
mode="r")["volumes/masks/" + self.m + "_cropped"][:])
clefts_evaluation = Clefts(test, truth, np.logical_not(mask))
results[s] = dict()
results[s][
"false negatives count"] = clefts_evaluation.count_false_negatives(
)
results[s][
"false positives count"] = clefts_evaluation.count_false_positives(
)
results[s][
"false negative distance"] = clefts_evaluation.acc_false_negatives(
)
results[s][
"false positive distance"] = clefts_evaluation.acc_false_positives(
)
progress += 100.0 / len(self.samples)
try:
self.set_progress_percentage(progress)
except:
pass
with self.output().open("w") as done:
json.dump(results, done)
