def main(): logging.debug("Starting to run partner finding") samples = ("A", "B", "C", "A+", "B+", "C+") inp = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post/pre_and_post-v9.0/run01/evaluation/186000/data2016-aligned/cc.msg") output = os.path.join(os.path.dirname(inp), "partners{0:}.msg") de = "data2016-aligned" thr = 127 cc_thr = 42 pre_thr = 42 post_thr = 35 dist_thr = 600 size_thr = 5 for s in samples: logging.debug("Starting with sample {0:}".format(s)) filename = os.path.join(os.path.dirname(inp), s + ".h5") syn_file = os.path.join(os.path.dirname(inp), s + ".n5") cleft_cc_ds = "clefts_cropped_thr{0:}_cc{1:}".format(thr, cc_thr) pre_ds = "pre_dist_cropped" post_ds = "post_dist_cropped" cleft_ds = "clefts_cropped" seg_file = os.path.join( config_loader.get_config()["synapses"]["cremieval_path"], de, s + ".n5" ) seg_ds = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped" if "unaligned" in de: aligned = False else: aligned = True off = tuple(np.array(offsets[s][aligned]) * np.array((40, 4, 4))) mm = Matchmaker( syn_file, cleft_cc_ds, cleft_ds, pre_ds, post_ds, seg_file, seg_ds, filename, offset=off, safe_mem=True, dist_thr=dist_thr, size_thr=size_thr, pre_thr=pre_thr, post_thr=post_thr, ) # mm.prepare_file() mm.write_partners() mm.cremi_file.close() del mm for s in samples: done = (output.format(s)).open("w") done.close()
def __init__(self, uri: Optional[str] = None, gt_version: str = "v0003", training_version: str = "v0003.2", write_access: bool = False): if uri is None: if write_access: uri = config_loader.get_config( )["organelles"]["database-private"] else: uri = config_loader.get_config( )["organelles"]["database-public"] super(MongoCosemDB, self).__init__(uri, gt_version, training_version, write_access)
def get_unet_setup(setup: str, training_version: str = "v0003.2") -> ModuleType: """ Load specific setup config. Args: setup: Setup to load. training_version: version of trainings associated with the desired setup script. Returns: Imported setup config as module. """ setup_dirs = config_loader.get_config( )["organelles"]["training_setups_paths"].split(",") for setup_dir_root in setup_dirs: setup_dir = os.path.join(setup_dir_root, training_version, setup) if os.path.exists(setup_dir): # setups can have names that are not python compatible (like include '.') so this trickery is necessary spec = importlib.util.spec_from_file_location( "unet_template", os.path.join(setup_dir, "unet_template.py")) config = importlib.util.module_from_spec(spec) spec.loader.exec_module(config) return config raise FileNotFoundError( "Setup config not found for {setup:}".format(setup=setup))
def main_test_blocks(): samples = ["A+", "B+", "C+"] filename_src = os.path.join(config_loader.get_config()["synapses"]["cremieval_path"], "{0:}/{1:}.n5") data_eval = ["data2016-aligned", "data2016-unaligned"] datasets_srcs = ["volumes/masks/groundtruth", "segmentation/multicut"] dataset_tgts = [ "volumes/masks/groundtruth_cropped", "volumes/labels/neuron_ids_constis_cropped", ] for de in data_eval: if "unaligned" in de: aligned = False else: aligned = True for sample in samples: logging.info("cropping sample {0:}".format(sample)) off = offsets[sample][aligned] sh = shapes[sample][aligned] for ds_src, ds_tgt in zip(datasets_srcs, dataset_tgts): logging.info(" dataset {0:}".format(ds_src)) crop_to_seg( filename_src.format(de, sample), ds_src, filename_src.format(de, sample), ds_tgt, off, sh, )
def main(): data_sources = ["A", "B", "C"] cremi_dir = config_loader.get_config()["synapses"]["cremi17_data_path"] csv_files = [ os.path.join(cremi_dir, "cleft-partners_" + sample + "_2017.csv") for sample in data_sources ] hf = h5py.File( os.path.join(cremi_dir, "sample_B_padded_20170424.aligned.0bg.hdf"), "r" ) clefts = np.array(hf["volumes/labels/clefts"][50:150, 1400:2400, 1900:2900]) labels = np.array(hf["volumes/labels/neuron_ids"][50:150, 1400:2400, 1900:2900]) raw = np.array(hf["volumes/raw"][50:150, 1400:2400, 1900:2900]) pre_dict, post_dict = make_cleft_to_prepostsyn_neuron_id_dict(csv_files) print("data loaded") pre_dist, post_dist = create_prepost_dt( clefts, labels, (40, 4, 4), pre_dict, post_dict ) print("dist computed") hf.close() tar = h5py.File("righthere.hdf", "w") tar.create_dataset("pre_dist", data=pre_dist) tar.create_dataset("post_dist", data=post_dist) tar.create_dataset("raw", data=raw) tar.close()
def crop_main(it): setups_path = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post") samples = ["A+", "B+", "C+", "A", "B", "C"] filename_src = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5") dataset_srcs = [ "predictions_it{0:}/cleft_dist".format(it), "predictions_it{0:}/pre_dist".format(it), "predictions_it{0:}/post_dist".format(it), ] filename_tgt = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5") dataset_tgts = [ "predictions_it{0:}/cleft_dist_cropped".format(it), "predictions_it{0:}/pre_dist_cropped".format(it), "predictions_it{0:}/post_dist_cropped".format(it), ] for sample in samples: logging.info("cropping sample {0:}".format(sample)) off = offsets[sample] sh = shapes[sample] for ds_src, ds_tgt in zip(dataset_srcs, dataset_tgts): logging.info(" dataset {0:}".format(ds_src)) crop.crop_to_seg( filename_src.format(sample), ds_src, filename_tgt.format(sample), ds_tgt, off, sh, )
def _best_manual( db: cosem_db.MongoCosemDB, label: str, setups: Sequence[str], cropno: Union[int, str], raw_ds: Optional[Sequence[str]] = None ) -> Optional[Dict[str, Union[str, int, bool]]]: # read csv file containing results of manual evaluation, first for best iteration c = db.get_crop_by_number(str(cropno)) csv_folder_manual = os.path.join( config_loader.get_config()["organelles"]["evaluation_path"], db.training_version, "manual") csv_file_iterations = open( os.path.join(csv_folder_manual, c["dataset_id"] + "_iteration.csv"), "r") fieldnames = ["setup", "labelname", "iteration", "raw_dataset"] reader = csv.DictReader(csv_file_iterations, fieldnames) # look for all possible matches with the given query best_manuals = [] for row in reader: if row["labelname"] == label and row["setup"] in setups: if raw_ds is None or row["raw_dataset"] in raw_ds: manual_result = { "setup": row["setup"], "label": row["labelname"], "iteration": int(row["iteration"]), "raw_dataset": row["raw_dataset"], "crop": str(cropno), "metric": "manual" } best_manuals.append(manual_result) if len( best_manuals ) == 0: # no manual evaluations with the given constraints were done return None elif len(best_manuals ) == 1: # if there's only one match it has to be the best one return best_manuals[0] else: # if there's several matches check the setup results for overall best # read csv file containing results of manual evaluations, now for best setup per label/crop csv_file_setups = open( os.path.join(csv_folder_manual, c["dataset_id"] + "_setup.csv"), "r") reader = csv.DictReader(csv_file_setups, fieldnames) for row in reader: if row["labelname"] == label and row["setup"] in setups: if raw_ds is None or row["raw_dataset"] in raw_ds: manual_result_best = { "setup": row["setup"], "label": row["labelname"], "iteration": int(row["iteration"]), "raw_dataset": row["raw_dataset"], "crop": str(cropno), "metric": "manual", "refined": False } return manual_result_best return None
def construct_pred_path(setup, iteration, crop, s1, training_version="v0003.2"): for tsp in config_loader.get_config()["organelles"]["training_setups_paths"].split(","): setup_path = os.path.join(tsp, training_version, setup) if os.path.exists(setup_path): pred_path = os.path.join(setup_path, crop_utils.get_data_path(crop, s1).format(iteration)) return pred_path raise FileNotFoundError("Have not found location for setup {0:}".format(setup))
def get_best_manual(dataset, labelname, setup=None, s1=False, training_version="v0003.2"): csv_folder = os.path.join( config_loader.get_config()["organelles"]["evaluation_path"], training_version, "manual") if setup is None: csv_file = os.path.join(csv_folder, dataset + "_setup.csv") f = open(csv_file, "r") fieldnames = ["labelname", "setup", "iteration", "s1"] else: csv_file = os.path.join(csv_folder, dataset + "_iteration.csv") f = open(csv_file, "r") fieldnames = ["setup", "labelname", "iteration", "s1"] reader = csv.DictReader(f, fieldnames) for row in reader: if setup is None or row["setup"] == setup: if row["labelname"] == labelname: if bool(int(row["s1"])) == s1: result = row break iteration = int(result["iteration"]) setup = row["setup"] f.close() return labelname, setup, iteration, s1
def cc_main(it): thrs_mult = [[127, 42]] samples = ["A", "B", "C", "A+", "B+", "C+"] setups_path = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post") filename_src = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5") dataset_srcs = [ "predictions_it{0:}".format(it) + "/cleft_dist_cropped_thr{0:}" ] filename_tgt = os.path.join(setups_path, "pre_and_post-v6.3/cremi/{0:}.n5") dataset_tgts = [ "predictions_it{0:}".format(it) + "/cleft_dist_cropped_thr{0:}_cc{1:}" ] for sample in samples: logging.info( "finding connected components for sample {0:}".format(sample)) for thrs in thrs_mult: for ds_src, ds_tgt in zip(dataset_srcs, dataset_tgts): logging.info(" dataset {0:}".format(ds_tgt.format(*thrs))) connected_components_double_threshold.cc2( filename_src.format(sample), ds_src.format(thrs[0]), ds_src.format(thrs[1]), filename_tgt.format(sample), ds_tgt.format(*thrs), )
def read_gt(crop, label, gt_version="v0003", data_path=None): if data_path is None: data_path = config_loader.get_config()["organelles"]["data_path"] n5file = zarr.open(os.path.join(data_path, crop["parent"]), mode="r") blueprint_label_ds = "volumes/groundtruth/{version:}/crop{cropno:}/labels/{{label:}}" label_ds = blueprint_label_ds.format(version=gt_version.lstrip("v"), cropno=crop["number"]) if label.separate_labelset: label_ds_name = label.labelname else: label_ds_name = "all" gt_seg = n5file[label_ds.format(label=label_ds_name)] resolution = gt_seg.attrs["resolution"] # blueprint_labelmask_ds = "volumes/groundtruth/{version:}/crop{cropno:}/masks/{{label:}}" # labelmask_ds = blueprint_labelmask_ds.format(version=gt_version.lstrip("v"), cropno=crop["number"]) # labelmask_ds.format(label=label_ds_name) # if labelmask_ds in n5file: # mask = n5file[labelmask_ds] # else: # if label.generic_label is not None: # specific_labels = list(set(label.labelid) - set(label.generic_label)) # generic_condition = (all(l in get_all_annotated_label_ids(crop) for l in label.generic_label) or # all(l in get_all_annotated_label_ids(crop) for l in specific_labels)) # else: # generic_condition = False # if all(l in get_all_annotated_label_ids(crop) for l in label.labelid) or generic_condition: # mask = ((gt_seg > 0) * 1).astype(np.bool) # else: # mask = ((gt_seg > 0) * 0).astype(np.bool) return np.array(gt_seg), resolution
def main(): thrs_mult = [[127, 42]] samples = ["A", "B", "C", "A+", "B+", "C+"] setups_path = config_loader.get_config( )["synapses"]["training_setups_path"] filename_src = os.path.join( setups_path, "pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5") dataset_srcs = [ "predictions_it100000/cleft_dist_cropped", "predictions_it100000/cleft_dist_cropped", ] filename_tgt = os.path.join( setups_path, "pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5") dataset_tgts = [ "predictions_it100000/cleft_dist_cropped_thr{0:}", "predictions_it100000/cleft_dist_cropped_thr{0:}", ] for sample in samples: logging.info("thresholding sample {0:}".format(sample)) for thrs in thrs_mult: for ds_src, ds_tgt, thr in zip(dataset_srcs, dataset_tgts, thrs): logging.info(" dataset {0:} at {1:}".format(ds_src, thr)) threshold( filename_src.format(sample), ds_src, filename_tgt.format(sample), ds_tgt.format(thr), thr, )
def main(): samples = ["A", "B", "C"] filename_src = os.path.join(config_loader.get_config()["synapses"]["cremieval_path"], "{0:}/{1:}.n5") data_eval = [ "data2016-aligned", "data2016-unaligned", "data2017-aligned", "data2017-unaligned", ] dataset_srcs = ["volumes/masks/groundtruth"] # filename_tgt = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5' dataset_tgts = ["volumes/masks/groundtruth_cropped"] for de in data_eval: if "unaligned" in de: aligned = False else: aligned = True for sample in samples: logging.info("cropping sample {0:}".format(sample)) off = offsets[sample][aligned] sh = shapes[sample][aligned] for ds_src, ds_tgt in zip(dataset_srcs, dataset_tgts): logging.info(" dataset {0:}".format(ds_src)) crop_to_seg( filename_src.format(de, sample), ds_src, filename_src.format(de, sample), ds_tgt, off, sh, )
def main(s, mode=0, data=None): # samples = ['A','B', 'C'] samples = [(s.split("/")[-1]).split("_")[0]] for sample in samples: logging.info( "evaluating synapse predictions for sample {0:}".format(sample)) truth_fn = (os.path.join( config_loader.get_config()["synapses"]["cremi17_data_path"], "sample_{0:}_padded_20170424.aligned.hdf".format(sample))) if data is not None: logging.info("sample {0:} in mode {1:} using {2:}".format( sample, mode, data)) if (data == "val" or data == "validation" or data == "VAL" or data == "VALIDATION"): assert s.endswith(".hdf") test = CremiFile(s.replace(".hdf", ".validation.hdf"), "a") truth = CremiFile(truth_fn.replace(".hdf", ".validation.hdf"), "a") elif (data == "train" or data == "training" or data == "TRAIN" or data == "TRAINING"): assert s.endswith(".hdf") test = CremiFile(s.replace(".hdf", ".training.hdf"), "a") truth = CremiFile(truth_fn.replace(".hdf", ".training.hdf"), "a") else: test = CremiFile(s, "a") truth = CremiFile(truth_fn, "a") if mode == 0: evaluate(test, truth) elif mode == 1: evaluate_multrecgt(test, truth, add_in_file=True) elif mode == 2: evaluate_multrecgt(test, truth)
def main_seg(): samples = ["A", "B", "C"] # ['A', 'C', 'B+', 'C+'] setups_path = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post") filename_src = os.path.join(setups_path, "cremi/{0:}.n5") dataset_src = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped" filename_mask = os.path.join(setups_path, "cremi/{0:}.n5") dataset_mask = "volumes/labels/neuron_ids_gt_cropped" filename_tgt = os.path.join(setups_path, "cremi/{0:}.n5") dataset_tgt = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped_masked" for sample in samples: print(sample) off = offsets[sample] sh = shapes[sample] set_mask_to_zero( filename_src.format(sample), dataset_src, filename_mask.format(sample), dataset_mask, filename_tgt.format(sample), dataset_tgt, off, sh, )
def _get_iteration_queries(cropno: Sequence[Union[int, str]], db: cosem_db.MongoCosemDB) -> List[Dict[str, str]]: csv_folder_manual = os.path.join( config_loader.get_config()["organelles"]["evaluation_path"], db.training_version, "manual") csv_result_files = _get_csv_files(csv_folder_manual, "iteration", cropno, db) iteration_queries = [] for csv_f in csv_result_files: f = open(os.path.join(csv_folder_manual, csv_f), "r") fieldnames = ["setup", "labelname", "iteration", "raw_dataset"] cell_id = re.split("_(setup|iteration).csv", csv_f)[0] crop = db.get_validation_crop_by_cell_id(cell_id) reader = csv.DictReader(f, fieldnames) for row in reader: if any(lbl in get_label_ids_by_category(crop, "present_annotated") for lbl in hierarchy[row["labelname"]].labelid): query = { "label": row["labelname"], "raw_dataset": row["raw_dataset"], "setups": [row["setup"]], "crop": crop["number"] } iteration_queries.append(query) return iteration_queries
def run(): filepath = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "cremi_all/cremi_all_0116_01/prediction_cremi_warped_sampleC+_200000.n5" ) dataset = "syncleft_dist" thr = 0.0 dataset_tgt = "syncleft_dist_thr{0:}".format(thr) threshold(filepath, dataset, filepath, dataset_tgt, thr)
def run(self): logging.debug("Starting to run partner finding") progress = 0.0 self.set_progress_percentage(progress) thr = 127 cc_thr = 42 pre_thr = 42 post_thr = 35 dist_thr = 600 size_thr = 5 for s in self.samples: logging.debug("Starting with sample {0:}".format(s)) filename = os.path.join(os.path.dirname(self.input().fn), s + ".h5") syn_file = os.path.join(os.path.dirname(self.input().fn), s + ".n5") cleft_cc_ds = "clefts_cropped_thr{0:}_cc{1:}".format(thr, cc_thr) pre_ds = "pre_dist_cropped" post_ds = "post_dist_cropped" cleft_ds = "clefts_cropped" seg_file = os.path.join( config_loader.get_config()["synapses"]["cremieval_path"], self.de, s + ".n5", ) seg_ds = "volumes/labels/neuron_ids_constis_slf1_sf750_cropped" if "unaligned" in self.de: aligned = False else: aligned = True off = tuple(np.array(offsets[s][aligned]) * np.array((40, 4, 4))) mm = Matchmaker( syn_file, cleft_cc_ds, cleft_ds, pre_ds, post_ds, seg_file, seg_ds, filename, offset=off, safe_mem=True, dist_thr=dist_thr, size_thr=size_thr, pre_thr=pre_thr, post_thr=post_thr, ) # mm.prepare_file() mm.write_partners() mm.cremi_file.close() del mm progress += 100.0 / len(self.samples) try: self.set_progress_percentage(progress) except: pass for o in self.output(): done = o.open("w") done.close()
def get_offset_and_shape_from_crop(crop, gt_version="v0003"): n5file = zarr.open(os.path.join( config_loader.get_config()["organelles"]["data_path"], crop["parent"]), mode="r") label_ds = "volumes/groundtruth/{version:}/crop{cropno:}/labels/all".format( version=gt_version.lstrip("v"), cropno=crop["number"]) offset_wc = n5file[label_ds].attrs["offset"][::-1] offset = tuple(np.array(offset_wc) / 4.) shape = tuple(np.array(n5file[label_ds].shape) / 2.) return offset, shape
def main_seg(): samples = ["A", "B", "C", "A+", "B+", "C+"] # ['A', 'C', 'B+', 'C+'] filename_src = os.path.join(config_loader.get_config()["synapses"]["cremi17_data_path"], "sample_{0:}_padded_20170424.aligned.0bg.n5") dataset_src = "volumes/labels/neuron_ids" filename_tgt = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post/cremi/{0:}.n5") dataset_tgt = "volumes/labels/neuron_ids_gt_cropped" for sample in samples: print(sample) off = offsets[sample] sh = shapes[sample] crop_to_seg( filename_src.format(sample), dataset_src, filename_tgt.format(sample), dataset_tgt, off, sh, )
def transfer(training_version="v0003.2"): db = MongoCosemDB(training_version=training_version) eval_col = db.access("evaluation", db.training_version) eval_results_csv_folder = os.path.join( config_loader.get_config()["organelles"]["evaluation_path"], training_version, "evaluation_results") csv_d = CosemCSV(eval_results_csv_folder) for l in hierarchy.keys(): csv_d.erase(l) for db_entry in eval_col.find(): csv_d.write_evaluation_result(db_entry)
class SingleEvaluation(luigi.WrapperTask): iteration = luigi.IntParameter(default=186000) path = luigi.Parameter(default=os.path.join(config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post/pre_and_post-v9.0/run01/")) data_eval = luigi.TupleParameter(default=("data2016-aligned", "data2016-unaligned")) samples = luigi.TupleParameter(default=("A", "B", "C", "A+", "B+", "C+")) def requires(self): for de in self.data_eval: if "A+" in self.samples or "B+" in self.samples or "C+" in self.samples: test_samples = [] for s in self.samples: if "+" in s: test_samples.append(s) test_samples = tuple(test_samples) yield SplitModi( self.iteration, self.path, de, "groundtruth", test_samples, self.data_eval, ) if "A" in self.samples or "B" in self.samples or "C" in self.samples: training_samples = [] for s in self.samples: if not ("+" in s): training_samples.append(s) training_samples = tuple(training_samples) yield PartnerReport( self.iteration, self.path, de, "groundtruth", training_samples, self.data_eval, ) yield PartnerReport( self.iteration, self.path, de, "validation", training_samples, self.data_eval, ) yield PartnerReport( self.iteration, self.path, de, "training", training_samples, self.data_eval, )
def get_refined_comparisons( db: cosem_db.MongoCosemDB, cropno: Union[None, str, int, Sequence[Union[str, int]]] = None ) -> List[Dict[str, Any]]: """ Get list of queries for predictions that have been refined (as read from csv file) Args: db: Database with crop information. cropno: Specific crop number or list of crop numbers that should be included in queries. Returns: List of queries for which refined predictions exist. """ csv_folder_refined = os.path.join( config_loader.get_config()["organelles"]["evaluation_path"], db.training_version, "refined") # get list of csv files relevant for crops if cropno is None: csv_result_files = os.listdir(csv_folder_refined) else: if isinstance(cropno, str) or isinstance(cropno, int): cropno = [cropno] csv_result_files = [] for cno in cropno: crop = db.get_crop_by_number(cno) csv_result_files.append( os.path.join(csv_folder_refined, crop["dataset_id"] + "_setup.csv")) # collect entries from those csv files queries = [] for csv_f in csv_result_files: f = open(os.path.join(csv_folder_refined, csv_f), "r") fieldnames = ["setup", "labelname", "iteration", "raw_dataset"] cell_id = re.split("_setup.csv", csv_f)[0] crop = db.get_validation_crop_by_cell_id(cell_id) reader = csv.DictReader(f, fieldnames) for row in reader: # only consider results that we can evaluate automatically (actually contained in the crop) if any(lbl in get_label_ids_by_category(crop, "present_annotated") for lbl in hierarchy[row["labelname"]].labelid): query = { "label": row["labelname"], "raw_dataset": row["raw_dataset"], "setup": row["setup"], "crop": crop["number"], "iteration": int(row["iteration"]) } queries.append(query) return queries
def output(self): base = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "data_and_augmentations", self.dt, self.aug, "unet_checkpoint_" + str(self.it), ) return [ luigi.LocalTarget(base + ".data-00000-of-00001"), luigi.LocalTarget(base + ".index"), luigi.LocalTarget(base + ".meta"), ]
def main(): thrs_mult = [[153, 76, 76], [127, 63, 63]] samples = ["B"] # ['A+', 'B+', 'C+', 'A', 'B', 'C'] filename_src = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5" ) dataset_srcs = ["predictions_it400000/cleft_dist_cropped_thr{0:}"] filename_tgt = os.path.join(config_loader.get_config()["synapses"]["training_setups_path"], "pre_and_post/pre_and_post-v3.0/cremi/{0:}.n5" ) dataset_tgts = ["predictions_it400000/cleft_dist_cropped_thr{0:}_cc"] for sample in samples: logging.info("finding connected components for sample {0:}".format(sample)) for thrs in thrs_mult: for ds_src, ds_tgt, thr in zip(dataset_srcs, dataset_tgts, thrs): logging.info(" dataset {0:}".format(ds_src.format(thr))) cc( filename_src.format(sample), ds_src.format(thr), filename_tgt.format(sample), ds_tgt.format(thr), )
def main(): thrs = [127, 63, 63] samples = ["A+", "B+", "C+"] # samples = ['B+', 'C+'] slf = 1 offsets = { "A+": (37 * 40, 1176 * 4, 955 * 4), "B+": (37 * 40, 1076 * 4, 1284 * 4), "C+": (37 * 40, 1002 * 4, 1165 * 4), "A": (38 * 40, 942 * 4, 951 * 4), "B": (37 * 40, 1165 * 4, 1446 * 4), "C": (37 * 40, 1032 * 4, 1045 * 4), } # segf_name = {'A+': 'sample_A+_85_aff_0.8_cf_hq_dq_dm1_mf0.81', # 'B+': 'sample_B+_median_aff_0.8_cf_hq_dq_dm1_mf0.87', # 'C+': 'sample_C+_85_aff_0.8_cf_hq_dq_dm1_mf0.75', # } # filename_src = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/{0:}.n5' # dataset_src = 'main' # filename_tgt = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/{0:}_sizefiltered750.n5' # dataset_tgt = 'main' # dat_file = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/pre_and_post-v3.0/cremi/{0:}_sizefilter750.dat' # filename_src = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_warped/sample{0}.n5' # dataset_src = 'segmentations/multicut' # dataset_src = 'segmentations/mc_glia_global2' # filename_src = '/groups/saalfeld/saalfeldlab/larissa/data/cremi-2017/sample_{0:}_padded_20170424.aligned.0bg.n5' # dataset_src = 'volumes/labels/neuron_ids' # filename_src = '/groups/saalfeld/home/papec/Work/neurodata_hdd/cremi_new/sample{0:}.n5' # dataset_src = 'segmentation/multicut' # filename_tgt = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/cremi/{0}.n5' filename = os.path.join(config_loader.get_config()["synapses"]["cremieval_path"],"data2016-aligned/{0:}.n5") dataset_src = "volumes/labels/neuron_ids_constis_cropped" dataset_tgt = "volumes/labels/neuron_ids_constis_slf{0:}_cropped" # dat_file = '/nrs/saalfeld/heinrichl/synapses/pre_and_post/pre_and_post-v3.0/cremi/{0:}_constis_slicefilter{' \ # '1:}.dat' for sample in samples: print(sample) slicefilter( filename.format(sample), dataset_src, filename.format(sample), dataset_tgt.format(slf), slf, ) # , dat_file.format(sample, slf))
def load_result(data_train, augmentation, data_eval, iteration, mode): result_json = os.path.join( config_loader.get_config()["synapses"]["training_setups_path"], "data_and_augmentations", data_train, augmentation, "evaluation", str(iteration), data_eval, "partners.{0:}.json".format(mode), ) try: with open(result_json, "r") as f: resdict = json.load(f) except IOError: return None return resdict
def _get_setup_queries( cropno: Sequence[Union[int, str]], db: cosem_db.MongoCosemDB ) -> List[Dict[str, Union[str, Sequence[str]]]]: csv_folder_manual = os.path.join( config_loader.get_config()["organelles"]["evaluation_path"], db.training_version, "manual") csv_result_files = _get_csv_files(csv_folder_manual, "setup", cropno, db) setup_queries = [] for csv_f in csv_result_files: f = open(os.path.join(csv_folder_manual, csv_f), "r") fieldnames = ["setup", "labelname", "iteration", "raw_dataset"] cell_id = re.split("_(setup|iteration).csv", csv_f)[0] crop = db.get_validation_crop_by_cell_id(cell_id) reader = csv.DictReader(f, fieldnames) for row in reader: if any(lbl in get_label_ids_by_category(crop, "present_annotated") for lbl in hierarchy[row["labelname"]].labelid): # find the csv files with the list of setups compared for each label (4nm or 8nm) if row["raw_dataset"] == "volumes/raw/s0": ff = open( os.path.join(csv_folder_manual, "compared_4nm_setups.csv"), "r") elif row["raw_dataset"] == "volumes/subsampled/raw/0" or row[ "raw_dataset"] == "volumes/raw/s1": ff = open( os.path.join(csv_folder_manual, "compared_8nm_setups.csv"), "r") else: raise ValueError("The raw_dataset {0:} ".format( row["raw_dataset"])) # get that list of compared setups from the csv file compare_reader = csv.reader(ff) for compare_row in compare_reader: if compare_row[0] == row["labelname"]: setups = compare_row[1:] break # collect result query = { "label": row["labelname"], "raw_dataset": row["raw_dataset"], "setups": setups, "crop": crop["number"] } setup_queries.append(query) return setup_queries
def run(self): progress = 0.0 self.set_progress_percentage(progress) for s in self.samples: print(s) filename = os.path.join(os.path.dirname(self.input()[0].fn), s + ".h5") mask_filename = os.path.join( config_loader.get_config()["synapses"]["cremieval_path"], self.de, s + ".n5", ) mask_dataset = "volumes/masks/" + self.m filename_tgt = filename.replace("h5", self.m + ".h5") # shutil.copy(filename, filename_tgt) f = CremiFile(filename, "a") g = CremiFile(filename_tgt, "a") maskf = zarr.open(mask_filename, mode="r") mask = maskf[mask_dataset] off = mask.attrs["offset"] res = mask.attrs["resolution"] mask = np.array(mask[:]) ann = f.read_annotations() shift = sub(ann.offset, off) ids = ann.ids() rmids = [] for i in ids: t, loc = ann.get_annotation(i) vx_idx = (np.array(add(loc, shift)) / res).astype(np.int) if not mask[tuple(vx_idx)]: rmids.append(i) print(rmids) for i in rmids: print("removing {0:}".format(i)) ann.remove_annotation(i) print(ann.comments.keys()) print(ann.pre_post_partners) g.write_annotations(ann) progress += 100.0 / len(self.samples) try: self.set_progress_percentage(progress) except: pass for o in self.output(): done = o.open("w") done.close()
def run(self): progress = 0.0 self.set_progress_percentage(progress) results = dict() for s in self.samples: thr = 127 testfile = os.path.join(os.path.dirname(self.input().fn), s + ".n5") truthfile = os.path.join( config_loader.get_config()["synapses"]["cremieval_path"], self.de, s + ".n5", ) test = np.array( zarr.open(testfile, mode="r")["clefts_cropped_thr" + str(thr)][:]) truth = np.array( zarr.open(truthfile, mode="r")["volumes/labels/clefts_cropped"][:]) mask = np.array( zarr.open(truthfile, mode="r")["volumes/masks/" + self.m + "_cropped"][:]) clefts_evaluation = Clefts(test, truth, np.logical_not(mask)) results[s] = dict() results[s][ "false negatives count"] = clefts_evaluation.count_false_negatives( ) results[s][ "false positives count"] = clefts_evaluation.count_false_positives( ) results[s][ "false negative distance"] = clefts_evaluation.acc_false_negatives( ) results[s][ "false positive distance"] = clefts_evaluation.acc_false_positives( ) progress += 100.0 / len(self.samples) try: self.set_progress_percentage(progress) except: pass with self.output().open("w") as done: json.dump(results, done)