def prepare_submission(sample, path_segm, inner_path_segm, path_bbox_slice, ds_factor=None): """ :param path_segm: :param inner_path_segm: :param path_bbox_slice: path to the csv file :param ds_factor: for example (1, 2, 2) """ segm = segm_utils.readHDF5(path_segm, inner_path_segm) bbox_data = np.genfromtxt(path_bbox_slice, delimiter=';', dtype='int') assert bbox_data.shape[0] == segm.ndim and bbox_data.shape[1] == 2 # bbox_slice = tuple(slice(b_data[0], b_data[1]) for b_data in bbox_data) if ds_factor is not None: assert len(ds_factor) == segm.ndim segm = zoom(segm, ds_factor, order=0) padding = tuple( (slc[0], shp - slc[1]) for slc, shp in zip(bbox_data, shape_padded_aligned_datasets[sample])) padded_segm = np.pad(segm, pad_width=padding, mode="constant") # Apply Constantin crop and then backalign: cropped_segm = padded_segm[magic_bboxes[sample]] tmp_file = path_segm.replace(".h5", "_submission_temp.hdf") backalign_segmentation(sample, cropped_segm, tmp_file, key="temp_data", postprocess=False) # Create a CREMI-style file ready to submit: final_submission_path = path_segm.replace(".h5", "_submission.hdf") file = CremiFile(final_submission_path, "w") # Write volumes representing the neuron and synaptic cleft segmentation. backaligned_segm = segm_utils.readHDF5(tmp_file, "temp_data") neuron_ids = Volume(backaligned_segm.astype('uint64'), resolution=(40.0, 4.0, 4.0), comment="Emb-submission") file.write_neuron_ids(neuron_ids) file.close() os.remove(tmp_file)
def filter(h5filepath, csv_file_src, csv_file_tgt=None, cleft_ds_name="syncleft_dist_thr0.0_cc"): logging.info("Filtering clefts in {0:}/{1:} with {2:}".format( h5filepath, cleft_ds_name, csv_file_src)) cf = CremiFile(h5filepath, "r+") ann = cf.read_annotations() cleft_to_pre, cleft_to_post = make_cleft_to_prepostsyn_neuron_id_dict( csv_file_src) cleft_list_verified = cleft_to_pre.keys() logging.info("List of verified clefts:\n{0:}".format(cleft_list_verified)) cleft_ds = np.array(cf.read_volume(cleft_ds_name).data) cleft_list_all = list(np.unique(cleft_ds)) for bg_id in BG_IDS: cleft_list_all.remove(bg_id) logging.info("List of all clefts:\n{0:}".format(cleft_list_all)) cleft_list_unmatched = list(set(cleft_list_all) - set(cleft_list_verified)) logging.info( "List of unmatched clefts:\n{0:}".format(cleft_list_unmatched)) if csv_file_tgt is not None: with open(csv_file_tgt, "w") as f: writer = csv.writer(f) for i in cleft_list_unmatched: writer.writerow([i]) next_id = max(ann.ids()) + 1 logging.info("Adding annotations...") for cleft_id in cleft_list_unmatched: logging.info("... for cleft {0:}".format(cleft_id)) cleft_coords = np.where(cleft_ds == cleft_id) cleft_center = ( 40.0 * cleft_coords[0][int(len(cleft_coords[0]) / 2.0)], 4.0 * cleft_coords[1][int(len(cleft_coords[1]) / 2.0)], 4.0 * cleft_coords[2][int(len(cleft_coords[2]) / 2.0)], ) ann.add_annotation(next_id, "synapse", cleft_center) ann.add_comment(next_id, str(cleft_id)) next_id += 1 logging.info("Saving annotations...") cf.write_annotations(ann) cf.close() logging.info("...done \n\n")
def prepare_submission(): from cremi.io import CremiFile from cremi.Volume import Volume base = "/home/fabian/drives/datasets/results/nnUNet/test_sets/Task061_CREMI/" # a+ pred = sitk.GetArrayFromImage( sitk.ReadImage(join(base, 'results_3d_fullres', "sample_a+.nii.gz"))).astype(np.uint64) pred[pred == 0] = 0xffffffffffffffff out_a = CremiFile(join(base, 'sample_A+_20160601.hdf'), 'w') clefts = Volume(pred, (40., 4., 4.)) out_a.write_clefts(clefts) out_a.close() pred = sitk.GetArrayFromImage( sitk.ReadImage(join(base, 'results_3d_fullres', "sample_b+.nii.gz"))).astype(np.uint64) pred[pred == 0] = 0xffffffffffffffff out_b = CremiFile(join(base, 'sample_B+_20160601.hdf'), 'w') clefts = Volume(pred, (40., 4., 4.)) out_b.write_clefts(clefts) out_b.close() pred = sitk.GetArrayFromImage( sitk.ReadImage(join(base, 'results_3d_fullres', "sample_c+.nii.gz"))).astype(np.uint64) pred[pred == 0] = 0xffffffffffffffff out_c = CremiFile(join(base, 'sample_C+_20160601.hdf'), 'w') clefts = Volume(pred, (40., 4., 4.)) out_c.write_clefts(clefts) out_c.close()
# Create some dummy annotation data annotations = Annotations() for id in [ 0, 1, 2, 3 ]: location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100)) annotations.add_annotation(id, "presynaptic_site", location) for id in [ 4, 5, 6, 7 ]: location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100)) annotations.add_annotation(id, "postsynaptic_site", location) for (pre, post) in [ (0, 4), (1, 5), (2, 6), (3, 7) ]: annotations.set_pre_post_partners(pre, post) annotations.add_comment(6, "unsure") # Open a file for writing (deletes previous file, if exists) file = CremiFile("example.hdf", "w") # Write the raw volume. This is given here just for illustration. For your # submission, you don't need to store the raw data. We have it already. raw = Volume(np.zeros((10,100,100), dtype=np.uint8), resolution=(40.0, 4.0, 4.0)) file.write_raw(raw) # Write volumes representing the neuron and synaptic cleft segmentation. neuron_ids = Volume(np.ones((10,100,100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just ones") clefts = Volume(np.zeros((10,100,100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just zeros") file.write_neuron_ids(neuron_ids) file.write_clefts(clefts) # Write synaptic partner annotations. file.write_annotations(annotations) file.close()
location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100)) annotations.add_annotation(id, "postsynaptic_site", location) for (pre, post) in [(0, 4), (1, 5), (2, 6), (3, 7)]: annotations.set_pre_post_partners(pre, post) annotations.add_comment(6, "unsure") # Open a file for writing (deletes previous file, if exists) file = CremiFile("example.hdf", "w") # Write the raw volume. This is given here just for illustration. For your # submission, you don't need to store the raw data. We have it already. raw = Volume(np.zeros((10, 100, 100), dtype=np.uint8), resolution=(40.0, 4.0, 4.0)) file.write_raw(raw) # Write volumes representing the neuron and synaptic cleft segmentation. neuron_ids = Volume(np.ones((10, 100, 100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just ones") clefts = Volume(np.zeros((10, 100, 100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just zeros") file.write_neuron_ids(neuron_ids) file.write_clefts(clefts) # Write synaptic partner annotations. file.write_annotations(annotations) file.close()
class AbstractCremiFactory(metaclass=ABCMeta): data_source = None def __init__(self, output_path, mode="r"): self.output_path = str(output_path) self._cremi_file = None self.resolution = None self.translation = None self.timestamp = datetime.now().astimezone().isoformat() CremiFile(self.output_path, mode).close() self.mode = mode if mode.startswith("r") else "a" @abstractmethod def get_raw(self, offset_px, shape_px): pass @abstractmethod def set_input_stack(self, n5_ds, resolution_nm_zyx=None, translation_nm_zyx=None): pass def populate( self, offset_from_stack_px, shape_px, padding_low_px=0, padding_high_px=None, skip_if_exists=False, ): """ Parameters ---------- offset_from_stack_px : CoordZYX Offset of unpadded ROI from stack origin, in pixels shape_px : CoordZYX Shape of unpadded ROI padding_low_px : CoordZYX or Number Padding to add to lower corner of ROI padding_high_px : CoordZYX or Number Padding to add to higher corner of ROI Returns ------- """ logger.info("populating cremi file") padding_low_px, padding_high_px = resolve_padding( padding_low_px, padding_high_px, math.ceil) padded_offset_from_stack_px = math.floor(offset_from_stack_px - padding_low_px) padded_shape_px = math.ceil(shape_px + padding_low_px + padding_high_px) self._populate_raw(padded_offset_from_stack_px, padded_shape_px, skip_if_exists) self._populate_clefts(shape_px, padding_low_px, skip_if_exists) self._populate_annotations( padded_offset_from_stack_px, padded_shape_px, padding_low_px, padding_high_px, skip_if_exists, ) def has_raw(self): with self._open("r"): return self._cremi_file.has_raw() def has_clefts(self): with self._open("r"): return self._cremi_file.has_clefts() def has_annotations(self): with self._open("r"): return self._cremi_file.has_annotations() def _populate_raw(self, padded_offset_from_stack_px, padded_shape_px, skip_if_exists): """ Parameters ---------- padded_offset_from_stack_px : CoordZYX padded_shape_px : CoordZYX skip_if_exists Returns ------- """ if self.has_raw(): if skip_if_exists: logger.info("Raw data already exists, skipping") return else: raise RuntimeError("Raw data already exists") logger.debug("reading raw volume") raw_data = self.get_raw(padded_offset_from_stack_px, padded_shape_px) raw_volume = Volume(raw_data, resolution=self.resolution) logger.debug("writing raw volume") with self: self._cremi_file.write_raw(raw_volume) self._cremi_file.h5file["volumes/raw"].attrs[ "data_source"] = self.data_source self._cremi_file.h5file["volumes/raw"].attrs[ "populated_on"] = self.timestamp self._cremi_file.h5file.attrs["roi_offset_from_stack"] = ( padded_offset_from_stack_px * CoordZYX(self.resolution)).to_list() def _populate_clefts(self, unpadded_shape_px, padding_low_px, skip_if_exists): """ Parameters ---------- unpadded_shape_px : CoordZYX padding_low_px : CoordZYX skip_if_exists Returns ------- """ if self.has_clefts(): if skip_if_exists: logger.info("Cleft data already exists, skipping") return else: raise RuntimeError("Cleft data already exists") logger.debug("generating cleft volume") cleft_volume = Volume( np.zeros(unpadded_shape_px.to_list(), dtype=np.uint64), resolution=self.resolution, offset=(padding_low_px * CoordZYX(self.resolution)).to_list(), ) logger.debug("writing clefts") with self: self._cremi_file.write_clefts(cleft_volume) self._cremi_file.h5file["volumes/labels/clefts"].attrs[ "refreshed_on"] = self.timestamp def _populate_annotations( self, padded_offset_from_stack_px, padded_shape_px, padding_low_px, padding_high_px, skip_if_exists, ): """ Parameters ---------- padded_offset_from_stack_px : CoordZYX padded_shape_px : CoordZYX padding_low_px : CoordZYX skip_if_exists Returns ------- """ if self.has_annotations(): if skip_if_exists: logger.info("Annotation data already exists, skipping") return else: raise RuntimeError("Annotation data already exists") logger.debug("fetching and mangling annotations") resolution = CoordZYX(self.resolution) id_gen = IdGenerator.from_hdf(self.output_path) # annotations = catmaid_to_annotations_conn( # annotations = catmaid_to_annotations_conn_to_tn( # annotations = catmaid_to_annotations_tn_to_tn( annotations, pre_to_conn = catmaid_to_annotations_near_conn_to_tn( CoordZYX(self.translation) + padded_offset_from_stack_px * resolution, padded_shape_px * resolution, padding_low_px * resolution, padding_high_px * resolution, comment=True, id_generator=id_gen, ) pre_to_conn_arr = np.array(sorted(pre_to_conn.items()), dtype=np.uint64) with self: logger.debug("writing annotations") self._cremi_file.write_annotations(annotations) f = self._cremi_file.h5file f["/annotations"].attrs["populated_on"] = self.timestamp ds = f.create_dataset("/annotations/presynaptic_site/pre_to_conn", data=pre_to_conn_arr) ds.attrs["explanation"] = ( "BIGCAT only displays one edge per presynapse, so this format creates new presynapses near the " "connector node. This dataset maps these nodes to the connector IDs" ) f.attrs["annotation_version"] = ANNOTATION_VERSION def _open(self, mode=None): mode = mode or self.mode self._cremi_file = CremiFile(self.output_path, mode) return self def close(self): try: self._cremi_file.close() except AttributeError: pass self._cremi_file = None def __enter__(self): if self._cremi_file is None: return self._open() return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()