def prepare_submission(sample,
path_segm,
inner_path_segm,
path_bbox_slice,
ds_factor=None):
"""
:param path_segm:
:param inner_path_segm:
:param path_bbox_slice: path to the csv file
:param ds_factor: for example (1, 2, 2)
"""
segm = segm_utils.readHDF5(path_segm, inner_path_segm)
bbox_data = np.genfromtxt(path_bbox_slice, delimiter=';', dtype='int')
assert bbox_data.shape[0] == segm.ndim and bbox_data.shape[1] == 2
# bbox_slice = tuple(slice(b_data[0], b_data[1]) for b_data in bbox_data)
if ds_factor is not None:
assert len(ds_factor) == segm.ndim
segm = zoom(segm, ds_factor, order=0)
padding = tuple(
(slc[0], shp - slc[1])
for slc, shp in zip(bbox_data, shape_padded_aligned_datasets[sample]))
padded_segm = np.pad(segm, pad_width=padding, mode="constant")
# Apply Constantin crop and then backalign:
cropped_segm = padded_segm[magic_bboxes[sample]]
tmp_file = path_segm.replace(".h5", "_submission_temp.hdf")
backalign_segmentation(sample,
cropped_segm,
tmp_file,
key="temp_data",
postprocess=False)
# Create a CREMI-style file ready to submit:
final_submission_path = path_segm.replace(".h5", "_submission.hdf")
file = CremiFile(final_submission_path, "w")
# Write volumes representing the neuron and synaptic cleft segmentation.
backaligned_segm = segm_utils.readHDF5(tmp_file, "temp_data")
neuron_ids = Volume(backaligned_segm.astype('uint64'),
resolution=(40.0, 4.0, 4.0),
comment="Emb-submission")
file.write_neuron_ids(neuron_ids)
file.close()
os.remove(tmp_file)
def filter(h5filepath,
csv_file_src,
csv_file_tgt=None,
cleft_ds_name="syncleft_dist_thr0.0_cc"):
logging.info("Filtering clefts in {0:}/{1:} with {2:}".format(
h5filepath, cleft_ds_name, csv_file_src))
cf = CremiFile(h5filepath, "r+")
ann = cf.read_annotations()
cleft_to_pre, cleft_to_post = make_cleft_to_prepostsyn_neuron_id_dict(
csv_file_src)
cleft_list_verified = cleft_to_pre.keys()
logging.info("List of verified clefts:\n{0:}".format(cleft_list_verified))
cleft_ds = np.array(cf.read_volume(cleft_ds_name).data)
cleft_list_all = list(np.unique(cleft_ds))
for bg_id in BG_IDS:
cleft_list_all.remove(bg_id)
logging.info("List of all clefts:\n{0:}".format(cleft_list_all))
cleft_list_unmatched = list(set(cleft_list_all) - set(cleft_list_verified))
logging.info(
"List of unmatched clefts:\n{0:}".format(cleft_list_unmatched))
if csv_file_tgt is not None:
with open(csv_file_tgt, "w") as f:
writer = csv.writer(f)
for i in cleft_list_unmatched:
writer.writerow([i])
next_id = max(ann.ids()) + 1
logging.info("Adding annotations...")
for cleft_id in cleft_list_unmatched:
logging.info("... for cleft {0:}".format(cleft_id))
cleft_coords = np.where(cleft_ds == cleft_id)
cleft_center = (
40.0 * cleft_coords[0][int(len(cleft_coords[0]) / 2.0)],
4.0 * cleft_coords[1][int(len(cleft_coords[1]) / 2.0)],
4.0 * cleft_coords[2][int(len(cleft_coords[2]) / 2.0)],
)
ann.add_annotation(next_id, "synapse", cleft_center)
ann.add_comment(next_id, str(cleft_id))
next_id += 1
logging.info("Saving annotations...")
cf.write_annotations(ann)
cf.close()
logging.info("...done \n\n")
def prepare_submission():
from cremi.io import CremiFile
from cremi.Volume import Volume
base = "/home/fabian/drives/datasets/results/nnUNet/test_sets/Task061_CREMI/"
# a+
pred = sitk.GetArrayFromImage(
sitk.ReadImage(join(base, 'results_3d_fullres',
"sample_a+.nii.gz"))).astype(np.uint64)
pred[pred == 0] = 0xffffffffffffffff
out_a = CremiFile(join(base, 'sample_A+_20160601.hdf'), 'w')
clefts = Volume(pred, (40., 4., 4.))
out_a.write_clefts(clefts)
out_a.close()
pred = sitk.GetArrayFromImage(
sitk.ReadImage(join(base, 'results_3d_fullres',
"sample_b+.nii.gz"))).astype(np.uint64)
pred[pred == 0] = 0xffffffffffffffff
out_b = CremiFile(join(base, 'sample_B+_20160601.hdf'), 'w')
clefts = Volume(pred, (40., 4., 4.))
out_b.write_clefts(clefts)
out_b.close()
pred = sitk.GetArrayFromImage(
sitk.ReadImage(join(base, 'results_3d_fullres',
"sample_c+.nii.gz"))).astype(np.uint64)
pred[pred == 0] = 0xffffffffffffffff
out_c = CremiFile(join(base, 'sample_C+_20160601.hdf'), 'w')
clefts = Volume(pred, (40., 4., 4.))
out_c.write_clefts(clefts)
out_c.close()
# Create some dummy annotation data
annotations = Annotations()
for id in [ 0, 1, 2, 3 ]:
location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100))
annotations.add_annotation(id, "presynaptic_site", location)
for id in [ 4, 5, 6, 7 ]:
location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100))
annotations.add_annotation(id, "postsynaptic_site", location)
for (pre, post) in [ (0, 4), (1, 5), (2, 6), (3, 7) ]:
annotations.set_pre_post_partners(pre, post)
annotations.add_comment(6, "unsure")
# Open a file for writing (deletes previous file, if exists)
file = CremiFile("example.hdf", "w")
# Write the raw volume. This is given here just for illustration. For your
# submission, you don't need to store the raw data. We have it already.
raw = Volume(np.zeros((10,100,100), dtype=np.uint8), resolution=(40.0, 4.0, 4.0))
file.write_raw(raw)
# Write volumes representing the neuron and synaptic cleft segmentation.
neuron_ids = Volume(np.ones((10,100,100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just ones")
clefts = Volume(np.zeros((10,100,100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just zeros")
file.write_neuron_ids(neuron_ids)
file.write_clefts(clefts)
# Write synaptic partner annotations.
file.write_annotations(annotations)
file.close()
location = (random.randint(0, 100), random.randint(0, 100),
random.randint(0, 100))
annotations.add_annotation(id, "postsynaptic_site", location)
for (pre, post) in [(0, 4), (1, 5), (2, 6), (3, 7)]:
annotations.set_pre_post_partners(pre, post)
annotations.add_comment(6, "unsure")
# Open a file for writing (deletes previous file, if exists)
file = CremiFile("example.hdf", "w")
# Write the raw volume. This is given here just for illustration. For your
# submission, you don't need to store the raw data. We have it already.
raw = Volume(np.zeros((10, 100, 100), dtype=np.uint8),
resolution=(40.0, 4.0, 4.0))
file.write_raw(raw)
# Write volumes representing the neuron and synaptic cleft segmentation.
neuron_ids = Volume(np.ones((10, 100, 100), dtype=np.uint64),
resolution=(40.0, 4.0, 4.0),
comment="just ones")
clefts = Volume(np.zeros((10, 100, 100), dtype=np.uint64),
resolution=(40.0, 4.0, 4.0),
comment="just zeros")
file.write_neuron_ids(neuron_ids)
file.write_clefts(clefts)
# Write synaptic partner annotations.
file.write_annotations(annotations)
file.close()
class AbstractCremiFactory(metaclass=ABCMeta):
data_source = None
def __init__(self, output_path, mode="r"):
self.output_path = str(output_path)
self._cremi_file = None
self.resolution = None
self.translation = None
self.timestamp = datetime.now().astimezone().isoformat()
CremiFile(self.output_path, mode).close()
self.mode = mode if mode.startswith("r") else "a"
@abstractmethod
def get_raw(self, offset_px, shape_px):
pass
@abstractmethod
def set_input_stack(self,
n5_ds,
resolution_nm_zyx=None,
translation_nm_zyx=None):
pass
def populate(
self,
offset_from_stack_px,
shape_px,
padding_low_px=0,
padding_high_px=None,
skip_if_exists=False,
):
"""
Parameters
----------
offset_from_stack_px : CoordZYX
Offset of unpadded ROI from stack origin, in pixels
shape_px : CoordZYX
Shape of unpadded ROI
padding_low_px : CoordZYX or Number
Padding to add to lower corner of ROI
padding_high_px : CoordZYX or Number
Padding to add to higher corner of ROI
Returns
-------
"""
logger.info("populating cremi file")
padding_low_px, padding_high_px = resolve_padding(
padding_low_px, padding_high_px, math.ceil)
padded_offset_from_stack_px = math.floor(offset_from_stack_px -
padding_low_px)
padded_shape_px = math.ceil(shape_px + padding_low_px +
padding_high_px)
self._populate_raw(padded_offset_from_stack_px, padded_shape_px,
skip_if_exists)
self._populate_clefts(shape_px, padding_low_px, skip_if_exists)
self._populate_annotations(
padded_offset_from_stack_px,
padded_shape_px,
padding_low_px,
padding_high_px,
skip_if_exists,
)
def has_raw(self):
with self._open("r"):
return self._cremi_file.has_raw()
def has_clefts(self):
with self._open("r"):
return self._cremi_file.has_clefts()
def has_annotations(self):
with self._open("r"):
return self._cremi_file.has_annotations()
def _populate_raw(self, padded_offset_from_stack_px, padded_shape_px,
skip_if_exists):
"""
Parameters
----------
padded_offset_from_stack_px : CoordZYX
padded_shape_px : CoordZYX
skip_if_exists
Returns
-------
"""
if self.has_raw():
if skip_if_exists:
logger.info("Raw data already exists, skipping")
return
else:
raise RuntimeError("Raw data already exists")
logger.debug("reading raw volume")
raw_data = self.get_raw(padded_offset_from_stack_px, padded_shape_px)
raw_volume = Volume(raw_data, resolution=self.resolution)
logger.debug("writing raw volume")
with self:
self._cremi_file.write_raw(raw_volume)
self._cremi_file.h5file["volumes/raw"].attrs[
"data_source"] = self.data_source
self._cremi_file.h5file["volumes/raw"].attrs[
"populated_on"] = self.timestamp
self._cremi_file.h5file.attrs["roi_offset_from_stack"] = (
padded_offset_from_stack_px *
CoordZYX(self.resolution)).to_list()
def _populate_clefts(self, unpadded_shape_px, padding_low_px,
skip_if_exists):
"""
Parameters
----------
unpadded_shape_px : CoordZYX
padding_low_px : CoordZYX
skip_if_exists
Returns
-------
"""
if self.has_clefts():
if skip_if_exists:
logger.info("Cleft data already exists, skipping")
return
else:
raise RuntimeError("Cleft data already exists")
logger.debug("generating cleft volume")
cleft_volume = Volume(
np.zeros(unpadded_shape_px.to_list(), dtype=np.uint64),
resolution=self.resolution,
offset=(padding_low_px * CoordZYX(self.resolution)).to_list(),
)
logger.debug("writing clefts")
with self:
self._cremi_file.write_clefts(cleft_volume)
self._cremi_file.h5file["volumes/labels/clefts"].attrs[
"refreshed_on"] = self.timestamp
def _populate_annotations(
self,
padded_offset_from_stack_px,
padded_shape_px,
padding_low_px,
padding_high_px,
skip_if_exists,
):
"""
Parameters
----------
padded_offset_from_stack_px : CoordZYX
padded_shape_px : CoordZYX
padding_low_px : CoordZYX
skip_if_exists
Returns
-------
"""
if self.has_annotations():
if skip_if_exists:
logger.info("Annotation data already exists, skipping")
return
else:
raise RuntimeError("Annotation data already exists")
logger.debug("fetching and mangling annotations")
resolution = CoordZYX(self.resolution)
id_gen = IdGenerator.from_hdf(self.output_path)
# annotations = catmaid_to_annotations_conn(
# annotations = catmaid_to_annotations_conn_to_tn(
# annotations = catmaid_to_annotations_tn_to_tn(
annotations, pre_to_conn = catmaid_to_annotations_near_conn_to_tn(
CoordZYX(self.translation) +
padded_offset_from_stack_px * resolution,
padded_shape_px * resolution,
padding_low_px * resolution,
padding_high_px * resolution,
comment=True,
id_generator=id_gen,
)
pre_to_conn_arr = np.array(sorted(pre_to_conn.items()),
dtype=np.uint64)
with self:
logger.debug("writing annotations")
self._cremi_file.write_annotations(annotations)
f = self._cremi_file.h5file
f["/annotations"].attrs["populated_on"] = self.timestamp
ds = f.create_dataset("/annotations/presynaptic_site/pre_to_conn",
data=pre_to_conn_arr)
ds.attrs["explanation"] = (
"BIGCAT only displays one edge per presynapse, so this format creates new presynapses near the "
"connector node. This dataset maps these nodes to the connector IDs"
)
f.attrs["annotation_version"] = ANNOTATION_VERSION
def _open(self, mode=None):
mode = mode or self.mode
self._cremi_file = CremiFile(self.output_path, mode)
return self
def close(self):
try:
self._cremi_file.close()
except AttributeError:
pass
self._cremi_file = None
def __enter__(self):
if self._cremi_file is None:
return self._open()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
