def run(self):
progress = 0.0
self.set_progress_percentage(progress)
for s in self.samples:
print(s)
filename = os.path.join(os.path.dirname(self.input()[0].fn),
s + ".h5")
mask_filename = os.path.join(
config_loader.get_config()["synapses"]["cremieval_path"],
self.de,
s + ".n5",
)
mask_dataset = "volumes/masks/" + self.m
filename_tgt = filename.replace("h5", self.m + ".h5")
# shutil.copy(filename, filename_tgt)
f = CremiFile(filename, "a")
g = CremiFile(filename_tgt, "a")
maskf = zarr.open(mask_filename, mode="r")
mask = maskf[mask_dataset]
off = mask.attrs["offset"]
res = mask.attrs["resolution"]
mask = np.array(mask[:])
ann = f.read_annotations()
shift = sub(ann.offset, off)
ids = ann.ids()
rmids = []
for i in ids:
t, loc = ann.get_annotation(i)
vx_idx = (np.array(add(loc, shift)) / res).astype(np.int)
if not mask[tuple(vx_idx)]:
rmids.append(i)
print(rmids)
for i in rmids:
print("removing {0:}".format(i))
ann.remove_annotation(i)
print(ann.comments.keys())
print(ann.pre_post_partners)
g.write_annotations(ann)
progress += 100.0 / len(self.samples)
try:
self.set_progress_percentage(progress)
except:
pass
for o in self.output():
done = o.open("w")
done.close()
def run(self):
progress = 0.
self.set_progress_percentage(progress)
for s in self.samples:
print(s)
filename = os.path.join(os.path.dirname(self.input()[0].fn),
s + '.h5')
mask_filename = os.path.join(
'/groups/saalfeld/saalfeldlab/larissa/data/cremieval', self.de,
s + '.n5')
mask_dataset = 'volumes/masks/' + self.m
filename_tgt = filename.replace('h5', self.m + '.h5')
#shutil.copy(filename, filename_tgt)
f = CremiFile(filename, 'a')
g = CremiFile(filename_tgt, 'a')
maskf = z5py.File(mask_filename, use_zarr_format=False)
mask = maskf[mask_dataset]
off = mask.attrs['offset']
res = mask.attrs['resolution']
mask = np.array(mask[:])
ann = f.read_annotations()
shift = sub(ann.offset, off)
ids = ann.ids()
rmids = []
for i in ids:
t, loc = ann.get_annotation(i)
vx_idx = (np.array(add(loc, shift)) / res).astype(np.int)
if not mask[tuple(vx_idx)]:
rmids.append(i)
print(rmids)
for i in rmids:
print('removing {0:}'.format(i))
ann.remove_annotation(i)
print(ann.comments.keys())
print(ann.pre_post_partners)
g.write_annotations(ann)
progress += 100. / len(self.samples)
try:
self.set_progress_percentage(progress)
except:
pass
for o in self.output():
done = o.open('w')
done.close()
def filter(h5filepath,
csv_file_src,
csv_file_tgt=None,
cleft_ds_name="syncleft_dist_thr0.0_cc"):
logging.info("Filtering clefts in {0:}/{1:} with {2:}".format(
h5filepath, cleft_ds_name, csv_file_src))
cf = CremiFile(h5filepath, "r+")
ann = cf.read_annotations()
cleft_to_pre, cleft_to_post = make_cleft_to_prepostsyn_neuron_id_dict(
csv_file_src)
cleft_list_verified = cleft_to_pre.keys()
logging.info("List of verified clefts:\n{0:}".format(cleft_list_verified))
cleft_ds = np.array(cf.read_volume(cleft_ds_name).data)
cleft_list_all = list(np.unique(cleft_ds))
for bg_id in BG_IDS:
cleft_list_all.remove(bg_id)
logging.info("List of all clefts:\n{0:}".format(cleft_list_all))
cleft_list_unmatched = list(set(cleft_list_all) - set(cleft_list_verified))
logging.info(
"List of unmatched clefts:\n{0:}".format(cleft_list_unmatched))
if csv_file_tgt is not None:
with open(csv_file_tgt, "w") as f:
writer = csv.writer(f)
for i in cleft_list_unmatched:
writer.writerow([i])
next_id = max(ann.ids()) + 1
logging.info("Adding annotations...")
for cleft_id in cleft_list_unmatched:
logging.info("... for cleft {0:}".format(cleft_id))
cleft_coords = np.where(cleft_ds == cleft_id)
cleft_center = (
40.0 * cleft_coords[0][int(len(cleft_coords[0]) / 2.0)],
4.0 * cleft_coords[1][int(len(cleft_coords[1]) / 2.0)],
4.0 * cleft_coords[2][int(len(cleft_coords[2]) / 2.0)],
)
ann.add_annotation(next_id, "synapse", cleft_center)
ann.add_comment(next_id, str(cleft_id))
next_id += 1
logging.info("Saving annotations...")
cf.write_annotations(ann)
cf.close()
logging.info("...done \n\n")
def remove_annotations_in_mask(filename, mask_filename, mask_ds):
fh = CremiFile(filename, 'a')
if mask_filename.endswith('.h5') or mask_filename.endswith('.hdf'):
maskfh = h5py.File(mask_filename, 'r')
else:
maskfh = z5py.File(mask_filename, use_zarr_format=False)
mask = maskfh[mask_ds]
off = mask.attrs['offset']
res = mask.attrs['resolution']
mask = mask[:]
ann = fh.read_annotations()
shift = sub(ann.offset, off)
ids = ann.ids()
rmids = []
for i in ids:
t, loc = ann.get_annotation(i)
vx_idx = (np.array(add(loc, shift)) / res).astype(np.int)
if not mask[tuple(vx_idx)]:
rmids.append(i)
for i in rmids:
print('removing {0:}'.format(i))
ann.remove_annotation(i)
fh.write_annotations(ann)
def remove_annotations_in_mask(filename, mask_filename, mask_ds):
fh = CremiFile(filename, "a")
if mask_filename.endswith(".h5") or mask_filename.endswith(".hdf"):
maskfh = h5py.File(mask_filename, "r")
else:
maskfh = zarr.open(mask_filename, mode="r")
mask = maskfh[mask_ds]
off = mask.attrs["offset"]
res = mask.attrs["resolution"]
mask = mask[:]
ann = fh.read_annotations()
shift = sub(ann.offset, off)
ids = ann.ids()
rmids = []
for i in ids:
t, loc = ann.get_annotation(i)
vx_idx = (np.array(add(loc, shift)) / res).astype(np.int)
if not mask[tuple(vx_idx)]:
rmids.append(i)
for i in rmids:
print("removing {0:}".format(i))
ann.remove_annotation(i)
fh.write_annotations(ann)
# Create some dummy annotation data
annotations = Annotations()
for id in [ 0, 1, 2, 3 ]:
location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100))
annotations.add_annotation(id, "presynaptic_site", location)
for id in [ 4, 5, 6, 7 ]:
location = (random.randint(0, 100), random.randint(0, 100), random.randint(0, 100))
annotations.add_annotation(id, "postsynaptic_site", location)
for (pre, post) in [ (0, 4), (1, 5), (2, 6), (3, 7) ]:
annotations.set_pre_post_partners(pre, post)
annotations.add_comment(6, "unsure")
# Open a file for writing (deletes previous file, if exists)
file = CremiFile("example.hdf", "w")
# Write the raw volume. This is given here just for illustration. For your
# submission, you don't need to store the raw data. We have it already.
raw = Volume(np.zeros((10,100,100), dtype=np.uint8), resolution=(40.0, 4.0, 4.0))
file.write_raw(raw)
# Write volumes representing the neuron and synaptic cleft segmentation.
neuron_ids = Volume(np.ones((10,100,100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just ones")
clefts = Volume(np.zeros((10,100,100), dtype=np.uint64), resolution=(40.0, 4.0, 4.0), comment="just zeros")
file.write_neuron_ids(neuron_ids)
file.write_clefts(clefts)
# Write synaptic partner annotations.
file.write_annotations(annotations)
file.close()
location = (random.randint(0, 100), random.randint(0, 100),
random.randint(0, 100))
annotations.add_annotation(id, "postsynaptic_site", location)
for (pre, post) in [(0, 4), (1, 5), (2, 6), (3, 7)]:
annotations.set_pre_post_partners(pre, post)
annotations.add_comment(6, "unsure")
# Open a file for writing (deletes previous file, if exists)
file = CremiFile("example.hdf", "w")
# Write the raw volume. This is given here just for illustration. For your
# submission, you don't need to store the raw data. We have it already.
raw = Volume(np.zeros((10, 100, 100), dtype=np.uint8),
resolution=(40.0, 4.0, 4.0))
file.write_raw(raw)
# Write volumes representing the neuron and synaptic cleft segmentation.
neuron_ids = Volume(np.ones((10, 100, 100), dtype=np.uint64),
resolution=(40.0, 4.0, 4.0),
comment="just ones")
clefts = Volume(np.zeros((10, 100, 100), dtype=np.uint64),
resolution=(40.0, 4.0, 4.0),
comment="just zeros")
file.write_neuron_ids(neuron_ids)
file.write_clefts(clefts)
# Write synaptic partner annotations.
file.write_annotations(annotations)
file.close()