def __get_syn_points(self, roi, syn_file, dataset_offset=None):
presyn_points_dict, postsyn_points_dict = {}, {}
presyn_node_ids = syn_file[
'annotations/presynaptic_site/partners'][:, 0].tolist()
postsyn_node_ids = syn_file[
'annotations/presynaptic_site/partners'][:, 1].tolist()
for node_nr, node_id in enumerate(syn_file['annotations/ids']):
location = syn_file['annotations/locations'][node_nr]
if dataset_offset is not None:
logging.debug(
'adding global offset to points %i %i %i' %
(dataset_offset[0], dataset_offset[1], dataset_offset[2]))
location += dataset_offset
# cremi synapse locations are in physical space
if roi.contains(Coordinate(location)):
if node_id in presyn_node_ids:
kind = 'PreSyn'
assert syn_file['annotations/types'][
node_nr] == 'presynaptic_site'
syn_id = int(np.where(presyn_node_ids == node_id)[0])
partner_node_id = postsyn_node_ids[syn_id]
elif node_id in postsyn_node_ids:
kind = 'PostSyn'
assert syn_file['annotations/types'][
node_nr] == 'postsynaptic_site'
syn_id = int(np.where(postsyn_node_ids == node_id)[0])
partner_node_id = presyn_node_ids[syn_id]
else:
raise Exception('Node id neither pre- no post-synaptic')
partners_ids = [int(partner_node_id)]
location_id = int(node_id)
props = {}
if node_id in syn_file['annotations/comments/target_ids']:
props = {'unsure': True}
# create synpaseLocation & add to dict
if kind == 'PreSyn':
syn_point = PreSynPoint(location=location,
location_id=location_id,
synapse_id=syn_id,
partner_ids=partners_ids,
props=props)
presyn_points_dict[int(node_id)] = copy.deepcopy(syn_point)
elif kind == 'PostSyn':
syn_point = PostSynPoint(location=location,
location_id=location_id,
synapse_id=syn_id,
partner_ids=partners_ids,
props=props)
postsyn_points_dict[int(node_id)] = copy.deepcopy(
syn_point)
return presyn_points_dict, postsyn_points_dict
def provide(self, request):
batch = Batch()
roi_points = request[PointsKeys.PRESYN].roi
trg_points = request[PointsKeys.POSTSYN].roi
# get all pre points inside the requested ROI
pre_points = {}
post_points = {}
syn_id = 0
for pre_id, post_id in self.partners:
loc = self.points[pre_id]
if roi_points.contains(loc):
pre_point = PreSynPoint(location=loc,
partner_ids=[post_id],
location_id=pre_id,
synapse_id=syn_id)
pre_points[pre_id] = pre_point
loc = self.points[post_id]
if trg_points.contains(loc):
post_point = PostSynPoint(location=loc,
partner_ids=[pre_id],
location_id=post_id,
synapse_id=syn_id)
post_points[post_id] = post_point
syn_id += 1
batch.points[PointsKeys.PRESYN] = Points(pre_points,
PointsSpec(roi=roi_points))
batch.points[PointsKeys.POSTSYN] = Points(post_points,
PointsSpec(roi=trg_points))
if ArrayKeys.OBJECTMASK in request:
roi_array = request[ArrayKeys.OBJECTMASK].roi
spec = self.spec[ArrayKeys.OBJECTMASK].copy()
spec.roi = roi_array
batch.arrays[ArrayKeys.OBJECTMASK] = Array(
self.objectmask[(roi_array / self.voxel_size).to_slices()],
spec=ArraySpec(roi=roi_array, voxel_size=self.voxel_size))
return batch
def __get_syn_points(self, pre_roi, post_roi, syn_file):
presyn_points_dict, postsyn_points_dict = {}, {}
annotation_ids = syn_file['annotations/ids'][:]
locs = syn_file['annotations/locations'][:]
if 'offset' in syn_file['annotations'].attrs:
offset = np.array(syn_file['annotations'].attrs['offset'])
logger.debug("Retrieving offset")
else:
offset = None
logger.debug('No offset')
syn_id = 0
for pre, post in list(
syn_file['annotations/presynaptic_site/partners'][:]):
pre_index = int(np.where(pre == annotation_ids)[0][0])
post_index = int(np.where(post == annotation_ids)[0][0])
pre_site = locs[pre_index]
post_site = locs[post_index]
if offset is not None:
pre_site += offset
post_site += offset
if pre_roi.contains(Coordinate(pre_site)):
syn_point = PreSynPoint(location=pre_site,
location_id=pre_index,
synapse_id=syn_id,
partner_ids=[post_index])
presyn_points_dict[pre_index] = copy.deepcopy(syn_point)
if post_roi.contains(Coordinate(post_site)):
syn_point = PostSynPoint(location=post_site,
location_id=post_index,
synapse_id=syn_id,
partner_ids=[pre_index])
postsyn_points_dict[post_index] = copy.deepcopy(syn_point)
if pre_roi.contains(Coordinate(pre_site)) or post_roi.contains(
Coordinate(post_site)):
syn_id += 1
return presyn_points_dict, postsyn_points_dict
def __read_syn_points(self, roi):
""" read json file from dvid source, in json format to create a PreSynPoint/PostSynPoint for every location given """
if PointsKeys.PRESYN in self.points_voxel_size:
voxel_size = self.points_voxel_size[PointsKeys.PRESYN]
elif PointsKeys.POSTSYN in self.points_voxel_size:
voxel_size = self.points_voxel_size[PointsKeys.POSTSYN]
syn_file_json = self.__load_json_annotations(array_shape_voxel = roi.get_shape() // voxel_size,
array_offset_voxel = roi.get_offset() // voxel_size,
array_name = self.datasets[PointsKeys.PRESYN])
presyn_points_dict, postsyn_points_dict = {}, {}
location_to_location_id_dict, location_id_to_partner_locations = {}, {}
for node_nr, node in enumerate(syn_file_json):
# collect information
kind = str(node['Kind'])
location = np.asarray((node['Pos'][2], node['Pos'][1], node['Pos'][0])) * voxel_size
location_id = int(node_nr)
# some synapses are wrongly annotated in dvid source, have 'Tag': null ???, they are skipped
try:
syn_id = int(node['Tags'][0][3:])
except:
continue
location_to_location_id_dict[str(location)] = location_id
partner_locations = []
try:
for relation in node['Rels']:
partner_locations.append((np.asarray([relation['To'][2], relation['To'][1], relation['To'][0]]))*voxel_size)
except:
partner_locations = []
location_id_to_partner_locations[int(node_nr)] = partner_locations
# check if property given, not always given
props = {}
if 'conf' in node['Prop']:
props['conf'] = float(node['Prop']['conf'])
if 'agent' in node['Prop']:
props['agent'] = str(node['Prop']['agent'])
if 'flagged' in node['Prop']:
str_value_flagged = str(node['Prop']['flagged'])
props['flagged'] = bool(distutils.util.strtobool(str_value_flagged))
if 'multi' in node['Prop']:
str_value_multi = str(node['Prop']['multi'])
props['multi'] = bool(distutils.util.strtobool(str_value_multi))
# create synPoint with information collected so far (partner_ids not completed yet)
if kind == 'PreSyn':
syn_point = PreSynPoint(location=location, location_id=location_id,
synapse_id=syn_id, partner_ids=[], props=props)
presyn_points_dict[int(node_nr)] = deepcopy(syn_point)
elif kind == 'PostSyn':
syn_point = PostSynPoint(location=location, location_id=location_id,
synapse_id=syn_id, partner_ids=[], props=props)
postsyn_points_dict[int(node_nr)] = deepcopy(syn_point)
# add partner ids
last_node_nr = len(syn_file_json)-1
for current_syn_point_id in location_id_to_partner_locations.keys():
all_partner_ids = []
for partner_loc in location_id_to_partner_locations[current_syn_point_id]:
if location_to_location_id_dict.has_key(str(partner_loc)):
all_partner_ids.append(int(location_to_location_id_dict[str(partner_loc)]))
else:
last_node_nr = last_node_nr + 1
assert not location_to_location_id_dict.has_key(str(partner_loc))
all_partner_ids.append(int(last_node_nr))
if current_syn_point_id in presyn_points_dict:
presyn_points_dict[current_syn_point_id].partner_ids = all_partner_ids
elif current_syn_point_id in postsyn_points_dict:
postsyn_points_dict[current_syn_point_id].partner_ids = all_partner_ids
else:
raise Exception("current syn_point id not found in any dictionary")
return presyn_points_dict, postsyn_points_dict
def __get_pre_and_postsyn_locations(self, roi):
presyn_locs, postsyn_locs = {}, {}
min_dist_between_presyn_locs = 250
voxel_size_points = self.spec[ArrayKeys.RAW].voxel_size
min_dist_pre_to_postsyn_loc, max_dist_pre_to_postsyn_loc = 60, 120
num_presyn_locations = roi.size() // (np.prod(
50 * np.asarray(voxel_size_points))) # 1 synapse per 50vx^3 cube
num_postsyn_locations = np.random.randint(
low=1, high=3) # 1 to 3 postsyn partners
loc_id = 0
all_presyn_locs = []
for nr_presyn_loc in range(num_presyn_locations):
loc_id = loc_id + 1
presyn_loc_id = loc_id
presyn_loc_too_close = True
while presyn_loc_too_close:
presyn_location = np.asarray([
np.random.randint(low=roi.get_begin()[0],
high=roi.get_end()[0]),
np.random.randint(low=roi.get_begin()[1],
high=roi.get_end()[1]),
np.random.randint(low=roi.get_begin()[2],
high=roi.get_end()[2])
])
# ensure that partner locations of diff presyn locations are not overlapping
presyn_loc_too_close = False
for previous_loc in all_presyn_locs:
if np.linalg.norm(presyn_location - previous_loc) < (
min_dist_between_presyn_locs):
presyn_loc_too_close = True
syn_id = nr_presyn_loc
partner_ids = []
for nr_partner_loc in range(num_postsyn_locations):
loc_id = loc_id + 1
partner_ids.append(loc_id)
postsyn_loc_is_inside = False
while not postsyn_loc_is_inside:
postsyn_location = presyn_location + np.random.choice((-1,1),size=3, replace=True) \
* np.random.randint(min_dist_pre_to_postsyn_loc, max_dist_pre_to_postsyn_loc, size=3)
if roi.contains(Coordinate(postsyn_location)):
postsyn_loc_is_inside = True
postsyn_locs[int(loc_id)] = deepcopy(
PostSynPoint(location=postsyn_location,
location_id=loc_id,
synapse_id=syn_id,
partner_ids=[presyn_loc_id],
props={}))
presyn_locs[int(presyn_loc_id)] = deepcopy(
PreSynPoint(location=presyn_location,
location_id=presyn_loc_id,
synapse_id=syn_id,
partner_ids=partner_ids,
props={}))
return presyn_locs, postsyn_locs