def __get_syn_points(self, roi, syn_file, dataset_offset=None): presyn_points_dict, postsyn_points_dict = {}, {} presyn_node_ids = syn_file[ 'annotations/presynaptic_site/partners'][:, 0].tolist() postsyn_node_ids = syn_file[ 'annotations/presynaptic_site/partners'][:, 1].tolist() for node_nr, node_id in enumerate(syn_file['annotations/ids']): location = syn_file['annotations/locations'][node_nr] if dataset_offset is not None: logging.debug( 'adding global offset to points %i %i %i' % (dataset_offset[0], dataset_offset[1], dataset_offset[2])) location += dataset_offset # cremi synapse locations are in physical space if roi.contains(Coordinate(location)): if node_id in presyn_node_ids: kind = 'PreSyn' assert syn_file['annotations/types'][ node_nr] == 'presynaptic_site' syn_id = int(np.where(presyn_node_ids == node_id)[0]) partner_node_id = postsyn_node_ids[syn_id] elif node_id in postsyn_node_ids: kind = 'PostSyn' assert syn_file['annotations/types'][ node_nr] == 'postsynaptic_site' syn_id = int(np.where(postsyn_node_ids == node_id)[0]) partner_node_id = presyn_node_ids[syn_id] else: raise Exception('Node id neither pre- no post-synaptic') partners_ids = [int(partner_node_id)] location_id = int(node_id) props = {} if node_id in syn_file['annotations/comments/target_ids']: props = {'unsure': True} # create synpaseLocation & add to dict if kind == 'PreSyn': syn_point = PreSynPoint(location=location, location_id=location_id, synapse_id=syn_id, partner_ids=partners_ids, props=props) presyn_points_dict[int(node_id)] = copy.deepcopy(syn_point) elif kind == 'PostSyn': syn_point = PostSynPoint(location=location, location_id=location_id, synapse_id=syn_id, partner_ids=partners_ids, props=props) postsyn_points_dict[int(node_id)] = copy.deepcopy( syn_point) return presyn_points_dict, postsyn_points_dict
def provide(self, request): batch = Batch() roi_points = request[PointsKeys.PRESYN].roi trg_points = request[PointsKeys.POSTSYN].roi # get all pre points inside the requested ROI pre_points = {} post_points = {} syn_id = 0 for pre_id, post_id in self.partners: loc = self.points[pre_id] if roi_points.contains(loc): pre_point = PreSynPoint(location=loc, partner_ids=[post_id], location_id=pre_id, synapse_id=syn_id) pre_points[pre_id] = pre_point loc = self.points[post_id] if trg_points.contains(loc): post_point = PostSynPoint(location=loc, partner_ids=[pre_id], location_id=post_id, synapse_id=syn_id) post_points[post_id] = post_point syn_id += 1 batch.points[PointsKeys.PRESYN] = Points(pre_points, PointsSpec(roi=roi_points)) batch.points[PointsKeys.POSTSYN] = Points(post_points, PointsSpec(roi=trg_points)) if ArrayKeys.OBJECTMASK in request: roi_array = request[ArrayKeys.OBJECTMASK].roi spec = self.spec[ArrayKeys.OBJECTMASK].copy() spec.roi = roi_array batch.arrays[ArrayKeys.OBJECTMASK] = Array( self.objectmask[(roi_array / self.voxel_size).to_slices()], spec=ArraySpec(roi=roi_array, voxel_size=self.voxel_size)) return batch
def __get_syn_points(self, pre_roi, post_roi, syn_file): presyn_points_dict, postsyn_points_dict = {}, {} annotation_ids = syn_file['annotations/ids'][:] locs = syn_file['annotations/locations'][:] if 'offset' in syn_file['annotations'].attrs: offset = np.array(syn_file['annotations'].attrs['offset']) logger.debug("Retrieving offset") else: offset = None logger.debug('No offset') syn_id = 0 for pre, post in list( syn_file['annotations/presynaptic_site/partners'][:]): pre_index = int(np.where(pre == annotation_ids)[0][0]) post_index = int(np.where(post == annotation_ids)[0][0]) pre_site = locs[pre_index] post_site = locs[post_index] if offset is not None: pre_site += offset post_site += offset if pre_roi.contains(Coordinate(pre_site)): syn_point = PreSynPoint(location=pre_site, location_id=pre_index, synapse_id=syn_id, partner_ids=[post_index]) presyn_points_dict[pre_index] = copy.deepcopy(syn_point) if post_roi.contains(Coordinate(post_site)): syn_point = PostSynPoint(location=post_site, location_id=post_index, synapse_id=syn_id, partner_ids=[pre_index]) postsyn_points_dict[post_index] = copy.deepcopy(syn_point) if pre_roi.contains(Coordinate(pre_site)) or post_roi.contains( Coordinate(post_site)): syn_id += 1 return presyn_points_dict, postsyn_points_dict
def __read_syn_points(self, roi): """ read json file from dvid source, in json format to create a PreSynPoint/PostSynPoint for every location given """ if PointsKeys.PRESYN in self.points_voxel_size: voxel_size = self.points_voxel_size[PointsKeys.PRESYN] elif PointsKeys.POSTSYN in self.points_voxel_size: voxel_size = self.points_voxel_size[PointsKeys.POSTSYN] syn_file_json = self.__load_json_annotations(array_shape_voxel = roi.get_shape() // voxel_size, array_offset_voxel = roi.get_offset() // voxel_size, array_name = self.datasets[PointsKeys.PRESYN]) presyn_points_dict, postsyn_points_dict = {}, {} location_to_location_id_dict, location_id_to_partner_locations = {}, {} for node_nr, node in enumerate(syn_file_json): # collect information kind = str(node['Kind']) location = np.asarray((node['Pos'][2], node['Pos'][1], node['Pos'][0])) * voxel_size location_id = int(node_nr) # some synapses are wrongly annotated in dvid source, have 'Tag': null ???, they are skipped try: syn_id = int(node['Tags'][0][3:]) except: continue location_to_location_id_dict[str(location)] = location_id partner_locations = [] try: for relation in node['Rels']: partner_locations.append((np.asarray([relation['To'][2], relation['To'][1], relation['To'][0]]))*voxel_size) except: partner_locations = [] location_id_to_partner_locations[int(node_nr)] = partner_locations # check if property given, not always given props = {} if 'conf' in node['Prop']: props['conf'] = float(node['Prop']['conf']) if 'agent' in node['Prop']: props['agent'] = str(node['Prop']['agent']) if 'flagged' in node['Prop']: str_value_flagged = str(node['Prop']['flagged']) props['flagged'] = bool(distutils.util.strtobool(str_value_flagged)) if 'multi' in node['Prop']: str_value_multi = str(node['Prop']['multi']) props['multi'] = bool(distutils.util.strtobool(str_value_multi)) # create synPoint with information collected so far (partner_ids not completed yet) if kind == 'PreSyn': syn_point = PreSynPoint(location=location, location_id=location_id, synapse_id=syn_id, partner_ids=[], props=props) presyn_points_dict[int(node_nr)] = deepcopy(syn_point) elif kind == 'PostSyn': syn_point = PostSynPoint(location=location, location_id=location_id, synapse_id=syn_id, partner_ids=[], props=props) postsyn_points_dict[int(node_nr)] = deepcopy(syn_point) # add partner ids last_node_nr = len(syn_file_json)-1 for current_syn_point_id in location_id_to_partner_locations.keys(): all_partner_ids = [] for partner_loc in location_id_to_partner_locations[current_syn_point_id]: if location_to_location_id_dict.has_key(str(partner_loc)): all_partner_ids.append(int(location_to_location_id_dict[str(partner_loc)])) else: last_node_nr = last_node_nr + 1 assert not location_to_location_id_dict.has_key(str(partner_loc)) all_partner_ids.append(int(last_node_nr)) if current_syn_point_id in presyn_points_dict: presyn_points_dict[current_syn_point_id].partner_ids = all_partner_ids elif current_syn_point_id in postsyn_points_dict: postsyn_points_dict[current_syn_point_id].partner_ids = all_partner_ids else: raise Exception("current syn_point id not found in any dictionary") return presyn_points_dict, postsyn_points_dict
def __get_pre_and_postsyn_locations(self, roi): presyn_locs, postsyn_locs = {}, {} min_dist_between_presyn_locs = 250 voxel_size_points = self.spec[ArrayKeys.RAW].voxel_size min_dist_pre_to_postsyn_loc, max_dist_pre_to_postsyn_loc = 60, 120 num_presyn_locations = roi.size() // (np.prod( 50 * np.asarray(voxel_size_points))) # 1 synapse per 50vx^3 cube num_postsyn_locations = np.random.randint( low=1, high=3) # 1 to 3 postsyn partners loc_id = 0 all_presyn_locs = [] for nr_presyn_loc in range(num_presyn_locations): loc_id = loc_id + 1 presyn_loc_id = loc_id presyn_loc_too_close = True while presyn_loc_too_close: presyn_location = np.asarray([ np.random.randint(low=roi.get_begin()[0], high=roi.get_end()[0]), np.random.randint(low=roi.get_begin()[1], high=roi.get_end()[1]), np.random.randint(low=roi.get_begin()[2], high=roi.get_end()[2]) ]) # ensure that partner locations of diff presyn locations are not overlapping presyn_loc_too_close = False for previous_loc in all_presyn_locs: if np.linalg.norm(presyn_location - previous_loc) < ( min_dist_between_presyn_locs): presyn_loc_too_close = True syn_id = nr_presyn_loc partner_ids = [] for nr_partner_loc in range(num_postsyn_locations): loc_id = loc_id + 1 partner_ids.append(loc_id) postsyn_loc_is_inside = False while not postsyn_loc_is_inside: postsyn_location = presyn_location + np.random.choice((-1,1),size=3, replace=True) \ * np.random.randint(min_dist_pre_to_postsyn_loc, max_dist_pre_to_postsyn_loc, size=3) if roi.contains(Coordinate(postsyn_location)): postsyn_loc_is_inside = True postsyn_locs[int(loc_id)] = deepcopy( PostSynPoint(location=postsyn_location, location_id=loc_id, synapse_id=syn_id, partner_ids=[presyn_loc_id], props={})) presyn_locs[int(presyn_loc_id)] = deepcopy( PreSynPoint(location=presyn_location, location_id=presyn_loc_id, synapse_id=syn_id, partner_ids=partner_ids, props={})) return presyn_locs, postsyn_locs