def _get_next_shift(self, center_shift, voxel_size): # gets next coordinate from list if self.choose_randomly: self.loc_i = randrange(len(self.coordinates)) else: self.loc_i += 1 if self.loc_i >= len(self.coordinates): self.loc_i = 0 logger.warning('Ran out of specified locations, looping list') next_shift = Coordinate(self.coordinates[self.loc_i]) - center_shift if self.jitter is not None: rnd = [] for i in range(len(self.jitter)): rnd.append( np.random.randint(-self.jitter[i], self.jitter[i] + 1)) next_shift += Coordinate(rnd) logger.debug("Shift before rounding: %s" % str(next_shift)) # make sure shift is a multiple of voxel size (round to nearest) next_shift = Coordinate([ int(vs * round(float(shift) / vs)) for vs, shift in zip(voxel_size, next_shift) ]) logger.debug("Shift after rounding: %s" % str(next_shift)) return next_shift
def prepare(self, request): if self.settings.mode == 'ball': context = np.ceil(self.settings.radius).astype(np.int) elif self.settings.mode == 'peak': context = np.ceil(2*self.settings.radius).astype(np.int) else: raise RuntimeError('unknown raster mode %s'%self.settings.mode) dims = self.array_spec.roi.dims() if len(context) == 1: context = context.repeat(dims) # request graph in a larger area to get rasterization from outside # graph graph_roi = request[self.array].roi.grow( Coordinate(context), Coordinate(context)) # however, restrict the request to the graph actually provided graph_roi = graph_roi.intersect(self.spec[self.graph].roi) deps = BatchRequest() deps[self.graph] = GraphSpec(roi=graph_roi) if self.settings.mask is not None: mask_voxel_size = self.spec[self.settings.mask].voxel_size assert self.spec[self.array].voxel_size == mask_voxel_size, ( "Voxel size of mask and rasterized volume need to be equal") new_mask_roi = graph_roi.snap_to_grid(mask_voxel_size) deps[self.settings.mask] = ArraySpec(roi=new_mask_roi) return deps
def _query_kdtree(self, node: cKDTreeNode, bb: Tuple[np.ndarray, np.ndarray]) -> List[np.ndarray]: def substitute_dim(bound: np.ndarray, sub_dim: int, sub: float): # replace bound[sub_dim] with sub return np.array( [bound[i] if i != sub_dim else sub for i in range(3)]) if node is None: return [] if node.split_dim != -1: # if the split location is above the roi, no need to split (cannot be above None) if bb[1][node.split_dim] is not None and node.split > bb[1][ node.split_dim]: return self._query_kdtree(node.lesser, bb) elif (bb[0][node.split_dim] is not None and node.split < bb[0][node.split_dim]): return self._query_kdtree(node.greater, bb) else: return self._query_kdtree( node.greater, bb) + self._query_kdtree(node.lesser, bb) else: # handle leaf node bbox = Roi( Coordinate(bb[0]), Coordinate( tuple(y - x if x is not None and y is not None else y for x, y in zip(*bb))), ) points = [ ind for ind, point in zip(node.indices, node.data_points) if bbox.contains(np.round(point)) ] return points
def setup(self): assert self.labels in self.spec, ( "Upstream does not provide %s needed by " "AddAffinities"%self.labels) voxel_size = self.spec[self.labels].voxel_size dims = self.affinity_neighborhood.shape[1] self.padding_neg = Coordinate( min([0] + [a[d] for a in self.affinity_neighborhood]) for d in range(dims) )*voxel_size self.padding_pos = Coordinate( max([0] + [a[d] for a in self.affinity_neighborhood]) for d in range(dims) )*voxel_size logger.debug("padding neg: " + str(self.padding_neg)) logger.debug("padding pos: " + str(self.padding_pos)) spec = self.spec[self.labels].copy() if spec.roi is not None: spec.roi = spec.roi.grow(self.padding_neg, -self.padding_pos) spec.dtype = np.float32 self.provides(self.affinities, spec) if self.affinities_mask: self.provides(self.affinities_mask, spec) self.enable_autoskip()
def __transpose_roi(self, roi, total_roi, transpose, lcm_voxel_size): logger.debug("original roi = %s", roi) center = total_roi.get_center() if lcm_voxel_size is not None: nearest_voxel_shift = Coordinate( (d % v) for d, v in zip(center, lcm_voxel_size)) center = center - nearest_voxel_shift logger.debug("center = %s", center) # Get distance from center, then transpose dist_to_center = center - roi.get_offset() dist_to_center = Coordinate(dist_to_center[transpose[d]] for d in range(self.dims)) logger.debug("dist_to_center = %s", dist_to_center) # Using the tranposed distance to center, get the correct offset. new_offset = center - dist_to_center logger.debug("new_offset = %s", new_offset) shape = tuple(roi.get_shape()[transpose[d]] for d in range(self.dims)) roi.set_offset(new_offset) roi.set_shape(shape) logger.debug("tranposed roi = %s", roi)
def process(self, batch, request): # create vector map and add it to batch for (array_key, (src_points_key, trg_points_key)) in self.array_to_src_trg_points.items(): if array_key in request: vector_map = self.__get_vector_map( batch=batch, request=request, vector_map_array_key=array_key) spec = self.spec[array_key].copy() spec.roi = request[array_key].roi batch.arrays[array_key] = Array(data=vector_map, spec=spec) # restore request / remove not requested points in padding-for-neighbors region & shrink batch roi for (array_key, (src_points_key, trg_points_key)) in self.array_to_src_trg_points.items(): if array_key in request: if trg_points_key in request: for loc_id, point in batch.points[ trg_points_key].data.items(): if not request[trg_points_key].roi.contains( Coordinate(point.location)): del batch.points[trg_points_key].data[loc_id] neg_pad_for_partners = Coordinate( (self.pad_for_partners * np.asarray([-1])).tolist()) batch.points[trg_points_key].spec.roi = batch.points[ trg_points_key].spec.roi.grow(neg_pad_for_partners, neg_pad_for_partners) elif trg_points_key in batch.points: del batch.points[trg_points_key]
def shift_points(points, request_roi, sub_shift_array, shift_axis, lcm_voxel_size): """ Shift a set of points received from upstream and crop out those not the the target downstream region :param points: the points from upstream :param request_roi: the downstream ROI :param sub_shift_array: the cropped section of the global shift array that applies to this specific request :param shift_axis: the axis to perform the shift along :param lcm_voxel_size: the least common voxel size for the arrays in the request :return a Points object with the updated point locations and ROI """ data = points.data spec = points.spec shift_axis_start_pos = spec.roi.get_offset()[shift_axis] shifted_data = {} for id_, point in data.items(): loc = Coordinate(point.location) shift_axis_position = loc[shift_axis] shift_array_index = (shift_axis_position - shift_axis_start_pos ) // lcm_voxel_size[shift_axis] assert (shift_array_index >= 0) shift = Coordinate(sub_shift_array[shift_array_index]) new_loc = loc + shift if request_roi.contains(new_loc): point.location = new_loc shifted_data[id_] = point points.data = shifted_data points.spec.roi = request_roi return points
def _resample_relative( self, inside: np.ndarray, outside: np.ndarray, bb: Roi ) -> Optional[np.ndarray]: offset = outside - inside with np.errstate(divide="ignore", invalid="ignore"): # bb_crossings will be 0 if inside is on the bb, 1 if outside is on the bb bb_x = np.asarray( [ (np.asarray(bb.get_begin()) - inside) / offset, (np.asarray(bb.get_end() - Coordinate([1, 1, 1])) - inside) / offset, ] ) if np.sum(np.logical_and((bb_x > 0), (bb_x <= 1))) > 0: # all values of bb_x between 0, 1 represent a crossing of a bounding plane # the minimum of which is the (normalized) distance to the closest bounding plane s = np.min(bb_x[np.logical_and((bb_x > 0), (bb_x <= 1))]) return Coordinate(np.floor(np.array(inside) + s * offset)) else: logging.debug( ( "Could not create a node on the bounding box {} " + "given points (inside:{}, ouside:{})" ).format(bb, inside, outside) ) return None
def __get_spec(self, array_key): info = self.__get_info(array_key) roi_min = info['Extended']['MinPoint'] if roi_min is not None: roi_min = Coordinate(roi_min[::-1]) roi_max = info['Extended']['MaxPoint'] if roi_max is not None: roi_max = Coordinate(roi_max[::-1]) data_roi = Roi(offset=roi_min, shape=(roi_max - roi_min)) data_dims = Coordinate(data_roi.get_shape()) if self.ndims is None: self.ndims = len(data_dims) else: assert self.ndims == len(data_dims) if array_key in self.array_specs: spec = self.array_specs[array_key].copy() else: spec = ArraySpec() if spec.voxel_size is None: spec.voxel_size = Coordinate(info['Extended']['VoxelSize']) if spec.roi is None: spec.roi = data_roi * spec.voxel_size data_dtype = dvision.DVIDDataInstance(self.hostname, self.port, self.uuid, self.datasets[array_key]).dtype if spec.dtype is not None: assert spec.dtype == data_dtype, ( "dtype %s provided in array_specs for %s, " "but differs from instance %s dtype %s" % (self.array_specs[array_key].dtype, array_key, self.datasets[array_key], data_dtype)) else: spec.dtype = data_dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning( "WARNING: You didn't set 'interpolatable' for %s. " "Based on the dtype %s, it has been set to %s. " "This might not be what you want.", array_key, spec.dtype, spec.interpolatable) return spec
def __read_spec(self, array_key, data_file, ds_name): dataset = data_file[ds_name] if array_key in self.array_specs: spec = self.array_specs[array_key].copy() else: spec = ArraySpec() if spec.voxel_size is None: voxel_size = self._get_voxel_size(dataset) if voxel_size is None: voxel_size = Coordinate((1, ) * len(dataset.shape)) logger.warning( "WARNING: File %s does not contain resolution information " "for %s (dataset %s), voxel size has been set to %s. This " "might not be what you want.", self.filename, array_key, ds_name, spec.voxel_size) spec.voxel_size = voxel_size self.ndims = len(spec.voxel_size) if spec.roi is None: offset = self._get_offset(dataset) if offset is None: offset = Coordinate((0, ) * self.ndims) if self.channels_first: shape = Coordinate(dataset.shape[-self.ndims:]) else: shape = Coordinate(dataset.shape[:self.ndims]) spec.roi = Roi(offset, shape * spec.voxel_size) if spec.dtype is not None: assert spec.dtype == dataset.dtype, ( "dtype %s provided in array_specs for %s, " "but differs from dataset %s dtype %s" % (self.array_specs[array_key].dtype, array_key, ds_name, dataset.dtype)) else: spec.dtype = dataset.dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning( "WARNING: You didn't set 'interpolatable' for %s " "(dataset %s). Based on the dtype %s, it has been " "set to %s. This might not be what you want.", array_key, ds_name, spec.dtype, spec.interpolatable) return spec
def __read_spec(self, cv): # dataset = data_file[ds_name] dims = Coordinate(cv.bounds.maxpt[::-1]) * self._get_voxel_size(cv) if self.ndims is None: self.ndims = cv.bounds.ndim else: assert self.ndims == len(dims) if self.array_spec is not None: spec = self.array_spec.copy() else: spec = ArraySpec() if spec.voxel_size is None: voxel_size = self._get_voxel_size(cv) if voxel_size is None: voxel_size = Coordinate((1,) * self.ndims) logger.warning( "WARNING: File %s does not contain resolution information " "for %s , voxel size has been set to %s. This " "might not be what you want.", self.filename, array_key, spec.voxel_size) spec.voxel_size = voxel_size if spec.roi is None: offset = self._get_offset(cv) if offset is None: offset = Coordinate((0,) * self.ndims) spec.roi = Roi(offset, dims * spec.voxel_size) if spec.dtype is not None: assert spec.dtype == cv.dtype, ( "dtype %s provided in array_specs for %s, " "but differs from cloudvolume dtype %s" % (self.array_spec.dtype, self.array_key, dataset.dtype)) else: spec.dtype = cv.dtype if spec.interpolatable is None: spec.interpolatable = spec.dtype in [ np.float, np.float32, np.float64, np.float128, np.uint8 # assuming this is not used for labels ] logger.warning("WARNING: You didn't set 'interpolatable' for %s " ". Based on the dtype %s, it has been " "set to %s. This might not be what you want.", self.array_key, spec.dtype, spec.interpolatable) return spec
def _get_offset(self, dataset): if 'offset' not in dataset.attrs: return None if self.output_filename.endswith('.n5'): return Coordinate(dataset.attrs['offset'][::-1]) else: return Coordinate(dataset.attrs['offset'])
def _get_voxel_size(self, dataset): if 'resolution' not in dataset.attrs: return None if self.output_filename.endswith('.n5'): return Coordinate(dataset.attrs['resolution'][::-1]) else: return Coordinate(dataset.attrs['resolution'])
def guarantee_nonempty(pipeline, setup_config, key): voxel_size = Coordinate(setup_config["VOXEL_SIZE"]) output_size = Coordinate(setup_config["OUTPUT_SHAPE"]) * voxel_size num_components = setup_config["NUM_COMPONENTS"] pipeline = pipeline + RejectIfEmpty( key, centroid_size=output_size, num_components=num_components) return pipeline
def __get_roi(self, array_name): data_instance = dvision.DVIDDataInstance(self.hostname, self.port, self.uuid, array_name) info = data_instance.info roi_min = info['Extended']['MinPoint'] if roi_min is not None: roi_min = Coordinate(roi_min[::-1]) roi_max = info['Extended']['MaxPoint'] if roi_max is not None: roi_max = Coordinate(roi_max[::-1]) return Roi(offset=roi_min, shape=roi_max - roi_min)
def __init__( self, dbname: str, url: str, points: List[GraphKey], graph_specs: Optional[Union[GraphSpec, List[GraphSpec]]] = None, directed: bool = False, total_roi: Roi = None, nodes_collection: str = "nodes", edges_collection: str = "edges", meta_collection: str = "meta", endpoint_names: Tuple[str, str] = ("u", "v"), position_attribute: str = "position", node_attrs: Optional[List[str]] = None, edge_attrs: Optional[List[str]] = None, nodes_filter: Optional[Dict[str, Any]] = None, edges_filter: Optional[Dict[str, Any]] = None, edge_inclusion: str = "either", node_inclusion: str = "dangling", fail_on_inconsistent_node: bool = False, ): self.points = points graph_specs = (graph_specs if graph_specs is not None else GraphSpec( Roi(Coordinate([None] * 3), Coordinate([None] * 3)), directed=False)) specs = (graph_specs if isinstance(graph_specs, list) and len(graph_specs) == len(points) else [graph_specs] * len(points)) self.specs = {key: spec for key, spec in zip(points, specs)} self.directed = directed self.nodes_collection = nodes_collection self.edges_collection = edges_collection self.meta_collection = meta_collection self.endpoint_names = endpoint_names self.position_attribute = position_attribute self.position_attribute = position_attribute self.node_attrs = node_attrs self.edge_attrs = edge_attrs self.nodes_filter = nodes_filter self.edges_filter = edges_filter self.edge_inclusion = edge_inclusion self.node_inclusion = node_inclusion self.dbname = dbname self.url = url self.nodes_collection = nodes_collection self.fail_on_inconsistent_node = fail_on_inconsistent_node self.graph_provider = None
def __init__( self, dbname: str, url: str, points: List[GraphKey], graph_specs: Optional[Union[GraphSpec, List[GraphSpec]]] = None, directed: bool = False, total_roi: Roi = None, nodes_collection: str = "nodes", edges_collection: str = "edges", meta_collection: str = "meta", endpoint_names: Tuple[str, str] = ("u", "v"), position_attribute: str = "position", node_attrs: Optional[List[str]] = None, edge_attrs: Optional[List[str]] = None, nodes_filter: Optional[Dict[str, Any]] = None, edges_filter: Optional[Dict[str, Any]] = None, num_nodes=100000, dist_attribute=None, min_dist=29000, ): self.points = points graph_specs = (graph_specs if graph_specs is not None else GraphSpec( Roi(Coordinate([None] * 3), Coordinate([None] * 3)), directed=False)) specs = (graph_specs if isinstance(graph_specs, list) and len(graph_specs) == len(points) else [graph_specs] * len(points)) self.specs = {key: spec for key, spec in zip(points, specs)} self.position_attribute = position_attribute self.node_attrs = node_attrs self.edge_attrs = edge_attrs self.nodes_filter = nodes_filter self.edges_filter = edges_filter self.dist_attribute = dist_attribute self.min_dist = min_dist self.num_nodes = num_nodes self.graph_provider = MongoDbGraphProvider( dbname, url, mode="r+", directed=directed, total_roi=None, nodes_collection=nodes_collection, edges_collection=edges_collection, meta_collection=meta_collection, endpoint_names=endpoint_names, position_attribute=position_attribute, )
def provide(self, batch_spec): logger.info("batch with spec " + str(batch_spec) + " requested") stride = self.chunk_spec_template.output_roi.get_shape() begin = batch_spec.input_roi.get_begin() end = batch_spec.input_roi.get_end() batch = None offset = np.array(begin) while (offset < end).all(): # create a copy of the requested batch spec chunk_spec = copy.deepcopy(batch_spec) # change size and offset of the batch spec chunk_spec.input_roi = self.chunk_spec_template.input_roi + Coordinate( offset) chunk_spec.output_roi = self.chunk_spec_template.output_roi + Coordinate( offset) logger.info("requesting chunk " + str(chunk_spec)) # get a chunk chunk = self.get_upstream_provider().request_batch(chunk_spec) if batch is None: batch = self.__setup_batch(batch_spec, chunk) for (volume_type, volume) in chunk.volumes: # input roi for RAW, output roi for others if volume_type == VolumeType.RAW: self.__fill(batch[volume_type].data, volume.data, batch_spec.input_roi, chunk.spec.input_roi) else: self.__fill(batch[volume_type].data, volume.data, batch_spec.output_roi, chunk.spec.output_roi) for d in range(self.dims): offset[d] += stride[d] if offset[d] >= end[d]: if d == self.dims - 1: break offset[d] = begin[d] else: break return batch
def __get_source_roi(self, transformation): dims = transformation.shape[0] # get bounding box of needed data for transformation bb_min = Coordinate( int(math.floor(transformation[d].min())) for d in range(dims)) bb_max = Coordinate( int(math.ceil(transformation[d].max())) + 1 for d in range(dims)) # create roi sufficiently large to feed transformation source_roi = Roi(bb_min, bb_max - bb_min) return source_roi
def setup(self): self._read_points() if self.points_spec is not None: self.provides(self.points, self.points_spec) return min_bb = Coordinate(np.floor(np.amin(self.data[:, :self.ndims], 0))) max_bb = Coordinate(np.ceil(np.amax(self.data[:, :self.ndims], 0)) + 1) roi = Roi(min_bb, max_bb - min_bb) self.provides(self.points, GraphSpec(roi=roi))
def compute_upstream_roi(request_roi, sub_shift_array): """ Compute the ROI to pass upstream for a specific item (array or points) in a request :param request_roi: the downstream ROI passed to the Jitter node :param sub_shift_array: the portion of the global shift array that should be used to shift the item :return: the expanded ROI to pass upstream """ max_shift = Coordinate(sub_shift_array.max(axis=0)) min_shift = Coordinate(sub_shift_array.min(axis=0)) downstream_offset = request_roi.get_offset() upstream_offset = downstream_offset - max_shift upstream_shape = request_roi.get_shape() + max_shift - min_shift return Roi(offset=upstream_offset, shape=upstream_shape)
def get_lcm_voxel_size(self, array_keys=None): '''Get the least common multiple of the voxel sizes in this spec. Args: array_keys (list of :class:`ArrayKey`, optional): If given, consider only the given array types. ''' if array_keys is None: array_keys = self.array_specs.keys() if not array_keys: raise RuntimeError("Can not compute lcm voxel size -- there are " "no array specs in this provider spec.") else: if not array_keys: raise RuntimeError("Can not compute lcm voxel size -- list of " "given array specs is empty.") lcm_voxel_size = None for key in array_keys: voxel_size = self.array_specs[key].voxel_size if lcm_voxel_size is None: lcm_voxel_size = voxel_size else: lcm_voxel_size = Coordinate( (a * b // fractions.gcd(a, b) for a, b in zip(lcm_voxel_size, voxel_size))) return lcm_voxel_size
def setup(self): self.data = self.read_points(self.filename) self.ndims = self.data.shape[1] if self.points_spec is not None: self.provides(self.points, self.points_spec) return min_bb = Coordinate(np.floor(np.amin(self.data, 0))) max_bb = Coordinate(np.ceil(np.amax(self.data, 0))) roi = Roi(min_bb, max_bb - min_bb) self.provides(self.points, PointsSpec(roi=roi))
def get_lcm_voxel_size(self, array_keys=None): '''Get the least common multiple of the voxel sizes in this spec. Args: array_keys (list of :class:`ArrayKey`, optional): If given, consider only the given array types. ''' if array_keys is None: array_keys = self.array_specs.keys() if not array_keys: return None lcm_voxel_size = None for key in array_keys: voxel_size = self.array_specs[key].voxel_size if voxel_size is None: continue if lcm_voxel_size is None: lcm_voxel_size = voxel_size else: lcm_voxel_size = Coordinate( (a * b // math.gcd(a, b) for a, b in zip(lcm_voxel_size, voxel_size))) return lcm_voxel_size
def shift_points(points, request_roi, sub_shift_array, shift_axis, lcm_voxel_size): """ Shift a set of points received from upstream and crop out those not the the target downstream region :param points: the points from upstream :param request_roi: the downstream ROI :param sub_shift_array: the cropped section of the global shift array that applies to this specific request :param shift_axis: the axis to perform the shift along :param lcm_voxel_size: the least common voxel size for the arrays in the request :return a Graph object with the updated point locations and ROI """ nodes = list(points.nodes) spec = points.spec shift_axis_start_pos = spec.roi.get_offset()[shift_axis] for node in nodes: loc = node.location shift_axis_position = loc[shift_axis] shift_array_index = int((shift_axis_position - shift_axis_start_pos) // lcm_voxel_size[shift_axis]) assert(shift_array_index >= 0) shift = Coordinate(sub_shift_array[shift_array_index]) loc += shift if not request_roi.contains(loc): points.remove_node(node) points.spec.roi = request_roi return points
def prepare(self, request): logger.debug("request: %s", request.array_specs) logger.debug("my spec: %s", self.spec) shift_roi = self.__get_possible_shifts(request) if request.array_specs.keys(): lcm_voxel_size = self.spec.get_lcm_voxel_size( request.array_specs.keys()) lcm_shift_roi = shift_roi/lcm_voxel_size logger.debug("lcm voxel size: %s", lcm_voxel_size) logger.debug( "restricting random locations to multiples of voxel size %s", lcm_voxel_size) else: lcm_voxel_size = Coordinate((1,)*shift_roi.dims()) lcm_shift_roi = shift_roi random_shift = self.__select_random_shift( request, lcm_shift_roi, lcm_voxel_size) self.random_shift = random_shift self.__shift_request(request, random_shift)
def __mirror_roi(self, roi, total_roi, mirror): total_roi_offset = total_roi.get_offset() total_roi_shape = total_roi.get_shape() roi_offset = roi.get_offset() roi_shape = roi.get_shape() roi_in_total_offset = roi_offset - total_roi_offset end_of_roi_in_total = roi_in_total_offset + roi_shape roi_in_total_offset_mirrored = total_roi_shape - end_of_roi_in_total roi_offset = Coordinate( total_roi_offset[d] + roi_in_total_offset_mirrored[d] if mirror[d] else roi_offset[d] for d in range(self.dims)) logger.debug("Mirror numbers for roi: " + str(roi) + "\nMirror: " + str(mirror) + "\tTranpose: " + str(self.transpose) + "\ntotal roi: " + str(total_roi) + "\nroi_in_total_offset: " + str(roi_in_total_offset) + "\nend_of_roi_in_total: " + str(end_of_roi_in_total) + "\nroi_in_total_offset_mirrored: " + str(roi_in_total_offset_mirrored) + "\nroi_offset: " + str(roi_offset)) roi.set_offset(roi_offset)
def check_batch_consistency(self, batch, request): for (array_key, request_spec) in request.array_specs.items(): assert array_key in batch.arrays, "%s requested, but %s did not provide it." % ( array_key, self.name()) array = batch.arrays[array_key] assert array.spec.roi == request_spec.roi, "%s ROI %s requested, but ROI %s provided by %s." % ( array_key, request_spec.roi, array.spec.roi, self.name()) assert array.spec.voxel_size == self.spec[array_key].voxel_size, ( "voxel size of %s announced, but %s " "delivered for %s" % (self.spec[array_key].voxel_size, array.spec.voxel_size, array_key)) # ensure that the spatial dimensions are the same (other dimensions # on top are okay, e.g., for affinities) dims = request_spec.roi.dims() data_shape = Coordinate(array.data.shape[-dims:]) voxel_size = self.spec[array_key].voxel_size assert data_shape == request_spec.roi.get_shape( ) / voxel_size, "%s ROI %s requested, but size of array is %s*%s=%s provided by %s." % ( array_key, request_spec.roi, data_shape, voxel_size, data_shape * voxel_size, self.name()) for (points_key, request_spec) in request.points_specs.items(): assert points_key in batch.points, "%s requested, but %s did not provide it." % ( points_key, self.name()) points = batch.points[points_key] assert points.spec.roi == request_spec.roi, "%s ROI %s requested, but ROI %s provided by %s." % ( points_key, request_spec.roi, points.spec.roi, self.name()) for _, point in points.data.items(): assert points.spec.roi.contains(point.location), ( "points provided by %s with ROI %s contain point at %s" % (self.name(), points.spec.roi, point.location))
def process(self, batch, request): output = Batch() for in_key, out_key in zip(self.arrays, self.output_arrays): array = batch[in_key] data = array.data d_min = data.min() d_max = data.max() assert ( d_min >= 0 and d_max <= 1 ), f"Clahe expects data in range (0,1), got ({d_min}, {d_max})" if np.isclose(d_max, d_min): output[out_key] = Array(data, array.spec) continue if self.normalize: data = (data - d_min) / (d_max - d_min) shape = data.shape data_dims = len(shape) kernel_dims = len(self.kernel_size) extra_dims = data_dims - kernel_dims voxel_size = array.spec.voxel_size for index in itertools.product(*[range(s) for s in shape[:extra_dims]]): data[index] = clahe( data[index], kernel_size=Coordinate(self.kernel_size / voxel_size), clip_limit=self.clip_limit, nbins=self.nbins, ) assert ( data.min() >= 0 and data.max() <= 1 ), f"Clahe should output data in range (0,1), got ({data.min()}, {data.max()})" output[out_key] = Array(data, array.spec).crop(request[out_key].roi) return output
def __get_stride(self): '''Get the maximal amount by which ``reference`` can be moved, such that it tiles the space.''' stride = None # get the least common multiple of all voxel sizes, we have to stride # at least that far lcm_voxel_size = self.spec.get_lcm_voxel_size( self.reference.array_specs.keys()) # that's just the minimal size in each dimension for key, reference_spec in self.reference.items(): shape = reference_spec.roi.get_shape() for d in range(len(lcm_voxel_size)): assert shape[d] >= lcm_voxel_size[d], ( "Shape of reference " "ROI %s for %s is " "smaller than least " "common multiple of " "voxel size " "%s" % (reference_spec.roi, key, lcm_voxel_size)) if stride is None: stride = shape else: stride = Coordinate((min(a, b) for a, b in zip(stride, shape))) return stride