def _get_next_shift(self, center_shift, voxel_size):
# gets next coordinate from list
if self.choose_randomly:
self.loc_i = randrange(len(self.coordinates))
else:
self.loc_i += 1
if self.loc_i >= len(self.coordinates):
self.loc_i = 0
logger.warning('Ran out of specified locations, looping list')
next_shift = Coordinate(self.coordinates[self.loc_i]) - center_shift
if self.jitter is not None:
rnd = []
for i in range(len(self.jitter)):
rnd.append(
np.random.randint(-self.jitter[i], self.jitter[i] + 1))
next_shift += Coordinate(rnd)
logger.debug("Shift before rounding: %s" % str(next_shift))
# make sure shift is a multiple of voxel size (round to nearest)
next_shift = Coordinate([
int(vs * round(float(shift) / vs))
for vs, shift in zip(voxel_size, next_shift)
])
logger.debug("Shift after rounding: %s" % str(next_shift))
return next_shift
def prepare(self, request):
if self.settings.mode == 'ball':
context = np.ceil(self.settings.radius).astype(np.int)
elif self.settings.mode == 'peak':
context = np.ceil(2*self.settings.radius).astype(np.int)
else:
raise RuntimeError('unknown raster mode %s'%self.settings.mode)
dims = self.array_spec.roi.dims()
if len(context) == 1:
context = context.repeat(dims)
# request graph in a larger area to get rasterization from outside
# graph
graph_roi = request[self.array].roi.grow(
Coordinate(context),
Coordinate(context))
# however, restrict the request to the graph actually provided
graph_roi = graph_roi.intersect(self.spec[self.graph].roi)
deps = BatchRequest()
deps[self.graph] = GraphSpec(roi=graph_roi)
if self.settings.mask is not None:
mask_voxel_size = self.spec[self.settings.mask].voxel_size
assert self.spec[self.array].voxel_size == mask_voxel_size, (
"Voxel size of mask and rasterized volume need to be equal")
new_mask_roi = graph_roi.snap_to_grid(mask_voxel_size)
deps[self.settings.mask] = ArraySpec(roi=new_mask_roi)
return deps
def _query_kdtree(self, node: cKDTreeNode,
bb: Tuple[np.ndarray, np.ndarray]) -> List[np.ndarray]:
def substitute_dim(bound: np.ndarray, sub_dim: int, sub: float):
# replace bound[sub_dim] with sub
return np.array(
[bound[i] if i != sub_dim else sub for i in range(3)])
if node is None:
return []
if node.split_dim != -1:
# if the split location is above the roi, no need to split (cannot be above None)
if bb[1][node.split_dim] is not None and node.split > bb[1][
node.split_dim]:
return self._query_kdtree(node.lesser, bb)
elif (bb[0][node.split_dim] is not None
and node.split < bb[0][node.split_dim]):
return self._query_kdtree(node.greater, bb)
else:
return self._query_kdtree(
node.greater, bb) + self._query_kdtree(node.lesser, bb)
else:
# handle leaf node
bbox = Roi(
Coordinate(bb[0]),
Coordinate(
tuple(y - x if x is not None and y is not None else y
for x, y in zip(*bb))),
)
points = [
ind for ind, point in zip(node.indices, node.data_points)
if bbox.contains(np.round(point))
]
return points
def setup(self):
assert self.labels in self.spec, (
"Upstream does not provide %s needed by "
"AddAffinities"%self.labels)
voxel_size = self.spec[self.labels].voxel_size
dims = self.affinity_neighborhood.shape[1]
self.padding_neg = Coordinate(
min([0] + [a[d] for a in self.affinity_neighborhood])
for d in range(dims)
)*voxel_size
self.padding_pos = Coordinate(
max([0] + [a[d] for a in self.affinity_neighborhood])
for d in range(dims)
)*voxel_size
logger.debug("padding neg: " + str(self.padding_neg))
logger.debug("padding pos: " + str(self.padding_pos))
spec = self.spec[self.labels].copy()
if spec.roi is not None:
spec.roi = spec.roi.grow(self.padding_neg, -self.padding_pos)
spec.dtype = np.float32
self.provides(self.affinities, spec)
if self.affinities_mask:
self.provides(self.affinities_mask, spec)
self.enable_autoskip()
def __transpose_roi(self, roi, total_roi, transpose, lcm_voxel_size):
logger.debug("original roi = %s", roi)
center = total_roi.get_center()
if lcm_voxel_size is not None:
nearest_voxel_shift = Coordinate(
(d % v)
for d, v in zip(center, lcm_voxel_size))
center = center - nearest_voxel_shift
logger.debug("center = %s", center)
# Get distance from center, then transpose
dist_to_center = center - roi.get_offset()
dist_to_center = Coordinate(dist_to_center[transpose[d]]
for d in range(self.dims))
logger.debug("dist_to_center = %s", dist_to_center)
# Using the tranposed distance to center, get the correct offset.
new_offset = center - dist_to_center
logger.debug("new_offset = %s", new_offset)
shape = tuple(roi.get_shape()[transpose[d]] for d in range(self.dims))
roi.set_offset(new_offset)
roi.set_shape(shape)
logger.debug("tranposed roi = %s", roi)
def process(self, batch, request):
# create vector map and add it to batch
for (array_key,
(src_points_key,
trg_points_key)) in self.array_to_src_trg_points.items():
if array_key in request:
vector_map = self.__get_vector_map(
batch=batch,
request=request,
vector_map_array_key=array_key)
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(data=vector_map, spec=spec)
# restore request / remove not requested points in padding-for-neighbors region & shrink batch roi
for (array_key,
(src_points_key,
trg_points_key)) in self.array_to_src_trg_points.items():
if array_key in request:
if trg_points_key in request:
for loc_id, point in batch.points[
trg_points_key].data.items():
if not request[trg_points_key].roi.contains(
Coordinate(point.location)):
del batch.points[trg_points_key].data[loc_id]
neg_pad_for_partners = Coordinate(
(self.pad_for_partners * np.asarray([-1])).tolist())
batch.points[trg_points_key].spec.roi = batch.points[
trg_points_key].spec.roi.grow(neg_pad_for_partners,
neg_pad_for_partners)
elif trg_points_key in batch.points:
del batch.points[trg_points_key]
def shift_points(points, request_roi, sub_shift_array, shift_axis,
lcm_voxel_size):
""" Shift a set of points received from upstream and crop out those not the the target downstream region
:param points: the points from upstream
:param request_roi: the downstream ROI
:param sub_shift_array: the cropped section of the global shift array that applies to this specific request
:param shift_axis: the axis to perform the shift along
:param lcm_voxel_size: the least common voxel size for the arrays in the request
:return a Points object with the updated point locations and ROI
"""
data = points.data
spec = points.spec
shift_axis_start_pos = spec.roi.get_offset()[shift_axis]
shifted_data = {}
for id_, point in data.items():
loc = Coordinate(point.location)
shift_axis_position = loc[shift_axis]
shift_array_index = (shift_axis_position - shift_axis_start_pos
) // lcm_voxel_size[shift_axis]
assert (shift_array_index >= 0)
shift = Coordinate(sub_shift_array[shift_array_index])
new_loc = loc + shift
if request_roi.contains(new_loc):
point.location = new_loc
shifted_data[id_] = point
points.data = shifted_data
points.spec.roi = request_roi
return points
def _resample_relative(
self, inside: np.ndarray, outside: np.ndarray, bb: Roi
) -> Optional[np.ndarray]:
offset = outside - inside
with np.errstate(divide="ignore", invalid="ignore"):
# bb_crossings will be 0 if inside is on the bb, 1 if outside is on the bb
bb_x = np.asarray(
[
(np.asarray(bb.get_begin()) - inside) / offset,
(np.asarray(bb.get_end() - Coordinate([1, 1, 1])) - inside)
/ offset,
]
)
if np.sum(np.logical_and((bb_x > 0), (bb_x <= 1))) > 0:
# all values of bb_x between 0, 1 represent a crossing of a bounding plane
# the minimum of which is the (normalized) distance to the closest bounding plane
s = np.min(bb_x[np.logical_and((bb_x > 0), (bb_x <= 1))])
return Coordinate(np.floor(np.array(inside) + s * offset))
else:
logging.debug(
(
"Could not create a node on the bounding box {} "
+ "given points (inside:{}, ouside:{})"
).format(bb, inside, outside)
)
return None
def __get_spec(self, array_key):
info = self.__get_info(array_key)
roi_min = info['Extended']['MinPoint']
if roi_min is not None:
roi_min = Coordinate(roi_min[::-1])
roi_max = info['Extended']['MaxPoint']
if roi_max is not None:
roi_max = Coordinate(roi_max[::-1])
data_roi = Roi(offset=roi_min, shape=(roi_max - roi_min))
data_dims = Coordinate(data_roi.get_shape())
if self.ndims is None:
self.ndims = len(data_dims)
else:
assert self.ndims == len(data_dims)
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
spec.voxel_size = Coordinate(info['Extended']['VoxelSize'])
if spec.roi is None:
spec.roi = data_roi * spec.voxel_size
data_dtype = dvision.DVIDDataInstance(self.hostname, self.port,
self.uuid,
self.datasets[array_key]).dtype
if spec.dtype is not None:
assert spec.dtype == data_dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from instance %s dtype %s" %
(self.array_specs[array_key].dtype, array_key,
self.datasets[array_key], data_dtype))
else:
spec.dtype = data_dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning(
"WARNING: You didn't set 'interpolatable' for %s. "
"Based on the dtype %s, it has been set to %s. "
"This might not be what you want.", array_key, spec.dtype,
spec.interpolatable)
return spec
def __read_spec(self, array_key, data_file, ds_name):
dataset = data_file[ds_name]
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
voxel_size = self._get_voxel_size(dataset)
if voxel_size is None:
voxel_size = Coordinate((1, ) * len(dataset.shape))
logger.warning(
"WARNING: File %s does not contain resolution information "
"for %s (dataset %s), voxel size has been set to %s. This "
"might not be what you want.", self.filename, array_key,
ds_name, spec.voxel_size)
spec.voxel_size = voxel_size
self.ndims = len(spec.voxel_size)
if spec.roi is None:
offset = self._get_offset(dataset)
if offset is None:
offset = Coordinate((0, ) * self.ndims)
if self.channels_first:
shape = Coordinate(dataset.shape[-self.ndims:])
else:
shape = Coordinate(dataset.shape[:self.ndims])
spec.roi = Roi(offset, shape * spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == dataset.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from dataset %s dtype %s" %
(self.array_specs[array_key].dtype, array_key, ds_name,
dataset.dtype))
else:
spec.dtype = dataset.dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning(
"WARNING: You didn't set 'interpolatable' for %s "
"(dataset %s). Based on the dtype %s, it has been "
"set to %s. This might not be what you want.", array_key,
ds_name, spec.dtype, spec.interpolatable)
return spec
def __read_spec(self, cv):
# dataset = data_file[ds_name]
dims = Coordinate(cv.bounds.maxpt[::-1]) * self._get_voxel_size(cv)
if self.ndims is None:
self.ndims = cv.bounds.ndim
else:
assert self.ndims == len(dims)
if self.array_spec is not None:
spec = self.array_spec.copy()
else:
spec = ArraySpec()
if spec.voxel_size is None:
voxel_size = self._get_voxel_size(cv)
if voxel_size is None:
voxel_size = Coordinate((1,) * self.ndims)
logger.warning(
"WARNING: File %s does not contain resolution information "
"for %s , voxel size has been set to %s. This "
"might not be what you want.",
self.filename, array_key, spec.voxel_size)
spec.voxel_size = voxel_size
if spec.roi is None:
offset = self._get_offset(cv)
if offset is None:
offset = Coordinate((0,) * self.ndims)
spec.roi = Roi(offset, dims * spec.voxel_size)
if spec.dtype is not None:
assert spec.dtype == cv.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from cloudvolume dtype %s" %
(self.array_spec.dtype,
self.array_key, dataset.dtype))
else:
spec.dtype = cv.dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning("WARNING: You didn't set 'interpolatable' for %s "
". Based on the dtype %s, it has been "
"set to %s. This might not be what you want.",
self.array_key, spec.dtype,
spec.interpolatable)
return spec
def _get_offset(self, dataset):
if 'offset' not in dataset.attrs:
return None
if self.output_filename.endswith('.n5'):
return Coordinate(dataset.attrs['offset'][::-1])
else:
return Coordinate(dataset.attrs['offset'])
def _get_voxel_size(self, dataset):
if 'resolution' not in dataset.attrs:
return None
if self.output_filename.endswith('.n5'):
return Coordinate(dataset.attrs['resolution'][::-1])
else:
return Coordinate(dataset.attrs['resolution'])
def guarantee_nonempty(pipeline, setup_config, key):
voxel_size = Coordinate(setup_config["VOXEL_SIZE"])
output_size = Coordinate(setup_config["OUTPUT_SHAPE"]) * voxel_size
num_components = setup_config["NUM_COMPONENTS"]
pipeline = pipeline + RejectIfEmpty(
key, centroid_size=output_size, num_components=num_components)
return pipeline
def __get_roi(self, array_name):
data_instance = dvision.DVIDDataInstance(self.hostname, self.port, self.uuid, array_name)
info = data_instance.info
roi_min = info['Extended']['MinPoint']
if roi_min is not None:
roi_min = Coordinate(roi_min[::-1])
roi_max = info['Extended']['MaxPoint']
if roi_max is not None:
roi_max = Coordinate(roi_max[::-1])
return Roi(offset=roi_min, shape=roi_max - roi_min)
def __init__(
self,
dbname: str,
url: str,
points: List[GraphKey],
graph_specs: Optional[Union[GraphSpec, List[GraphSpec]]] = None,
directed: bool = False,
total_roi: Roi = None,
nodes_collection: str = "nodes",
edges_collection: str = "edges",
meta_collection: str = "meta",
endpoint_names: Tuple[str, str] = ("u", "v"),
position_attribute: str = "position",
node_attrs: Optional[List[str]] = None,
edge_attrs: Optional[List[str]] = None,
nodes_filter: Optional[Dict[str, Any]] = None,
edges_filter: Optional[Dict[str, Any]] = None,
edge_inclusion: str = "either",
node_inclusion: str = "dangling",
fail_on_inconsistent_node: bool = False,
):
self.points = points
graph_specs = (graph_specs if graph_specs is not None else GraphSpec(
Roi(Coordinate([None] * 3), Coordinate([None] * 3)),
directed=False))
specs = (graph_specs if isinstance(graph_specs, list)
and len(graph_specs) == len(points) else [graph_specs] *
len(points))
self.specs = {key: spec for key, spec in zip(points, specs)}
self.directed = directed
self.nodes_collection = nodes_collection
self.edges_collection = edges_collection
self.meta_collection = meta_collection
self.endpoint_names = endpoint_names
self.position_attribute = position_attribute
self.position_attribute = position_attribute
self.node_attrs = node_attrs
self.edge_attrs = edge_attrs
self.nodes_filter = nodes_filter
self.edges_filter = edges_filter
self.edge_inclusion = edge_inclusion
self.node_inclusion = node_inclusion
self.dbname = dbname
self.url = url
self.nodes_collection = nodes_collection
self.fail_on_inconsistent_node = fail_on_inconsistent_node
self.graph_provider = None
def __init__(
self,
dbname: str,
url: str,
points: List[GraphKey],
graph_specs: Optional[Union[GraphSpec, List[GraphSpec]]] = None,
directed: bool = False,
total_roi: Roi = None,
nodes_collection: str = "nodes",
edges_collection: str = "edges",
meta_collection: str = "meta",
endpoint_names: Tuple[str, str] = ("u", "v"),
position_attribute: str = "position",
node_attrs: Optional[List[str]] = None,
edge_attrs: Optional[List[str]] = None,
nodes_filter: Optional[Dict[str, Any]] = None,
edges_filter: Optional[Dict[str, Any]] = None,
num_nodes=100000,
dist_attribute=None,
min_dist=29000,
):
self.points = points
graph_specs = (graph_specs if graph_specs is not None else GraphSpec(
Roi(Coordinate([None] * 3), Coordinate([None] * 3)),
directed=False))
specs = (graph_specs if isinstance(graph_specs, list)
and len(graph_specs) == len(points) else [graph_specs] *
len(points))
self.specs = {key: spec for key, spec in zip(points, specs)}
self.position_attribute = position_attribute
self.node_attrs = node_attrs
self.edge_attrs = edge_attrs
self.nodes_filter = nodes_filter
self.edges_filter = edges_filter
self.dist_attribute = dist_attribute
self.min_dist = min_dist
self.num_nodes = num_nodes
self.graph_provider = MongoDbGraphProvider(
dbname,
url,
mode="r+",
directed=directed,
total_roi=None,
nodes_collection=nodes_collection,
edges_collection=edges_collection,
meta_collection=meta_collection,
endpoint_names=endpoint_names,
position_attribute=position_attribute,
)
def provide(self, batch_spec):
logger.info("batch with spec " + str(batch_spec) + " requested")
stride = self.chunk_spec_template.output_roi.get_shape()
begin = batch_spec.input_roi.get_begin()
end = batch_spec.input_roi.get_end()
batch = None
offset = np.array(begin)
while (offset < end).all():
# create a copy of the requested batch spec
chunk_spec = copy.deepcopy(batch_spec)
# change size and offset of the batch spec
chunk_spec.input_roi = self.chunk_spec_template.input_roi + Coordinate(
offset)
chunk_spec.output_roi = self.chunk_spec_template.output_roi + Coordinate(
offset)
logger.info("requesting chunk " + str(chunk_spec))
# get a chunk
chunk = self.get_upstream_provider().request_batch(chunk_spec)
if batch is None:
batch = self.__setup_batch(batch_spec, chunk)
for (volume_type, volume) in chunk.volumes:
# input roi for RAW, output roi for others
if volume_type == VolumeType.RAW:
self.__fill(batch[volume_type].data, volume.data,
batch_spec.input_roi, chunk.spec.input_roi)
else:
self.__fill(batch[volume_type].data, volume.data,
batch_spec.output_roi, chunk.spec.output_roi)
for d in range(self.dims):
offset[d] += stride[d]
if offset[d] >= end[d]:
if d == self.dims - 1:
break
offset[d] = begin[d]
else:
break
return batch
def __get_source_roi(self, transformation):
dims = transformation.shape[0]
# get bounding box of needed data for transformation
bb_min = Coordinate(
int(math.floor(transformation[d].min())) for d in range(dims))
bb_max = Coordinate(
int(math.ceil(transformation[d].max())) + 1 for d in range(dims))
# create roi sufficiently large to feed transformation
source_roi = Roi(bb_min, bb_max - bb_min)
return source_roi
def setup(self):
self._read_points()
if self.points_spec is not None:
self.provides(self.points, self.points_spec)
return
min_bb = Coordinate(np.floor(np.amin(self.data[:, :self.ndims], 0)))
max_bb = Coordinate(np.ceil(np.amax(self.data[:, :self.ndims], 0)) + 1)
roi = Roi(min_bb, max_bb - min_bb)
self.provides(self.points, GraphSpec(roi=roi))
def compute_upstream_roi(request_roi, sub_shift_array):
""" Compute the ROI to pass upstream for a specific item (array or points) in a request
:param request_roi: the downstream ROI passed to the Jitter node
:param sub_shift_array: the portion of the global shift array that should be used to shift the item
:return: the expanded ROI to pass upstream
"""
max_shift = Coordinate(sub_shift_array.max(axis=0))
min_shift = Coordinate(sub_shift_array.min(axis=0))
downstream_offset = request_roi.get_offset()
upstream_offset = downstream_offset - max_shift
upstream_shape = request_roi.get_shape() + max_shift - min_shift
return Roi(offset=upstream_offset, shape=upstream_shape)
def get_lcm_voxel_size(self, array_keys=None):
'''Get the least common multiple of the voxel sizes in this spec.
Args:
array_keys (list of :class:`ArrayKey`, optional): If given,
consider only the given array types.
'''
if array_keys is None:
array_keys = self.array_specs.keys()
if not array_keys:
raise RuntimeError("Can not compute lcm voxel size -- there are "
"no array specs in this provider spec.")
else:
if not array_keys:
raise RuntimeError("Can not compute lcm voxel size -- list of "
"given array specs is empty.")
lcm_voxel_size = None
for key in array_keys:
voxel_size = self.array_specs[key].voxel_size
if lcm_voxel_size is None:
lcm_voxel_size = voxel_size
else:
lcm_voxel_size = Coordinate(
(a * b // fractions.gcd(a, b)
for a, b in zip(lcm_voxel_size, voxel_size)))
return lcm_voxel_size
def setup(self):
self.data = self.read_points(self.filename)
self.ndims = self.data.shape[1]
if self.points_spec is not None:
self.provides(self.points, self.points_spec)
return
min_bb = Coordinate(np.floor(np.amin(self.data, 0)))
max_bb = Coordinate(np.ceil(np.amax(self.data, 0)))
roi = Roi(min_bb, max_bb - min_bb)
self.provides(self.points, PointsSpec(roi=roi))
def get_lcm_voxel_size(self, array_keys=None):
'''Get the least common multiple of the voxel sizes in this spec.
Args:
array_keys (list of :class:`ArrayKey`, optional): If given,
consider only the given array types.
'''
if array_keys is None:
array_keys = self.array_specs.keys()
if not array_keys:
return None
lcm_voxel_size = None
for key in array_keys:
voxel_size = self.array_specs[key].voxel_size
if voxel_size is None:
continue
if lcm_voxel_size is None:
lcm_voxel_size = voxel_size
else:
lcm_voxel_size = Coordinate(
(a * b // math.gcd(a, b)
for a, b in zip(lcm_voxel_size, voxel_size)))
return lcm_voxel_size
def shift_points(points, request_roi, sub_shift_array, shift_axis, lcm_voxel_size):
""" Shift a set of points received from upstream and crop out those not the the target downstream region
:param points: the points from upstream
:param request_roi: the downstream ROI
:param sub_shift_array: the cropped section of the global shift array that applies to this specific request
:param shift_axis: the axis to perform the shift along
:param lcm_voxel_size: the least common voxel size for the arrays in the request
:return a Graph object with the updated point locations and ROI
"""
nodes = list(points.nodes)
spec = points.spec
shift_axis_start_pos = spec.roi.get_offset()[shift_axis]
for node in nodes:
loc = node.location
shift_axis_position = loc[shift_axis]
shift_array_index = int((shift_axis_position - shift_axis_start_pos) // lcm_voxel_size[shift_axis])
assert(shift_array_index >= 0)
shift = Coordinate(sub_shift_array[shift_array_index])
loc += shift
if not request_roi.contains(loc):
points.remove_node(node)
points.spec.roi = request_roi
return points
def prepare(self, request):
logger.debug("request: %s", request.array_specs)
logger.debug("my spec: %s", self.spec)
shift_roi = self.__get_possible_shifts(request)
if request.array_specs.keys():
lcm_voxel_size = self.spec.get_lcm_voxel_size(
request.array_specs.keys())
lcm_shift_roi = shift_roi/lcm_voxel_size
logger.debug("lcm voxel size: %s", lcm_voxel_size)
logger.debug(
"restricting random locations to multiples of voxel size %s",
lcm_voxel_size)
else:
lcm_voxel_size = Coordinate((1,)*shift_roi.dims())
lcm_shift_roi = shift_roi
random_shift = self.__select_random_shift(
request,
lcm_shift_roi,
lcm_voxel_size)
self.random_shift = random_shift
self.__shift_request(request, random_shift)
def __mirror_roi(self, roi, total_roi, mirror):
total_roi_offset = total_roi.get_offset()
total_roi_shape = total_roi.get_shape()
roi_offset = roi.get_offset()
roi_shape = roi.get_shape()
roi_in_total_offset = roi_offset - total_roi_offset
end_of_roi_in_total = roi_in_total_offset + roi_shape
roi_in_total_offset_mirrored = total_roi_shape - end_of_roi_in_total
roi_offset = Coordinate(
total_roi_offset[d] +
roi_in_total_offset_mirrored[d] if mirror[d] else roi_offset[d]
for d in range(self.dims))
logger.debug("Mirror numbers for roi: " + str(roi) + "\nMirror: " +
str(mirror) + "\tTranpose: " + str(self.transpose) +
"\ntotal roi: " + str(total_roi) +
"\nroi_in_total_offset: " + str(roi_in_total_offset) +
"\nend_of_roi_in_total: " + str(end_of_roi_in_total) +
"\nroi_in_total_offset_mirrored: " +
str(roi_in_total_offset_mirrored) + "\nroi_offset: " +
str(roi_offset))
roi.set_offset(roi_offset)
def check_batch_consistency(self, batch, request):
for (array_key, request_spec) in request.array_specs.items():
assert array_key in batch.arrays, "%s requested, but %s did not provide it." % (
array_key, self.name())
array = batch.arrays[array_key]
assert array.spec.roi == request_spec.roi, "%s ROI %s requested, but ROI %s provided by %s." % (
array_key, request_spec.roi, array.spec.roi, self.name())
assert array.spec.voxel_size == self.spec[array_key].voxel_size, (
"voxel size of %s announced, but %s "
"delivered for %s" % (self.spec[array_key].voxel_size,
array.spec.voxel_size, array_key))
# ensure that the spatial dimensions are the same (other dimensions
# on top are okay, e.g., for affinities)
dims = request_spec.roi.dims()
data_shape = Coordinate(array.data.shape[-dims:])
voxel_size = self.spec[array_key].voxel_size
assert data_shape == request_spec.roi.get_shape(
) / voxel_size, "%s ROI %s requested, but size of array is %s*%s=%s provided by %s." % (
array_key, request_spec.roi, data_shape, voxel_size,
data_shape * voxel_size, self.name())
for (points_key, request_spec) in request.points_specs.items():
assert points_key in batch.points, "%s requested, but %s did not provide it." % (
points_key, self.name())
points = batch.points[points_key]
assert points.spec.roi == request_spec.roi, "%s ROI %s requested, but ROI %s provided by %s." % (
points_key, request_spec.roi, points.spec.roi, self.name())
for _, point in points.data.items():
assert points.spec.roi.contains(point.location), (
"points provided by %s with ROI %s contain point at %s" %
(self.name(), points.spec.roi, point.location))
def process(self, batch, request):
output = Batch()
for in_key, out_key in zip(self.arrays, self.output_arrays):
array = batch[in_key]
data = array.data
d_min = data.min()
d_max = data.max()
assert (
d_min >= 0 and d_max <= 1
), f"Clahe expects data in range (0,1), got ({d_min}, {d_max})"
if np.isclose(d_max, d_min):
output[out_key] = Array(data, array.spec)
continue
if self.normalize:
data = (data - d_min) / (d_max - d_min)
shape = data.shape
data_dims = len(shape)
kernel_dims = len(self.kernel_size)
extra_dims = data_dims - kernel_dims
voxel_size = array.spec.voxel_size
for index in itertools.product(*[range(s) for s in shape[:extra_dims]]):
data[index] = clahe(
data[index],
kernel_size=Coordinate(self.kernel_size / voxel_size),
clip_limit=self.clip_limit,
nbins=self.nbins,
)
assert (
data.min() >= 0 and data.max() <= 1
), f"Clahe should output data in range (0,1), got ({data.min()}, {data.max()})"
output[out_key] = Array(data, array.spec).crop(request[out_key].roi)
return output
def __get_stride(self):
'''Get the maximal amount by which ``reference`` can be moved, such
that it tiles the space.'''
stride = None
# get the least common multiple of all voxel sizes, we have to stride
# at least that far
lcm_voxel_size = self.spec.get_lcm_voxel_size(
self.reference.array_specs.keys())
# that's just the minimal size in each dimension
for key, reference_spec in self.reference.items():
shape = reference_spec.roi.get_shape()
for d in range(len(lcm_voxel_size)):
assert shape[d] >= lcm_voxel_size[d], (
"Shape of reference "
"ROI %s for %s is "
"smaller than least "
"common multiple of "
"voxel size "
"%s" % (reference_spec.roi, key, lcm_voxel_size))
if stride is None:
stride = shape
else:
stride = Coordinate((min(a, b) for a, b in zip(stride, shape)))
return stride
