def process(self, batch, request):
output = Batch()
for in_key, out_key in zip(self.arrays, self.output_arrays):
array = batch[in_key]
data = array.data
d_min = data.min()
d_max = data.max()
assert (
d_min >= 0 and d_max <= 1
), f"Clahe expects data in range (0,1), got ({d_min}, {d_max})"
if np.isclose(d_max, d_min):
output[out_key] = Array(data, array.spec)
continue
if self.normalize:
data = (data - d_min) / (d_max - d_min)
shape = data.shape
data_dims = len(shape)
kernel_dims = len(self.kernel_size)
extra_dims = data_dims - kernel_dims
voxel_size = array.spec.voxel_size
for index in itertools.product(*[range(s) for s in shape[:extra_dims]]):
data[index] = clahe(
data[index],
kernel_size=Coordinate(self.kernel_size / voxel_size),
clip_limit=self.clip_limit,
nbins=self.nbins,
)
assert (
data.min() >= 0 and data.max() <= 1
), f"Clahe should output data in range (0,1), got ({data.min()}, {data.max()})"
output[out_key] = Array(data, array.spec).crop(request[out_key].roi)
return output
def process(self, batch, request):
if self.target not in request:
return
input_roi = batch.arrays[self.source].spec.roi
request_roi = request[self.target].roi
assert input_roi.contains(request_roi)
# upsample
order = 3 if batch.arrays[self.source].spec.interpolatable else 0
data = ndimage.zoom(batch.arrays[self.source].data,
np.array(self.factor),
order=order)
# Create output array, crop accordingly.
spec = self.spec[self.target].copy()
spec.roi = input_roi
ar = Array(data, spec)
batch.arrays[self.target] = ar.crop(request_roi)
if self.source in request:
# restore requested rois
request_roi = request[self.source].roi
if input_roi != request_roi:
assert input_roi.contains(request_roi)
logger.debug("restoring original request roi %s of %s from %s",
request_roi, self.source, input_roi)
cropped = batch.arrays[self.source].crop(request_roi)
batch.arrays[self.source] = cropped
def train_step(self, batch, request):
data = {}
for input_name, network_input in self.inputs.items():
if isinstance(network_input, ArrayKey):
if network_input in batch.arrays:
data[input_name] = batch.arrays[network_input].data
else:
logger.warn("batch does not contain %s, input %s will not "
"be set", network_input, input_name)
elif isinstance(network_input, np.ndarray):
data[input_name] = network_input
elif isinstance(network_input, str):
data[input_name] = getattr(batch, network_input)
else:
raise Exception(
"Unknown network input type {}, can't be given to "
"network".format(network_input))
self.net_io.set_inputs(data)
loss = self.solver.step(1)
# self.__consistency_check()
requested_outputs = {
name: array_key
for name, array_key in self.outputs.items()
if array_key in request.array_specs }
if requested_outputs:
output = self.net_io.get_outputs()
for output_name, array_key in requested_outputs.items():
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(
output[output_name][0], # strip #batch dimension
spec)
requested_gradients = {
name: array_key
for name, array_key in self.gradients.items()
if array_key in request.array_specs }
if requested_gradients:
diffs = self.net_io.get_output_diffs()
for output_name, array_key in requested_gradients.items():
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(
diffs[output_name][0], # strip #batch dimension
spec)
batch.loss = loss
batch.iteration = self.solver.iter
def provide(self, request):
timing = Timing(self)
timing.start()
batch = Batch()
_, request_spec = request.array_specs.items()[0]
logger.debug("Reading %s in %s...", self.array, request_spec.roi)
voxel_size = self.spec[self.array].voxel_size
# scale request roi to voxel units
dataset_roi = request_spec.roi/voxel_size
# shift request roi into dataset
dataset_roi = dataset_roi - self.spec[self.array].roi.get_offset()/voxel_size
# create array spec
array_spec = self.spec[self.array].copy()
array_spec.roi = request_spec.roi
# add array to batch
batch.arrays[self.array] = Array(
self.__read(dataset_roi),
array_spec)
logger.debug("done")
timing.stop()
batch.profiling_stats.add(timing)
return batch
def process(self, batch, request):
output = Batch()
for in_key, out_key in zip(self.arrays, self.output_arrays):
array = batch[in_key]
data = array.data
shape = data.shape
data_dims = len(shape)
kernel_dims = len(self.kernel_size)
extra_dims = data_dims - kernel_dims
if self.slice_wise:
for index in itertools.product(
*[range(s) for s in shape[:extra_dims]]):
data[index] = mclahe(
data[index],
kernel_size=self.kernel_size,
clip_limit=self.clip_limit,
n_bins=self.nbins,
use_gpu=False,
adaptive_hist_range=self.adaptive_hist_range,
)
else:
full_kernel = np.array(
(1, ) * extra_dims + tuple(self.kernel_size), dtype=int)
data = mclahe(
data,
kernel_size=full_kernel,
clip_limit=self.clip_limit,
n_bins=self.nbins,
# use_gpu=False,
).astype(self.spec[out_key].dtype)
output[out_key] = Array(data,
array.spec).crop(request[out_key].roi)
return output
def __setup_batch(self, batch_spec, chunk):
'''Allocate a batch matching the sizes of ``batch_spec``, using
``chunk`` as template.'''
batch = Batch()
for (array_key, spec) in batch_spec.array_specs.items():
roi = spec.roi
voxel_size = self.spec[array_key].voxel_size
# get the 'non-spatial' shape of the chunk-batch
# and append the shape of the request to it
array = chunk.arrays[array_key]
shape = array.data.shape[:-roi.dims()]
shape += (roi.get_shape() // voxel_size)
spec = self.spec[array_key].copy()
spec.roi = roi
logger.info("allocating array of shape %s for %s", shape,
array_key)
batch.arrays[array_key] = Array(data=np.zeros(shape), spec=spec)
for (points_key, spec) in batch_spec.points_specs.items():
roi = spec.roi
spec = self.spec[points_key].copy()
spec.roi = roi
batch.points[points_key] = Points(data={}, spec=spec)
logger.debug("setup batch to fill %s", batch)
return batch
def process(self, batch, request):
# create vector map and add it to batch
for (array_key,
(src_points_key,
trg_points_key)) in self.array_to_src_trg_points.items():
if array_key in request:
vector_map = self.__get_vector_map(
batch=batch,
request=request,
vector_map_array_key=array_key)
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(data=vector_map, spec=spec)
# restore request / remove not requested points in padding-for-neighbors region & shrink batch roi
for (array_key,
(src_points_key,
trg_points_key)) in self.array_to_src_trg_points.items():
if array_key in request:
if trg_points_key in request:
for loc_id, point in batch.points[
trg_points_key].data.items():
if not request[trg_points_key].roi.contains(
Coordinate(point.location)):
del batch.points[trg_points_key].data[loc_id]
neg_pad_for_partners = Coordinate(
(self.pad_for_partners * np.asarray([-1])).tolist())
batch.points[trg_points_key].spec.roi = batch.points[
trg_points_key].spec.roi.grow(neg_pad_for_partners,
neg_pad_for_partners)
elif trg_points_key in batch.points:
del batch.points[trg_points_key]
def process(self, batch, request):
outputs = Batch()
if self.target not in request:
return
input_roi = batch.arrays[self.source].spec.roi
request_roi = request[self.target].roi
assert input_roi.contains(request_roi)
# upsample
logger.debug("upsampling %s with %s", self.source, self.factor)
crop = batch.arrays[self.source].crop(request_roi)
data = crop.data
for d, f in enumerate(self.factor):
data = np.repeat(data, f, axis=d)
# create output array
spec = self.spec[self.target].copy()
spec.roi = request_roi
outputs.arrays[self.target] = Array(data, spec)
return outputs
def provide(self, request):
timing = Timing(self)
timing.start()
batch = Batch()
with self._open_file(self.filename) as data_file:
for (array_key, request_spec) in request.array_specs.items():
voxel_size = self.spec[array_key].voxel_size
# scale request roi to voxel units
dataset_roi = request_spec.roi / voxel_size
# shift request roi into dataset
dataset_roi = dataset_roi - self.spec[
array_key].roi.get_offset() / voxel_size
# create array spec
array_spec = self.spec[array_key].copy()
array_spec.roi = request_spec.roi
# add array to batch
batch.arrays[array_key] = Array(
self.__read(data_file, self.datasets[array_key],
dataset_roi, self.channel_ids[array_key]),
array_spec)
timing.stop()
batch.profiling_stats.add(timing)
return batch
def process(self, batch, request):
labels = batch.arrays[self.labels]
scales = np.ones(labels.data.shape, dtype=np.float32)
# check if fg in batch
if np.max(scales) > 0:
scales = scales * self.factors[0]
scales[labels.data > 0] = self.factors[1]
spec = self.spec[self.scales].copy()
spec.roi = labels.spec.roi
batch.arrays[self.scales] = Array(scales, spec)
def process(self, batch, request):
gt = batch.arrays[self.labels]
# 0 marks included regions (to be used directly with distance transform
# later)
include_mask = np.ones(gt.data.shape)
gt_labels = np.unique(gt.data)
logger.debug("batch contains GT labels: " + str(gt_labels))
for label in gt_labels:
if label in self.exclude:
logger.debug("excluding label " + str(label))
gt.data[gt.data == label] = self.background_value
else:
include_mask[gt.data == label] = 0
# if no ignore mask is provided or requested, we are done
if not self.ignore_mask or not self.ignore_mask in request:
return
voxel_size = self.spec[self.labels].voxel_size
distance_to_include = distance_transform_edt(include_mask,
sampling=voxel_size)
logger.debug("max distance to foreground is " +
str(distance_to_include.max()))
# 1 marks included regions, plus a context area around them
include_mask = distance_to_include < self.ignore_mask_erode
# include mask was computed on labels ROI, we need to copy it to
# the requested ignore_mask ROI
gt_ignore_roi = request[self.ignore_mask].roi
intersection = gt.spec.roi.intersect(gt_ignore_roi)
intersection_in_gt = intersection - gt.spec.roi.get_offset()
intersection_in_gt_ignore = intersection - gt_ignore_roi.get_offset()
# to voxel coordinates
intersection_in_gt //= voxel_size
intersection_in_gt_ignore //= voxel_size
gt_ignore = np.zeros((gt_ignore_roi // voxel_size).get_shape(),
dtype=np.uint8)
gt_ignore[intersection_in_gt_ignore.get_bounding_box()] = include_mask[
intersection_in_gt.get_bounding_box()]
spec = self.spec[self.labels].copy()
spec.roi = gt_ignore_roi
spec.dtype = np.uint8
batch.arrays[self.ignore_mask] = Array(gt_ignore, spec)
def process(self, batch, request):
# create vector map and add it to batch
for (
array_key,
(src_points_key, trg_points_key),
) in self.array_to_src_trg_points.items():
if array_key in request:
vector_map = self.__get_vector_map(
batch=batch, request=request, vector_map_array_key=array_key
)
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(data=vector_map, spec=spec)
def process(self, batch, request):
labels = batch.arrays[self.labels]
assert len(np.unique(labels.data)) <= 2, (
"Found more than two labels in %s."%self.labels)
assert np.min(labels.data) in [0.0, 1.0], (
"Labels %s are not binary."%self.labels)
assert np.max(labels.data) in [0.0, 1.0], (
"Labels %s are not binary."%self.labels)
# initialize error scale with 1s
error_scale = np.ones(labels.data.shape, dtype=np.float32)
# set error_scale to 0 in masked-out areas
for key in self.masks:
mask = batch.arrays[key]
assert labels.data.shape == mask.data.shape, (
"Shape of mask %s %s does not match %s %s"%(
mask,
mask.data.shape,
self.labels,
labels.data.shape))
error_scale *= mask.data
if not self.slab:
slab = error_scale.shape
else:
# slab with -1 replaced by shape
slab = tuple(
m if s == -1 else s
for m, s in zip(error_scale.shape, self.slab))
slab_ranges = (
range(0, m, s)
for m, s in zip(error_scale.shape, slab))
for start in itertools.product(*slab_ranges):
slices = tuple(
slice(start[d], start[d] + slab[d])
for d in range(len(slab)))
self.__balance(
labels.data[slices],
error_scale[slices])
spec = self.spec[self.scales].copy()
spec.roi = labels.spec.roi
batch.arrays[self.scales] = Array(error_scale, spec)
def train_step(self, batch, request):
array_outputs = self.__collect_requested_outputs(request)
inputs = self.__collect_provided_inputs(batch)
to_compute = {
'optimizer': self.optimizer,
'loss': self.loss,
'iteration': self.iteration_increment
}
to_compute.update(array_outputs)
# compute outputs, gradients, and update variables
if isinstance(self.summary, str):
to_compute["summaries"] = self.summary
elif isinstance(self.summary, dict):
for k, (v, f) in self.summary.items():
if int(self.current_step + 1) % f == 0:
to_compute[k] = v
outputs = self.session.run(to_compute, feed_dict=inputs)
for array_key in array_outputs:
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(outputs[array_key], spec)
batch.loss = outputs['loss']
batch.iteration = outputs['iteration'][0]
self.current_step = batch.iteration
if self.summary is not None:
if isinstance(self.summary, str) and \
(batch.iteration % self.log_every == 0 or batch.iteration == 1):
self.summary_saver.add_summary(outputs['summaries'],
batch.iteration)
else:
for k, (_, f) in self.summary.items():
if int(self.current_step) % f == 0:
self.summary_saver.add_summary(outputs[k],
batch.iteration)
if batch.iteration % self.save_every == 0:
checkpoint_name = (self.meta_graph_filename +
'_checkpoint_%i' % batch.iteration)
logger.info("Creating checkpoint %s", checkpoint_name)
self.full_saver.save(self.session, checkpoint_name)
def process(self, batch, request):
labels = batch.arrays[self.labels]
scales = np.ones(labels.data.shape, dtype=np.float32)
classes = np.unique(labels.data)
# heads up: skeleton only for all foreground classes
if np.max(classes) > 0:
foreground = labels.data > 0
skeleton = skeletonize_3d(foreground) > 0
scales = scales * self.factors[0]
scales[skeleton] = self.factors[1]
spec = self.spec[self.scales].copy()
spec.roi = labels.spec.roi
batch.arrays[self.scales] = Array(scales, spec)
def predict(self, batch, request):
logger.debug("predicting in batch %i", batch.id)
array_outputs = self.__collect_requested_outputs(request)
inputs = self.__collect_provided_inputs(batch)
# compute outputs
outputs = self.session.run(array_outputs, feed_dict=inputs)
for array_key in array_outputs:
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(outputs[array_key], spec)
logger.debug("predicted in batch %i", batch.id)
def process(self, batch, request):
for old_key, new_key in self.to_duplicate.items():
if old_key not in request:
continue
assert new_key not in batch.arrays, "key {} already present in batch".format(
new_key)
assert isinstance(new_key,
ArrayKey), "Can only duplicate array data"
array_ = batch.arrays[old_key]
array = Array(array_.data.copy(),
spec=array_.spec,
attrs=copy.deepcopy(array_.attrs))
batch.arrays[new_key] = array
def predict(self, batch, request):
self.net_io.set_inputs({
input_name: batch.arrays[array_key].data
for input_name, array_key in self.inputs.items()
})
self.net.forward()
output = self.net_io.get_outputs()
for output_name, array_key in self.outputs.items():
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(
output[output_name][0], # strip #batch dimension
spec)
return batch
def process(self, batch, request):
src_points = batch.points[self.src_points]
voxel_size = self.spec[self.array].voxel_size
# get roi used for creating the new array (points_roi does not
# necessarily align with voxel size)
enlarged_vol_roi = src_points.spec.roi.snap_to_grid(voxel_size)
offset = enlarged_vol_roi.get_begin() / voxel_size
shape = enlarged_vol_roi.get_shape() / voxel_size
data_roi = Roi(offset, shape)
logger.debug("Src points in %s", src_points.spec.roi)
for i, point in src_points.data.items():
logger.debug("%d, %s", i, point.location)
logger.debug("Data roi in voxels: %s", data_roi)
logger.debug("Data roi in world units: %s", data_roi * voxel_size)
mask_array = None if self.mask is None else batch.arrays[
self.mask].crop(enlarged_vol_roi)
partner_vectors_data, pointmask = self.__draw_partner_vectors(
src_points, batch.points[self.trg_points], data_roi, voxel_size,
enlarged_vol_roi.get_begin(), self.radius, mask_array)
# create array and crop it to requested roi
spec = self.spec[self.array].copy()
spec.roi = data_roi * voxel_size
partner_vectors = Array(data=partner_vectors_data, spec=spec)
logger.debug("Cropping partner vectors to %s", request[self.array].roi)
batch.arrays[self.array] = partner_vectors.crop(
request[self.array].roi)
if self.pointmask is not None and self.pointmask in request:
spec = self.spec[self.array].copy()
spec.roi = data_roi * voxel_size
pointmask = Array(data=np.array(pointmask, dtype=spec.dtype),
spec=spec)
batch.arrays[self.pointmask] = pointmask.crop(
request[self.pointmask].roi)
# restore requested ROI of src and target points.
if self.src_points in request:
self.__restore_points_roi(request, self.src_points,
batch.points[self.src_points])
if self.trg_points in request:
self.__restore_points_roi(request, self.trg_points,
batch.points[self.trg_points])
# restore requested objectmask
if self.mask is not None:
batch.arrays[self.mask] = batch.arrays[self.mask].crop(
request[self.mask].roi)
def process(self, batch, request):
if self.target not in request:
return
input_roi = batch.arrays[self.source].spec.roi
request_roi = request[self.target].roi
assert input_roi.contains(request_roi)
# downsample
if isinstance(self.factor, tuple):
slices = tuple(
slice(None, None, k)
for k in self.factor)
else:
slices = tuple(
slice(None, None, self.factor)
for i in range(input_roi.dims()))
logger.debug("downsampling %s with %s", self.source, slices)
crop = batch.arrays[self.source].crop(request_roi)
data = crop.data[slices]
# create output array
spec = self.spec[self.target].copy()
spec.roi = request_roi
batch.arrays[self.target] = Array(data, spec)
if self.source in request:
# restore requested rois
request_roi = request[self.source].roi
if input_roi != request_roi:
assert input_roi.contains(request_roi)
logger.debug(
"restoring original request roi %s of %s from %s",
request_roi, self.source, input_roi)
cropped = batch.arrays[self.source].crop(request_roi)
batch.arrays[self.source] = cropped
def process(self, batch, request):
points = batch.points[self.points]
voxel_size = self.spec[self.array].voxel_size
# get roi used for creating the new array (points_roi does not
# necessarily align with voxel size)
enlarged_vol_roi = points.spec.roi.snap_to_grid(voxel_size)
offset = enlarged_vol_roi.get_begin() / voxel_size
shape = enlarged_vol_roi.get_shape() / voxel_size
data_roi = Roi(offset, shape)
# points ROI is at least +- 1 in t of requested array ROI, we can save
# some time by shaving the excess off
data_roi = data_roi.grow((-1, 0, 0, 0), (-1, 0, 0, 0))
logger.debug("Points in %s", points.spec.roi)
for i, point in points.data.items():
logger.debug("%d, %s", i, point.location)
logger.debug("Data roi in voxels: %s", data_roi)
logger.debug("Data roi in world units: %s", data_roi * voxel_size)
parent_vectors_data, mask_data = self.__draw_parent_vectors(
points, data_roi, voxel_size, enlarged_vol_roi.get_begin(),
self.radius)
# create array and crop it to requested roi
spec = self.spec[self.array].copy()
spec.roi = data_roi * voxel_size
parent_vectors = Array(data=parent_vectors_data, spec=spec)
logger.debug("Cropping parent vectors to %s", request[self.array].roi)
batch.arrays[self.array] = parent_vectors.crop(request[self.array].roi)
# create mask and crop it to requested roi
spec = self.spec[self.mask].copy()
spec.roi = data_roi * voxel_size
mask = Array(data=mask_data, spec=spec)
logger.debug("Cropping mask to %s", request[self.mask].roi)
batch.arrays[self.mask] = mask.crop(request[self.mask].roi)
# restore requested ROI of points
if self.points in request:
request_roi = request[self.points].roi
points.spec.roi = request_roi
for i, p in list(points.data.items()):
if not request_roi.contains(p.location):
del points.data[i]
if len(points.data) == 0:
logger.warning("Returning empty batch for key %s and roi %s" %
(self.points, request_roi))
def provide(self, request):
timing = Timing(self)
timing.start()
batch = Batch()
for (array_key, request_spec) in request.array_specs.items():
logger.debug("Reading %s in %s...", array_key, request_spec.roi)
voxel_size = self.spec[array_key].voxel_size
# scale request roi to voxel units
dataset_roi = request_spec.roi / voxel_size
# shift request roi into dataset
dataset_roi = dataset_roi - self.spec[array_key].roi.get_offset(
) / voxel_size
# create array spec
array_spec = self.spec[array_key].copy()
array_spec.roi = request_spec.roi
# read the data
if array_key in self.datasets:
data = self.__read_array(self.datasets[array_key], dataset_roi)
elif array_key in self.masks:
data = self.__read_mask(self.masks[array_key], dataset_roi)
else:
assert False, (
"Encountered a request for %s that is neither a volume "
"nor a mask." % array_key)
# add array to batch
batch.arrays[array_key] = Array(data, array_spec)
logger.debug("done")
timing.stop()
batch.profiling_stats.add(timing)
return batch
def train_step(self, batch, request):
array_outputs = self.__collect_requested_outputs(request)
inputs = self.__collect_provided_inputs(batch)
to_compute = {
'optimizer': self.optimizer,
'loss': self.loss,
'iteration': self.iteration_increment}
to_compute.update(array_outputs)
# compute outputs, gradients, and update variables
if self.summary is not None:
outputs, summaries = self.session.run([to_compute, self.summary], feed_dict=inputs)
else:
outputs = self.session.run(to_compute, feed_dict=inputs)
for array_key in array_outputs:
spec = self.spec[array_key].copy()
spec.roi = request[array_key].roi
batch.arrays[array_key] = Array(
outputs[array_key],
spec)
batch.loss = outputs['loss']
batch.iteration = outputs['iteration'][0]
if self.summary is not None:
self.summary_saver.add_summary(summaries, batch.iteration)
if batch.iteration%self.save_every == 0:
checkpoint_name = (
self.meta_graph_filename +
'_checkpoint_%i'%batch.iteration)
logger.info(
"Creating checkpoint %s",
checkpoint_name)
self.full_saver.save(
self.session,
checkpoint_name)
def process(self, batch, request):
# do nothing if no gt binary maps were requested
if self.skip_next:
self.skip_next = False
return
for EZ_mask_type in self.EZ_masks_to_create:
binary_map_type = self.EZ_masks_to_binary_map[EZ_mask_type]
binary_map = batch.arrays[binary_map_type].data
resolution = batch.arrays[binary_map_type].resolution
EZ_mask = self.__get_exclusivezone_mask(
binary_map,
shape_EZ_mask=request.arrays[EZ_mask_type].get_shape(),
resolution=resolution)
batch.arrays[EZ_mask_type] = Array(
data=EZ_mask,
roi=request.arrays[EZ_mask_type],
resolution=resolution)
def process(self, batch, request):
outputs = Batch()
# downsample
if isinstance(self.factor, tuple):
slices = tuple(slice(None, None, k) for k in self.factor)
else:
slices = tuple(
slice(None, None, self.factor)
for i in range(batch[self.source].spec.roi.dims()))
logger.debug("downsampling %s with %s", self.source, slices)
data = batch.arrays[self.source].data[slices]
# create output array
spec = self.spec[self.target].copy()
spec.roi = request[self.target].roi
outputs.arrays[self.target] = Array(data, spec)
return outputs
def process(self, batch, request):
gt_labels = batch.arrays[self.labels_array_key]
next_id = gt_labels.data.max() + 1
gt_pos_pass = gt_labels.data
if self.ignore_array_key and self.ignore_array_key in batch.arrays:
gt_neg_pass = np.array(gt_labels.data)
gt_neg_pass[batch.arrays[self.ignore_array_key].data ==
0] = next_id
else:
gt_neg_pass = gt_pos_pass
spec = self.spec[self.malis_comp_array_key].copy()
spec.roi = request[self.labels_array_key].roi
batch.arrays[self.malis_comp_array_key] = Array(
np.array([gt_neg_pass, gt_pos_pass]), spec)
def provide(self, request):
timing = Timing(self)
timing.start()
batch = Batch()
cv = CloudVolume(self.cloudvolume_url, use_https=True, mip=self.mip)
request_spec = request.array_specs[self.array_key]
array_key = self.array_key
logger.debug("Reading %s in %s...", array_key, request_spec.roi)
voxel_size = self.array_spec.voxel_size
# scale request roi to voxel units
dataset_roi = request_spec.roi / voxel_size
# shift request roi into dataset
dataset_roi = dataset_roi - self.spec[
array_key].roi.get_offset() / voxel_size
# create array spec
array_spec = self.array_spec.copy()
array_spec.roi = request_spec.roi
# array_spec.voxel_size = array_spec.voxel_size
# add array to batch
batch.arrays[array_key] = Array(
self.__read(cv, dataset_roi),
array_spec)
logger.debug("done")
timing.stop()
batch.profiling_stats.add(timing)
return batch
def provide(self, request):
timing = Timing(self)
timing.start()
batch = Batch()
with h5py.File(self.filename, 'r') as hdf_file:
for (array_key, request_spec) in request.array_specs.items():
logger.debug("Reading %s in %s...", array_key,
request_spec.roi)
voxel_size = self.spec[array_key].voxel_size
# scale request roi to voxel units
dataset_roi = request_spec.roi / voxel_size
# shift request roi into dataset
dataset_roi = dataset_roi - self.spec[
array_key].roi.get_offset() / voxel_size
# create array spec
array_spec = self.spec[array_key].copy()
array_spec.roi = request_spec.roi
# add array to batch
batch.arrays[array_key] = Array(
self.__read(hdf_file, self.datasets[array_key],
dataset_roi), array_spec)
logger.debug("done")
timing.stop()
batch.profiling_stats.add(timing)
return batch
def provide(self, request):
timing = Timing(self)
timing.start()
batches = [
self.get_upstream_provider().request_batch(request)
for _ in range(self.num_repetitions)
]
batch = Batch()
for key, spec in request.array_specs.items():
data = np.stack([b[key].data for b in batches])
batch[key] = Array(data, batches[0][key].spec.copy())
# copy points of first batch requested
for key, spec in request.points_specs.items():
batch[key] = batches[0][key]
timing.stop()
batch.profiling_stats.add(timing)
return batch
def process(self, batch, request):
labels_roi = request[self.labels].roi
logger.debug("computing ground-truth affinities from labels")
affinities = malis.seg_to_affgraph(
batch.arrays[self.labels].data.astype(np.int32),
self.affinity_neighborhood
).astype(np.float32)
# crop affinities to original label ROI
offset = labels_roi.get_offset()
shift = -offset - self.padding_neg
crop_roi = labels_roi.shift(shift)
crop_roi /= self.spec[self.labels].voxel_size
crop = crop_roi.get_bounding_box()
logger.debug("cropping with " + str(crop))
affinities = affinities[(slice(None),)+crop]
spec = self.spec[self.affinities].copy()
spec.roi = labels_roi
batch.arrays[self.affinities] = Array(affinities, spec)
if self.affinities_mask and self.affinities_mask in request:
if self.labels_mask:
logger.debug("computing ground-truth affinities mask from "
"labels mask")
affinities_mask = malis.seg_to_affgraph(
batch.arrays[self.labels_mask].data.astype(np.int32),
self.affinity_neighborhood)
affinities_mask = affinities_mask[(slice(None),)+crop]
else:
affinities_mask = np.ones_like(affinities)
if self.unlabelled:
# 1 for all affinities between unlabelled voxels
unlabelled = (1 - batch.arrays[self.unlabelled].data)
unlabelled_mask = malis.seg_to_affgraph(
unlabelled.astype(np.int32),
self.affinity_neighborhood)
unlabelled_mask = unlabelled_mask[(slice(None),)+crop]
# 0 for all affinities between unlabelled voxels
unlabelled_mask = (1 - unlabelled_mask)
# combine with mask
affinities_mask = affinities_mask*unlabelled_mask
affinities_mask = affinities_mask.astype(np.float32)
batch.arrays[self.affinities_mask] = Array(affinities_mask, spec)
else:
if self.labels_mask is not None:
logger.warning("GT labels does have a mask, but affinities "
"mask is not requested.")
# crop labels to original label ROI
batch.arrays[self.labels] = batch.arrays[self.labels].crop(labels_roi)
# same for label mask
if self.labels_mask:
batch.arrays[self.labels_mask] = batch.arrays[self.labels_mask].crop(labels_roi)
# and unlabelled mask
if self.unlabelled:
batch.arrays[self.unlabelled] = batch.arrays[self.unlabelled].crop(labels_roi)
batch.affinity_neighborhood = self.affinity_neighborhood