def provide(self, request):
outputs = gp.Batch()
# RAW
raw_spec = copy.deepcopy(self.array_spec_raw)
raw_spec.roi = request[self.raw].roi
raw_shape = request[self.raw].roi.get_shape() / self.voxel_size
outputs[self.raw] = gp.Array(
np.random.randint(0, 256, raw_shape, dtype=raw_spec.dtype),
raw_spec)
# Unsqueeze
outputs[self.raw].data = np.expand_dims(outputs[self.raw].data, axis=0)
outputs[self.raw].data = np.expand_dims(outputs[self.raw].data, axis=0)
# LABELS
labels_spec = copy.deepcopy(self.array_spec_labels)
labels_spec.roi = request[self.labels].roi
labels_shape = request[self.labels].roi.get_shape() / self.voxel_size
labels = np.ones(labels_shape, dtype=labels_spec.dtype)
outputs[self.labels] = gp.Array(labels, labels_spec)
# Unsqueeze
outputs[self.labels].data = np.expand_dims(outputs[self.labels].data,
axis=0)
return outputs
def provide(self, request):
voxel_size = self.spec[self.raw].voxel_size
shape = gp.Coordinate((1, ) + request[self.raw].roi.get_shape())
noise = np.abs(np.random.randn(*shape))
smoothed_noise = gaussian_filter(noise, sigma=self.smoothness)
seeds = np.zeros(shape, dtype=int)
for i in range(self.n_objects):
if i == 0:
num_points = 100
else:
num_points = self.points_per_skeleton
points = np.stack(
[
np.random.randint(0, shape[dim], num_points)
for dim in range(3)
],
axis=1,
)
tree = skelerator.Tree(points)
skeleton = skelerator.Skeleton(tree, [1, 1, 1],
"linear",
generate_graph=False)
seeds = skeleton.draw(seeds, np.array([0, 0, 0]), i + 1)
seeds[maximum_filter(seeds, size=4) != seeds] = 0
seeds_dt = distance_transform_edt(seeds == 0) + 5.0 * smoothed_noise
gt_data = cwatershed(seeds_dt, seeds).astype(np.uint64)[0] - 1
labels = np.unique(gt_data)
raw_data = np.zeros_like(gt_data, dtype=np.uint8)
value = 0
for label in labels:
raw_data[gt_data == label] = value
value += 255.0 / self.n_objects
spec = request[self.raw].copy()
spec.voxel_size = (1, 1)
raw = gp.Array(raw_data, spec)
spec = request[self.gt].copy()
spec.voxel_size = (1, 1)
gt_crop = (request[self.gt].roi -
request[self.raw].roi.get_begin()) / voxel_size
gt_crop = gt_crop.to_slices()
gt = gp.Array(gt_data[gt_crop], spec)
batch = gp.Batch()
batch[self.raw] = raw
batch[self.gt] = gt
return batch
def provide(self, request):
roi_array = request[gp.ArrayKeys.M_PRED].roi
batch = gp.Batch()
batch.arrays[gp.ArrayKeys.M_PRED] = gp.Array(
self.m_pred[(roi_array / self.voxel_size).to_slices()],
spec=gp.ArraySpec(roi=roi_array, voxel_size=self.voxel_size))
slices = (roi_array / self.voxel_size).to_slices()
batch.arrays[gp.ArrayKeys.D_PRED] = gp.Array(
self.d_pred[:, slices[0], slices[1], slices[2]],
spec=gp.ArraySpec(roi=roi_array, voxel_size=self.voxel_size))
return batch
def process(self, batch, request):
ids_0 = set([n.id for n in batch[self.points_0].nodes])
ids_1 = set([n.id for n in batch[self.points_1].nodes])
common_ids = ids_0.intersection(ids_1)
locations_0 = []
locations_1 = []
# get list of only xy locations
# locations are in voxels, relative to output roi
points_roi = request[self.points_0].roi
voxel_size = batch[self.raw_0].spec.voxel_size
for i in common_ids:
location_0 = np.array(batch[self.points_0].node(i).location)
location_1 = np.array(batch[self.points_1].node(i).location)
if not points_roi.contains(location_0):
print(f"skipping point {i} at {location_0}")
continue
if not points_roi.contains(location_1):
print(f"skipping point {i} at {location_1}")
continue
location_0 -= points_roi.get_begin()
location_1 -= points_roi.get_begin()
location_0 /= voxel_size
location_1 /= voxel_size
locations_0.append(location_0)
locations_1.append(location_1)
locations_0 = np.array(locations_0, dtype=np.float32)
locations_1 = np.array(locations_1, dtype=np.float32)
if self.is_2d:
locations_0 = locations_0[:, 1:]
locations_1 = locations_1[:, 1:]
locations_0 = locations_0[np.newaxis]
locations_1 = locations_1[np.newaxis]
# create point location arrays (with batch dimension)
batch[self.locations_0] = gp.Array(locations_0,
self.spec[self.locations_0])
batch[self.locations_1] = gp.Array(locations_1,
self.spec[self.locations_1])
# add batch dimension to raw
batch[self.raw_0].data = batch[self.raw_0].data[np.newaxis, :]
batch[self.raw_1].data = batch[self.raw_1].data[np.newaxis, :]
# make sure raw is float32
batch[self.raw_0].data = batch[self.raw_0].data.astype(np.float32)
batch[self.raw_1].data = batch[self.raw_1].data.astype(np.float32)
def provide(self, request):
timing = Timing(self)
timing.start()
batch = gp.Batch()
# If a Array is requested then we will randomly choose
# the number of requested points
if isinstance(self.points, gp.ArrayKey):
points = np.random.choice(self.data.shape[0], self.num_points)
data = self.data[points][np.newaxis]
if self.scale is not None:
data = data * self.scale
if self.label_data is not None:
labels = self.label_data[points]
batch[self.points] = gp.Array(data, self.spec[self.points])
else:
# If a graph is request we must select points within the
# request ROI
min_bb = request[self.points].roi.get_begin()
max_bb = request[self.points].roi.get_end()
logger.debug("Points source got request for %s",
request[self.points].roi)
point_filter = np.ones((self.data.shape[0], ), dtype=np.bool)
for d in range(self.ndims):
point_filter = np.logical_and(point_filter,
self.data[:, d] >= min_bb[d])
point_filter = np.logical_and(point_filter,
self.data[:, d] < max_bb[d])
points_data, labels = self._get_points(point_filter)
logger.debug(f"Found {len(points_data)} points")
points_spec = gp.GraphSpec(roi=request[self.points].roi.copy())
batch.graphs[self.points] = gp.Graph(points_data, [], points_spec)
# Labels will always be an Array
if self.label_data is not None:
batch[self.labels] = gp.Array(labels, self.spec[self.labels])
timing.stop()
batch.profiling_stats.add(timing)
return batch
def provide(self, request):
timing = gp.profiling.Timing(self)
timing.start()
batch = gp.Batch()
for (array_key, request_spec) in request.array_specs.items():
logger.debug("Reading %s in %s...", array_key, request_spec.roi)
voxel_size = self.spec[array_key].voxel_size
# scale request roi to voxel units
dataset_roi = request_spec.roi / voxel_size
# shift request roi into dataset
# dataset_roi = dataset_roi - self.spec[array_key].roi.get_offset() / voxel_size
# create array spec
array_spec = self.spec[array_key].copy()
array_spec.roi = request_spec.roi
# add array to batch
batch.arrays[array_key] = gp.Array(
self.func(dataset_roi.get_shape()), array_spec)
logger.debug("done")
timing.stop()
batch.profiling_stats.add(timing)
return batch
def process(self, batch, request):
mask = (batch[self.array].data > self.threshold).astype(np.float32)
mask_spec = self.spec[self.mask].copy()
mask_spec.roi = batch[self.array].spec.roi
batch[self.mask] = gp.Array(mask, mask_spec)
return batch
def process(self, batch, request):
mask = (batch[self.array].data > self.threshold).astype(np.uint32)
mask_spec = copy.deepcopy(batch[self.array].spec)
mask_spec.dtype = np.uint32
batch[self.mask] = gp.Array(mask, mask_spec)
return batch
def process(self, batch, request):
spec = batch[self.gt].spec.copy()
spec.dtype = np.int32
binarized = gp.Array(data=(batch[self.gt].data > 0).astype(np.int32), spec=spec)
batch[self.gt_binary] = binarized
def provide(self, request):
batch = gp.Batch()
for (array_key, request_spec) in request.array_specs.items():
array_spec = self.spec[array_key].copy()
array_spec.roi = request_spec.roi
print "array_spec: ", array_spec.roi.get_shape()
data = np.zeros((array_spec.roi.get_shape()))
batch.arrays[array_key] = gp.Array(data, array_spec)
return batch
def process(self, batch, request):
output = gp.Batch()
gt_array = NumpyArray.from_gp_array(batch[self.gt_key])
target_array = self.predictor.create_target(gt_array)
mask_array = NumpyArray.from_gp_array(batch[self.mask_key])
weight_array = self.predictor.create_weight(
gt_array, target_array, mask=mask_array
)
request_spec = request[self.target_key]
request_spec.voxel_size = gt_array.voxel_size
output[self.target_key] = gp.Array(target_array[request_spec.roi], request_spec)
request_spec = request[self.weights_key]
request_spec.voxel_size = gt_array.voxel_size
output[self.weights_key] = gp.Array(
weight_array[request_spec.roi], request_spec
)
return output
def process(self, batch, request):
if self.target_key not in request:
return
arr = batch.arrays[self.source_key].data
func_output = self.func(arr)
spec = self.spec[self.target_key].copy()
spec.roi = request[self.target_key].roi
batch.arrays[self.target_key] = gp.Array(func_output, spec)
def process(self, batch, request):
if (self.sum_array_key not in request):
return
sum_arr = batch.arrays[self.array_keys[0]].data
for ak in self.array_keys[1:]:
sum_arr += batch.arrays[ak].data
spec = self.spec[self.sum_array_key].copy()
spec.roi = request[self.sum_array_key].roi
batch.arrays[self.sum_array_key] = gp.Array(sum_arr, spec)
def provide(self, request):
voxel_size = self.spec[self.raw].voxel_size
shape = gp.Coordinate((1, ) + request[self.raw].roi.get_shape())
gt_data = np.zeros(shape, dtype=int)
for i in range(self.n_objects):
points = np.stack(
[np.random.randint(0, shape[dim], 2) for dim in range(3)],
axis=1)
tree = skelerator.Tree(points)
skeleton = skelerator.Skeleton(tree, [1, 1, 1],
"linear",
generate_graph=False)
gt_data = skeleton.draw(gt_data, np.array([0, 0, 0]), i + 1)
gt_data = gt_data[0].astype(np.uint64)
gt_data = maximum_filter(gt_data, size=2)
labels = np.unique(gt_data)
raw_data = (gt_data > 0).astype(np.float32)
raw_data = np.clip(
raw_data + np.random.normal(scale=0.1, size=raw_data.shape), 0,
1).astype(np.float32)
spec = request[self.raw].copy()
spec.voxel_size = (1, 1)
raw = gp.Array(raw_data, spec)
spec = request[self.gt].copy()
spec.voxel_size = (1, 1)
gt_crop = (request[self.gt].roi -
request[self.raw].roi.get_begin()) / voxel_size
gt_crop = gt_crop.to_slices()
gt = gp.Array(gt_data[gt_crop], spec)
batch = gp.Batch()
batch[self.raw] = raw
batch[self.gt] = gt
return batch
def process(self, batch, request):
mask = batch[self.array].data.copy() > 0
spec = batch[self.array].spec.copy()
spec.dtype = np.float32
bg = np.logical_not(mask)
dt = np.tanh(ndimage.distance_transform_edt(mask) / self.scaling)
dt[bg] = np.tanh(ndimage.distance_transform_edt(bg) / self.scaling)[bg] * (-1)
batch[self.dt] = gp.Array(data=dt.astype(np.float32), spec=spec)
def process(self, batch, request):
spec = self.spec[self.fg].copy()
voxel_size = (1, ) + spec.voxel_size
merged = np.stack([batch[self.fg].data, batch[self.bg].data], axis=0)
batch[self.raw] = gp.Array(
data=merged.astype(spec.dtype),
spec=gp.ArraySpec(dtype=spec.dtype,
roi=Roi((0, 0, 0, 0), merged.shape) * voxel_size,
interpolatable=True,
voxel_size=voxel_size))
def process(self, batch, request):
if self.target_key not in request:
return
prod_arr = batch.arrays[self.keys[0]].data
for key in self.keys[1:]:
prod_arr *= batch.arrays[key].data
if self.target_spec.dtype is not None:
prod_arr.astype(self.target_spec.dtype)
spec = self.spec[self.target_key].copy()
spec.roi = request[self.target_key].roi
spec.dtype = prod_arr.dtype
batch.arrays[self.target_key] = gp.Array(prod_arr, spec)
def process(self, batch, request):
spec = batch[self.labels].spec.copy()
spec.dtype = np.uint8
reduce_channels = np.max(batch[self.labels].data > 0, axis=0).astype(np.uint8)
binarized = gp.Array(
data=reduce_channels,
spec=spec)
batch[self.labels_binary] = binarized
def process(self, batch, request):
outputs = gp.Batch()
if self.in_array not in batch:
return
data = batch[self.in_array].data
spec = batch[self.in_array].spec.copy()
spec.dtype = np.bool
binarized = data != self.target
outputs[self.out_array] = gp.Array(binarized, spec)
return outputs
def process(self, batch, request):
array = batch[self.array]
intensity_data = np.zeros_like(array.data, dtype=np.float32)
for i, label in enumerate(np.unique(array.data)):
if label == 0:
continue
mask = array.data == label
intensity_data[mask] = np.maximum(
intensity_data[mask],
self.intensities[i % len(self.intensities)])
spec = array.spec
spec.dtype = np.float32
batch[self.array] = gp.Array(intensity_data, spec)
def process(self, batch, request):
# get the raw and segmentation arrays from the current batch
raw = batch[self.raw]
seg = batch[self.seg]
print(f"RAW: {raw}")
print(f"SEG: {seg}")
# simulate cages, return brembow volumes for raw, cages, and density
simulated_raw = Volume(raw.data, raw.spec.voxel_size)
cage_map, density_map = simulate_random_cages(
simulated_raw, Volume(seg.data, seg.spec.voxel_size), self.cages,
self.min_density, self.max_density, self.psf, True, True,
self.no_cage_probability)
# create array specs for new gunpowder arrays
raw_spec = batch[self.raw].spec.copy()
cage_map_spec = batch[self.seg].spec.copy()
cage_map_spec.dtype = np.uint64
density_map_spec = batch[self.seg].spec.copy()
density_map_spec.dtype = np.float32
# create arrays and crop to requested size
print(cage_map_spec)
cage_map_array = gp.Array(data=cage_map, spec=cage_map_spec)
cage_map_array = cage_map_array.crop(request[self.cage_map].roi)
density_map_array = gp.Array(data=density_map, spec=density_map_spec)
density_map_array = density_map_array.crop(
request[self.density_map].roi)
# create a new batch with processed arrays
processed = gp.Batch()
processed[self.raw] = gp.Array(data=simulated_raw.data, spec=raw_spec)
processed[self.cage_map] = cage_map_array
processed[self.density_map] = density_map_array
return processed
def process(self, batch, request):
final_scores = {}
for key, array in batch.items():
if "SCORE" in str(key):
block = int(str(key).split("_")[1])
final_scores[block] = array.data
final_scores = [
final_scores[block] for block in range(1, 26)
if block in final_scores
]
outputs = gp.Batch()
outputs[self.output] = gp.Array(np.array(final_scores),
gp.ArraySpec(nonspatial=True))
return outputs
def process(self, batch, request):
random_pick = np.random.choice(self.input_labels, p=self.probabilities)
output_label = batch.arrays[random_pick].data.copy()
# logger.info(random_pick)
spec = self.spec[self.output_label].copy()
spec.roi = request[self.output_label].roi.copy()
batch.arrays[self.output_label] = gp.Array(output_label, spec)
for i in self.input_labels:
roi = request[i].roi
batch.arrays[i] = batch.arrays[i].crop(roi)
def process(self, batch, request):
outputs = gp.Batch()
# logger.debug("upsampeling %s with %s", self.source, self.factor)
# resize
data = batch.arrays[self.source].data
data = rescale(data, self.factor)
# create output array
spec = self.spec[self.target].copy()
spec.roi = request[self.target].roi
outputs.arrays[self.target] = gp.Array(data, spec)
return outputs
def evaluate_affs(pred_labels, gt_labels, return_results=False):
results = rand_voi(gt_labels.data, pred_labels.data)
results["voi_sum"] = results["voi_split"] + results["voi_merge"]
scores = {"sample": results, "average": results}
if return_results:
results = {
"pred_labels":
gp.Array(
pred_labels.data.astype(np.uint64),
gp.ArraySpec(roi=pred_labels.spec.roi,
voxel_size=pred_labels.spec.voxel_size)),
"gt_labels":
gp.Array(
gt_labels.data.astype(np.uint64),
gp.ArraySpec(roi=gt_labels.spec.roi,
voxel_size=gt_labels.spec.voxel_size)),
}
return scores, results
return scores
def process(self, batch, request):
joined_affinities = batch.arrays[self.joined_affinities].data.copy()
raw = random_noise(joined_affinities, 's&p', amount=self.sp)
raw = gaussian(raw, self.sigma)
# plt.imshow(raw[66], cmap="Greys_r")
raw = raw * self.contrast
# plt.show()
spec = self.spec[self.raw].copy()
spec.roi = request[self.raw].roi.copy()
batch.arrays[self.raw] = gp.Array(raw, spec)
roi = request[self.joined_affinities].roi
batch.arrays[self.joined_affinities] = batch.arrays[
self.joined_affinities].crop(roi)
def process(self, batch, request):
input_data = batch[self.input_array].data
input_spec = batch[self.input_array].spec
input_roi = input_spec.roi
output_roi_shape = request[self.output_array].roi.get_shape()
shift = (input_roi.get_shape() - output_roi_shape) / 2
output_roi = gp.Roi(shift, output_roi_shape)
output_data = input_data[tuple(
map(
slice,
output_roi.get_begin() / input_spec.voxel_size,
output_roi.get_end() / input_spec.voxel_size,
))]
output_spec = copy.deepcopy(input_spec)
output_spec.roi = output_roi
output_array = gp.Array(output_data, output_spec)
batch[self.output_array] = output_array
def provide(self, request):
output = gp.Batch()
timing_provide = Timing(self, "provide")
timing_provide.start()
spec = self.array_spec.copy()
spec.roi = request[self.key].roi
data = self.array[spec.roi]
if "c" not in self.array.axes:
# add a channel dimension
data = np.expand_dims(data, 0)
if np.any(np.isnan(data)):
raise ValueError("INPUT DATA CAN'T BE NAN")
output[self.key] = gp.Array(data, spec=spec)
timing_provide.stop()
output.profiling_stats.add(timing_provide)
return output
def provide(self, request):
batch = gp.Batch()
# print "n:", self.n
# print "pid: ", mp.current_process().pid
for (array_key, request_spec) in request.array_specs.items():
array_spec = self.spec[array_key].copy()
array_spec.roi = request_spec.roi
shape = array_spec.roi.get_shape()
# enlarge
lshape = list(shape)
inc = [0] * len(shape)
for i, s in enumerate(shape):
if s % 2 != 0:
inc[i] += 1
lshape[i] += 1
shape = gp.Coordinate(lshape)
data = create_segmentation(
shape=shape,
n_objects=self.n_objects,
points_per_skeleton=self.points_per_skeleton,
interpolation=self.interpolation,
smoothness=self.smoothness,
noise_strength=self.noise_strength,
seed=self.seed)
# seed=np.random.randint(10000))
segmentation = data["segmentation"]
# crop (more elegant & general way to do this?)
segmentation = segmentation[:lshape[0] - inc[0], :lshape[1] -
inc[1], :lshape[2] - inc[2]]
# segmentation = segmentation[:lshape_out[i] - inc[i] for i in range(len(shape))]
batch.arrays[array_key] = gp.Array(segmentation, array_spec)
# self.n +=1
return batch
def process(self, batch, request):
input_affinities = batch.arrays[self.input_affinities].data.copy()
affs1 = input_affinities[0]
affs2 = input_affinities[1]
affs3 = input_affinities[2]
joined_affinities = np.logical_and(affs1 == 1, affs2 == 1)
joined_affinities = np.logical_and(joined_affinities, affs3 == 1)
# crop to requested ROI
joined_affinities_roi = request[self.joined_affinities].roi.copy()
joined_affinities = self.__crop_center(
joined_affinities, joined_affinities_roi.get_shape())
spec = self.spec[self.joined_affinities].copy()
spec.roi = request[self.joined_affinities].roi.copy()
batch.arrays[self.joined_affinities] = gp.Array(
joined_affinities, spec)
roi = request[self.input_affinities].roi
batch.arrays[self.input_affinities] = batch.arrays[
self.input_affinities].crop(roi)
