def get_candidates(pipeline, setup_config, foreground):
# Data properties
voxel_size = gp.Coordinate(setup_config["VOXEL_SIZE"])
micron_scale = voxel_size[0]
# Config options
window_size = gp.Coordinate(setup_config["NMS_WINDOW_SIZE"]) * micron_scale
threshold = setup_config["NMS_THRESHOLD"]
# Candidate mode
mode = setup_config["CANDIDATE_MODE"]
# New array Key
maxima = ArrayKey("MAXIMA")
if mode == "skel":
pipeline = pipeline + Skeletonize(
foreground, maxima, min(window_size), threshold=threshold)
else:
pipeline = (
pipeline + UnSqueeze([foreground]) +
NonMaxSuppression(foreground, maxima, window_size, threshold) +
Squeeze([foreground, maxima]))
return pipeline, maxima
def process(self, batch, request):
if self.client is None:
self.client = pymongo.MongoClient(host=self.db_host)
self.db = self.client[self.db_name]
create_indices = 'nodes' not in self.db.list_collection_names()
self.cells = self.db['nodes']
if create_indices:
self.cells.create_index([(loc, pymongo.ASCENDING)
for loc in ['t', 'z', 'y', 'x']],
name='position')
self.cells.create_index([('id', pymongo.ASCENDING)],
name='id',
unique=True)
roi = batch[self.maxima].spec.roi
voxel_size = batch[self.maxima].spec.voxel_size
maxima = batch[self.maxima].data
cell_indicator = batch[self.cell_indicator].data
parent_vectors = batch[self.parent_vectors].data
cells = []
for index in np.argwhere(
maxima * cell_indicator > self.score_threshold):
index = gp.Coordinate(index)
logger.debug("Getting parent vector at index %s" % str(index))
score = cell_indicator[index]
if self.edge_length == 1:
parent_vector = tuple(
float(x) for x in parent_vectors[(Ellipsis, ) + index])
else:
parent_vector = WriteCells.get_avg_pv(parent_vectors, index,
self.edge_length)
position = roi.get_begin() + voxel_size * index
if self.volume_shape is not None and \
np.any(np.greater_equal(
position,
gp.Coordinate(self.volume_shape) * voxel_size)):
continue
cell_id = int(
math.cantor_number(roi.get_begin() / voxel_size + index))
cells.append({
'id': cell_id,
'score': float(score),
't': position[0],
'z': position[1],
'y': position[2],
'x': position[3],
'parent_vector': parent_vector
})
logger.debug("ID=%d, score=%f, parent_vector=%s" %
(cell_id, score, parent_vector))
if len(cells) > 0:
self.cells.insert_many(cells)
def setup(self):
self.ndims = self.data.shape[1]
if self.points_spec is not None:
self.provides(self.points, self.points_spec)
elif isinstance(self.points, gp.ArrayKey):
self.provides(self.points, gp.ArraySpec(voxel_size=((1, ))))
elif isinstance(self.points, gp.GraphKey):
print(self.ndims)
min_bb = gp.Coordinate(
np.floor(np.amin(self.data[:, :self.ndims], 0)))
max_bb = gp.Coordinate(
np.ceil(np.amax(self.data[:, :self.ndims], 0)) + 1)
roi = gp.Roi(min_bb, max_bb - min_bb)
logger.debug(f"Bounding Box: {roi}")
self.provides(self.points, gp.GraphSpec(roi=roi))
if self.labels is not None:
assert isinstance(self.labels, gp.ArrayKey), \
f"Label key must be an ArrayKey, \
was given {type(self.labels)}"
if self.labels_spec is not None:
self.provides(self.labels, self.labels_spec)
else:
self.provides(self.labels, gp.ArraySpec(voxel_size=((1, ))))
def setup(self):
self.provides(
self.array_key,
gp.ArraySpec(roi=gp.Roi(offset=gp.Coordinate(
(-10000, -10000, -10000)),
shape=gp.Coordinate(
(20000, 20000, 20000))),
voxel_size=(1, 1, 1)))
def test_context(self):
d_pred = gp.ArrayKeys.D_PRED
m_pred = gp.ArrayKeys.M_PRED
presyn = gp.PointsKeys.PRESYN
postsyn = gp.PointsKeys.POSTSYN
outdir = tempfile.mkdtemp()
voxel_size = gp.Coordinate((10, 10, 10))
size = ((200, 200, 200))
# Check whether the score of the entire cube is measured, although
# cube of borderpoint partially outside request ROI.
context = 40
shape = gp.Coordinate(size) / voxel_size
m_predar = np.zeros(shape, dtype=np.float32)
outsidepoint = gp.Coordinate((13, 13, 13))
borderpoint = (4, 4, 4)
m_predar[3:5, 3:5, 3:5] = 1
m_predar[outsidepoint] = 1
d_predar = np.ones((3, shape[0], shape[1], shape[2])) * 0
pipeline = (TestSource(m_predar, d_predar, voxel_size=voxel_size) +
ExtractSynapses(m_pred,
d_pred,
presyn,
postsyn,
out_dir=outdir,
settings=parameters,
context=context) +
gp.PrintProfilingStats())
request = gp.BatchRequest()
roi = gp.Roi((40, 40, 40), (80, 80, 80))
request[presyn] = gp.PointsSpec(roi=roi)
request[postsyn] = gp.PointsSpec(roi=roi)
with gp.build(pipeline):
batch = pipeline.request_batch(request)
synapsefile = os.path.join(outdir, "40", "40", "40.npz")
with np.load(synapsefile) as data:
data = dict(data)
self.assertTrue(len(data['ids']) == 1)
self.assertEqual(data['scores'][0], 2.0**3) # Size of the cube.
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][0][ii],
borderpoint[ii] * voxel_size[ii])
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][1][ii],
borderpoint[ii] * voxel_size[ii] + 0)
shutil.rmtree(outdir)
def test_output_basics(self):
d_pred = gp.ArrayKeys.D_PRED
m_pred = gp.ArrayKeys.M_PRED
presyn = gp.PointsKeys.PRESYN
postsyn = gp.PointsKeys.POSTSYN
voxel_size = gp.Coordinate((10, 10, 10))
size = ((200, 200, 200))
context = 40
shape = gp.Coordinate(size) / voxel_size
m_predar = np.zeros(shape, dtype=np.float32)
insidepoint = gp.Coordinate((10, 10, 10))
outsidepoint = gp.Coordinate((15, 15, 15))
m_predar[insidepoint] = 1
m_predar[outsidepoint] = 1
d_predar = np.ones((3, shape[0], shape[1], shape[2])) * 10
outdir = tempfile.mkdtemp()
pipeline = (TestSource(m_predar, d_predar, voxel_size=voxel_size) +
ExtractSynapses(m_pred,
d_pred,
presyn,
postsyn,
out_dir=outdir,
settings=parameters,
context=context))
request = gp.BatchRequest()
roi = gp.Roi((40, 40, 40), (80, 80, 80))
request[presyn] = gp.PointsSpec(roi=roi)
request[postsyn] = gp.PointsSpec(roi=roi)
with gp.build(pipeline):
batch = pipeline.request_batch(request)
print(outdir, "outdir")
synapsefile = os.path.join(outdir, "40", "40", "40.npz")
with np.load(synapsefile) as data:
data = dict(data)
self.assertTrue(len(data['ids']) == 1)
self.assertEqual(data['scores'][0], 1.0) # Size of the cube.
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][1][ii],
insidepoint[ii] * voxel_size[ii])
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][0][ii],
insidepoint[ii] * voxel_size[ii] + 10)
shutil.rmtree(outdir)
def get_avg_pv(parent_vectors, index, edge_length):
''' Computes the average parent vector offset from the parent vectors
in a cube centered at index. Accounts for the fact that each parent
vector is a relative offset from its source location, not from index.
Args:
parent_vectors (``np.array``):
A numpy array of parent vectors with dimensions
(channels, time, z, y, x).
index (``gp.Coordinate``):
A 4D coordiante (t, z, y, x) indicating the target
location to get the average parent vector for.
edge_length (``int``):
Length of each side of the cube within which the
parent vectors are averaged.
'''
radius = (edge_length - 1) // 2
logger.debug("Getting average parent vectors with radius"
" %d around index %s" % (radius, str(index)))
offsets = []
pv_shape = parent_vectors.shape
# channels, t, z, y, x
assert (len(pv_shape) == 5)
pv_max_z = pv_shape[2]
pv_max_y = pv_shape[3]
pv_max_x = pv_shape[4]
logger.debug("Type of index[1]: %s index[1] %s" %
(str(type(index[1])), str(index[1])))
for z in range(max(0, index[1] - radius),
min(index[1] + radius + 1, pv_max_z)):
for y in range(max(0, index[2] - radius),
min(index[2] + radius + 1, pv_max_y)):
for x in range(max(0, index[3] - radius),
min(index[3] + radius + 1, pv_max_x)):
c = gp.Coordinate((z, y, x))
c_with_time = gp.Coordinate((index[0], z, y, x))
relative_pos = c - index[1:]
offset_relative_to_c = parent_vectors[(Ellipsis, ) +
c_with_time]
offsets.append(offset_relative_to_c + relative_pos)
logger.debug("Offsets to average: %s" + str(offsets))
parent_vector = tuple(
float(sum(col) / len(col)) for col in zip(*offsets))
return parent_vector
def test_delete_points_in_context(self):
points = gp.PointsKey("POINTS")
pv_array = gp.ArrayKey("PARENT_VECTORS")
mask = gp.ArrayKey("MASK")
radius = [0.1, 0.1, 0.1, 0.1]
ts = TracksSource(TEST_FILE, points)
apv = AddParentVectors(points, pv_array, mask, radius)
request = gp.BatchRequest()
request.add(points, gp.Coordinate((1, 4, 4, 4)))
request.add(pv_array, gp.Coordinate((1, 4, 4, 4)))
request.add(mask, gp.Coordinate((1, 4, 4, 4)))
pipeline = (ts + gp.Pad(points, None) + apv)
with gp.build(pipeline):
pipeline.request_batch(request)
def _updated_spec(self, ref_spec):
spec = ref_spec.copy()
spec.dtype = np.float32
# if stardists are on downsampled grid voxel_size needs to be adapted
if self.grid != (1, 1, 1):
spec.voxel_size *= gp.Coordinate(self.grid)
return spec
def prepare(self, request):
logger.debug("request: %s", request.array_specs)
logger.debug("my spec: %s", self.spec)
shift_roi = self.__get_possible_shifts(request)
if request.array_specs.keys():
lcm_voxel_size = self.spec.get_lcm_voxel_size(
request.array_specs.keys())
shift_roi = shift_roi.snap_to_grid(lcm_voxel_size, mode='shrink')
lcm_shift_roi = shift_roi/lcm_voxel_size
logger.debug("lcm voxel size: %s", lcm_voxel_size)
logger.debug(
"restricting random locations to multiples of voxel size %s",
lcm_voxel_size)
else:
lcm_voxel_size = gp.Coordinate((1,)*shift_roi.dims())
lcm_shift_roi = shift_roi
random_shift = self.__select_random_shift(
request,
lcm_shift_roi,
lcm_voxel_size)
self.random_shift = random_shift
self.__shift_request(request, random_shift)
def test_shift_points5(self):
data = {
0: gp.Point([3, 0]),
1: gp.Point([3, 2]),
2: gp.Point([3, 4]),
3: gp.Point([3, 6]),
4: gp.Point([3, 8])
}
spec = gp.PointsSpec(gp.Roi(offset=(0, 0), shape=(15, 10)))
points = gp.Points(data, spec)
request_roi = gp.Roi(offset=(3, 0), shape=(9, 10))
shift_array = np.array([[3, 0], [-3, 0], [0, 0], [-3, 0], [3, 0]],
dtype=int)
lcm_voxel_size = gp.Coordinate((3, 2))
shifted_data = {
0: gp.Point([6, 0]),
2: gp.Point([3, 4]),
4: gp.Point([6, 8])
}
result = gp.ShiftAugment.shift_points(points,
request_roi,
shift_array,
shift_axis=1,
lcm_voxel_size=lcm_voxel_size)
# print("test 4", result.data, shifted_data)
self.assertTrue(self.points_equal(result.data, shifted_data))
self.assertTrue(result.spec == gp.PointsSpec(request_roi))
def build_source(self):
data = daisy.open_ds(filename, key)
if self.time_window is None:
source_roi = gp.Roi(data.roi.get_offset(), data.roi.get_shape())
else:
offs = list(data.roi.get_offset())
offs[1] += self.time_window[0]
sh = list(data.roi.get_shape())
offs[1] = self.time_window[1] - self.time_window[0]
source_roi = gp.Roi(tuple(offs), tuple(sh))
voxel_size = gp.Coordinate(data.voxel_size)
return gp.ZarrSource(filename,
{
self.raw_0: key,
self.raw_1: key
},
array_specs={
self.raw_0: gp.ArraySpec(
roi=source_roi,
voxel_size=voxel_size,
interpolatable=True),
self.raw_1: gp.ArraySpec(
roi=source_roi,
voxel_size=voxel_size,
interpolatable=True)
})
def __init__(self, filename, ds_name):
self.filename = filename
self.ds_name = ds_name
zarr_container = zarr.open(filename)
ds = zarr_container[ds_name]
# necessary fields:
self._shape = gp.Coordinate(ds.shape)
if "axes" in ds.attrs:
self._axes = {d: a for d, a in enumerate(ds.attrs["axes"])}
self._inv_axes = {a: d for d, a in self._axes.items()}
self._channel_dim = self._inv_axes.get("c")
self._sample_dim = self._inv_axes.get("s")
self._spatial_axes = sorted([
i for i in self._axes.keys()
if i != self.channel_dim and i != self.sample_dim
])
else:
raise DacapoConventionError(
"Dacapo expects zarr arrays to come with axis labels. "
"Note that label 'c' is reserved for the (optional) channel dimension. "
"Label 's' is reserved for the (optional) sample dimension. "
"Any other label will be treated as a spatial dimension.")
# optional fields
if "resolution" in ds.attrs:
self._voxel_size = gp.Coordinate(ds.attrs["resolution"])
else:
self._voxel_size = gp.Coordinate(
tuple(1 for i in self._spatial_axes))
if "offset" in ds.attrs:
self._offset = gp.Coordinate(ds.attrs["offset"])
else:
self._offset = gp.Coordinate(tuple(0 for i in self._spatial_axes))
# more optional fields: gt specific
if "num_classes" in ds.attrs:
self._num_classes = ds.attrs["num_classes"]
else:
self._num_classes = 0
if "background_label" in ds.attrs:
self._background_label = ds.attrs["background_label"]
else:
self._background_label = None
def test_pipeline3(self):
array_key = gp.ArrayKey("TEST_ARRAY")
points_key = gp.PointsKey("TEST_POINTS")
voxel_size = gp.Coordinate((1, 1))
spec = gp.ArraySpec(voxel_size=voxel_size, interpolatable=True)
hdf5_source = gp.Hdf5Source(self.fake_data_file,
{array_key: 'testdata'},
array_specs={array_key: spec})
csv_source = gp.CsvPointsSource(
self.fake_points_file, points_key,
gp.PointsSpec(
roi=gp.Roi(shape=gp.Coordinate((100, 100)), offset=(0, 0))))
request = gp.BatchRequest()
shape = gp.Coordinate((60, 60))
request.add(array_key, shape, voxel_size=gp.Coordinate((1, 1)))
request.add(points_key, shape)
shift_node = gp.ShiftAugment(prob_slip=0.2,
prob_shift=0.2,
sigma=5,
shift_axis=0)
pipeline = ((hdf5_source, csv_source) + gp.MergeProvider() +
gp.RandomLocation(ensure_nonempty=points_key) + shift_node)
with gp.build(pipeline) as b:
request = b.request_batch(request)
# print(request[points_key])
target_vals = [
self.fake_data[point[0]][point[1]] for point in self.fake_points
]
result_data = request[array_key].data
result_points = request[points_key].data
result_vals = [
result_data[int(point.location[0])][int(point.location[1])]
for point in result_points.values()
]
for result_val in result_vals:
self.assertTrue(
result_val in target_vals,
msg=
"result value {} at points {} not in target values {} at points {}"
.format(result_val, list(result_points.values()), target_vals,
self.fake_points))
def test_prepare1(self):
key = gp.ArrayKey("TEST_ARRAY")
spec = gp.ArraySpec(voxel_size=gp.Coordinate((1, 1)),
interpolatable=True)
hdf5_source = gp.Hdf5Source(self.fake_data_file, {key: 'testdata'},
array_specs={key: spec})
request = gp.BatchRequest()
shape = gp.Coordinate((3, 3))
request.add(key, shape, voxel_size=gp.Coordinate((1, 1)))
shift_node = gp.ShiftAugment(sigma=1, shift_axis=0)
with gp.build((hdf5_source + shift_node)):
shift_node.prepare(request)
self.assertTrue(shift_node.ndim == 2)
self.assertTrue(shift_node.shift_sigmas == tuple([0.0, 1.0]))
def test_squeeze_not_possible(self):
raw = gp.ArrayKey("RAW")
labels = gp.ArrayKey("LABELS")
voxel_size = gp.Coordinate((50, 5, 5))
input_voxels = gp.Coordinate((5, 5, 5))
input_size = input_voxels * voxel_size
request = gp.BatchRequest()
request.add(raw, input_size)
request.add(labels, input_size)
pipeline = (ExampleSourceSqueeze(voxel_size) +
gp.Squeeze([raw], axis=2))
with self.assertRaises(gp.PipelineRequestError):
with gp.build(pipeline) as p:
batch = p.request_batch(request)
def predict(iteration,path_to_dataGP):
input_size = (8, 96, 96)
output_size = (4, 64, 64)
amount_size = gp.Coordinate((2, 16, 16))
model = SpineUNet(crop_output='output_size')
raw = gp.ArrayKey('RAW')
affs_predicted = gp.ArrayKey('AFFS_PREDICTED')
reference_request = gp.BatchRequest()
reference_request.add(raw, input_size)
reference_request.add(affs_predicted, output_size)
source = gp.ZarrSource(
path_to_dataGP,
{
raw: 'validate/sample1/raw'
}
)
with gp.build(source):
source_roi = source.spec[raw].roi
request = gp.BatchRequest()
request[raw] = gp.ArraySpec(roi=source_roi)
request[affs_predicted] = gp.ArraySpec(roi=source_roi)
pipeline = (
source +
gp.Pad(raw,amount_size) +
gp.Normalize(raw) +
# raw: (d, h, w)
gp.Stack(1) +
# raw: (1, d, h, w)
AddChannelDim(raw) +
# raw: (1, 1, d, h, w)
gp_torch.Predict(
model,
inputs={'x': raw},
outputs={0: affs_predicted},
checkpoint=f'C:/Users/filip/spine_yodl/model_checkpoint_{iteration}') +
RemoveChannelDim(raw) +
RemoveChannelDim(raw) +
RemoveChannelDim(affs_predicted) +
# raw: (d, h, w)
# affs_predicted: (3, d, h, w)
gp.Scan(reference_request)
)
with gp.build(pipeline):
prediction = pipeline.request_batch(request)
return prediction[raw].data, prediction[affs_predicted].data
def test_pipeline2(self):
key = gp.ArrayKey("TEST_ARRAY")
spec = gp.ArraySpec(voxel_size=gp.Coordinate((3, 1)),
interpolatable=True)
hdf5_source = gp.Hdf5Source(self.fake_data_file, {key: 'testdata'},
array_specs={key: spec})
request = gp.BatchRequest()
shape = gp.Coordinate((3, 3))
request.add(key, shape, voxel_size=gp.Coordinate((3, 1)))
shift_node = gp.ShiftAugment(prob_slip=0.2,
prob_shift=0.2,
sigma=1,
shift_axis=0)
with gp.build((hdf5_source + shift_node)) as b:
b.request_batch(request)
def get_roi_from_swc(swc_cube, transform_file, voxel_size):
graph = parse_swc(
filename=swc_cube,
transform=transform_file,
offset=np.array([0, 0, 0]),
resolution=np.array(voxel_size),
transpose=[2, 1, 0],
)
mins = np.array([float("inf")] * 3)
maxs = np.array([-float("inf")] * 3)
for attrs in graph.nodes.values():
u_loc = attrs["location"]
min_stack = np.stack([mins, np.array(u_loc)])
max_stack = np.stack([maxs, np.array(u_loc)])
mins = np.min(min_stack, axis=0)
maxs = np.max(max_stack, axis=0)
return gp.Roi(gp.Coordinate(tuple(mins)),
gp.Coordinate(tuple(maxs - mins)))
def __select_random_location(self, lcm_shift_roi, lcm_voxel_size):
# select a random point inside ROI
random_shift = gp.Coordinate(
randint(int(begin), int(end-1))
for begin, end in zip(lcm_shift_roi.get_begin(), lcm_shift_roi.get_end()))
random_shift *= lcm_voxel_size
return random_shift
def test_squeeze(self):
raw = gp.ArrayKey("RAW")
labels = gp.ArrayKey("LABELS")
voxel_size = gp.Coordinate((50, 5, 5))
input_voxels = gp.Coordinate((5, 5, 5))
input_size = input_voxels * voxel_size
request = gp.BatchRequest()
request.add(raw, input_size)
request.add(labels, input_size)
pipeline = (ExampleSourceSqueeze(voxel_size) +
gp.Squeeze([raw], axis=1) + gp.Squeeze([raw, labels]))
with gp.build(pipeline) as p:
batch = p.request_batch(request)
assert batch[raw].data.shape == input_voxels
assert batch[labels].data.shape == input_voxels
def get_foreground_pipelines(config, blocks):
voxel_size = gp.Coordinate(config["VOXEL_SIZE"])
input_shape = gp.Coordinate(config["INPUT_SHAPE"])
output_shape = gp.Coordinate(config["OUTPUT_SHAPE"])
input_size = voxel_size * input_shape
output_size = voxel_size * output_shape
raw_pipelines, (raw, ) = get_raw_snapshot_source(config, blocks)
labels_pipelines, (labels, gt) = get_labels_snapshot_source(config, blocks)
fg_pipelines, (fg_pred, ) = add_fg_preds(config, raw_pipelines, raw)
fg_pipelines = add_scans(fg_pipelines, {
raw: input_size,
fg_pred: output_size
})
pipelines = merge_pipelines(fg_pipelines, labels_pipelines)
requests = get_requests(config, blocks, raw, fg_pred, labels, gt)
return pipelines, requests, (raw, fg_pred, labels, gt)
def provide(self, request):
voxel_size = self.spec[self.raw].voxel_size
shape = gp.Coordinate((1, ) + request[self.raw].roi.get_shape())
noise = np.abs(np.random.randn(*shape))
smoothed_noise = gaussian_filter(noise, sigma=self.smoothness)
seeds = np.zeros(shape, dtype=int)
for i in range(self.n_objects):
if i == 0:
num_points = 100
else:
num_points = self.points_per_skeleton
points = np.stack(
[
np.random.randint(0, shape[dim], num_points)
for dim in range(3)
],
axis=1,
)
tree = skelerator.Tree(points)
skeleton = skelerator.Skeleton(tree, [1, 1, 1],
"linear",
generate_graph=False)
seeds = skeleton.draw(seeds, np.array([0, 0, 0]), i + 1)
seeds[maximum_filter(seeds, size=4) != seeds] = 0
seeds_dt = distance_transform_edt(seeds == 0) + 5.0 * smoothed_noise
gt_data = cwatershed(seeds_dt, seeds).astype(np.uint64)[0] - 1
labels = np.unique(gt_data)
raw_data = np.zeros_like(gt_data, dtype=np.uint8)
value = 0
for label in labels:
raw_data[gt_data == label] = value
value += 255.0 / self.n_objects
spec = request[self.raw].copy()
spec.voxel_size = (1, 1)
raw = gp.Array(raw_data, spec)
spec = request[self.gt].copy()
spec.voxel_size = (1, 1)
gt_crop = (request[self.gt].roi -
request[self.raw].roi.get_begin()) / voxel_size
gt_crop = gt_crop.to_slices()
gt = gp.Array(gt_data[gt_crop], spec)
batch = gp.Batch()
batch[self.raw] = raw
batch[self.gt] = gt
return batch
def prepare(self, request):
context = self.context
dims = request[self.srcpoints].roi.dims()
assert type(context) == list
if len(context) == 1:
context = context * dims
# request array in a larger area to get predictions from outside
# write roi
m_roi = request[self.srcpoints].roi.grow(gp.Coordinate(context),
gp.Coordinate(context))
# however, restrict the request to the array actually provided
# m_roi = m_roi.intersect(self.spec[self.m_array].roi)
request[self.m_array] = gp.ArraySpec(roi=m_roi)
# Do the same for the direction vector array.
request[self.d_array] = gp.ArraySpec(roi=m_roi)
def get_requests(config, blocks, raw, emb_pred, labels, gt):
voxel_size = gp.Coordinate(config["VOXEL_SIZE"])
input_shape = gp.Coordinate(config["INPUT_SHAPE"])
output_shape = gp.Coordinate(config["OUTPUT_SHAPE"])
input_size = voxel_size * input_shape
output_size = voxel_size * output_shape
diff = input_size - output_size
cube_rois = [get_cube_roi(config, block) for block in blocks]
requests = []
for cube_roi in cube_rois:
context_roi = cube_roi.grow(diff // 2, diff // 2)
request = gp.BatchRequest()
request[raw] = gp.ArraySpec(roi=context_roi)
request[emb_pred] = gp.ArraySpec(roi=cube_roi)
request[labels] = gp.ArraySpec(roi=cube_roi)
request[gt] = gp.GraphSpec(roi=cube_roi)
requests.append(request)
return requests
def __init__(self, filename,
key,
density=None,
channels=0,
shape=(16, 256, 256),
time_window=None,
add_sparse_mosaic_channel=True,
random_rot=False):
self.filename = filename
self.key = key
self.shape = shape
self.density = density
self.raw = gp.ArrayKey('RAW_0')
self.add_sparse_mosaic_channel = add_sparse_mosaic_channel
self.random_rot = random_rot
self.channels = channels
data = daisy.open_ds(filename, key)
if time_window is None:
source_roi = gp.Roi(data.roi.get_offset(), data.roi.get_shape())
else:
offs = list(data.roi.get_offset())
offs[1] += time_window[0]
sh = list(data.roi.get_shape())
offs[1] = time_window[1] - time_window[0]
source_roi = gp.Roi(tuple(offs), tuple(sh))
voxel_size = gp.Coordinate(data.voxel_size)
self.pipeline = gp.ZarrSource(
filename,
{
self.raw: key
},
array_specs={
self.raw: gp.ArraySpec(
roi=source_roi,
voxel_size=voxel_size,
interpolatable=True)
}) + gp.RandomLocation() + IntensityDiffFilter(self.raw, 0, min_distance=0.1, channels=Slice(None))
# add augmentations
self.pipeline = self.pipeline + gp.ElasticAugment([40, 40],
[2, 2],
[0, math.pi / 2.0],
prob_slip=-1,
spatial_dims=2)
self.pipeline.setup()
np.random.seed(os.getpid() + int(time.time()))
def __read_spec(self, array_key):
if array_key in self.array_specs:
spec = self.array_specs[array_key].copy()
else:
spec = gp.ArraySpec()
assert spec.voxel_size is not None, "Voxel size needs to be given"
self.ndims = len(spec.voxel_size)
if spec.roi is None:
roi = gp.Roi(gp.Coordinate((0, ) * self.ndims),
shape=gp.Coordinate((1, ) * self.ndims))
roi.set_shape(None)
spec.roi = roi
arr = self.func((2, ) * self.ndims)
if spec.dtype is not None:
assert spec.dtype == arr.dtype, (
"dtype %s provided in array_specs for %s, "
"but differs from function output %s dtype %s" %
(self.array_specs[array_key].dtype, array_key, self.func,
arr.dtype))
else:
spec.dtype = arr.dtype
if spec.interpolatable is None:
spec.interpolatable = spec.dtype in [
np.float,
np.float32,
np.float64,
np.float128,
np.uint8 # assuming this is not used for labels
]
logger.warning(
"WARNING: You didn't set 'interpolatable' for %s "
"(func %s) . Based on the dtype %s, it has been "
"set to %s. This might not be what you want.", array_key,
self.func, spec.dtype, spec.interpolatable)
return spec
def test_get_sub_shift_array2(self):
total_roi = gp.Roi(offset=(0, 0), shape=(6, 6))
item_roi = gp.Roi(offset=(1, 2), shape=(3, 3))
shift_array = np.arange(12).reshape(6, 2).astype(int)
shift_axis = 0
lcm_voxel_size = gp.Coordinate((1, 1))
sub_shift_array = np.array([[2, 3], [4, 5], [6, 7]], dtype=int)
result = gp.ShiftAugment.get_sub_shift_array(total_roi, item_roi,
shift_array, shift_axis,
lcm_voxel_size)
self.assertTrue(np.array_equal(result, sub_shift_array))
def grow_labels(pipeline, setup_config, labels):
# Data Properties
voxel_size = gp.Coordinate(setup_config["VOXEL_SIZE"])
micron_scale = voxel_size[0]
label_radius = setup_config["NEURON_RADIUS"]
pipeline = pipeline + GrowLabels(labels,
radii=[label_radius * micron_scale])
return pipeline
def validation_data_sources_from_snapshots(config, blocks):
validation_blocks = Path(config["VALIDATION_BLOCKS"])
raw = gp.ArrayKey("RAW")
ground_truth = gp.GraphKey("GROUND_TRUTH")
labels = gp.ArrayKey("LABELS")
voxel_size = gp.Coordinate(config["VOXEL_SIZE"])
input_shape = gp.Coordinate(config["INPUT_SHAPE"])
output_shape = gp.Coordinate(config["OUTPUT_SHAPE"])
input_size = voxel_size * input_shape
output_size = voxel_size * output_shape
block_pipelines = []
for block in blocks:
pipelines = (
SnapshotSource(
validation_blocks / f"block_{block}.hdf",
{
labels: "volumes/labels",
ground_truth: "points/gt"
},
directed={ground_truth: True},
),
SnapshotSource(validation_blocks / f"block_{block}.hdf",
{raw: "volumes/raw"}),
)
cube_roi = get_cube_roi(config, block)
request = gp.BatchRequest()
input_roi = cube_roi.grow((input_size - output_size) // 2,
(input_size - output_size) // 2)
request[raw] = gp.ArraySpec(input_roi)
request[ground_truth] = gp.GraphSpec(cube_roi)
request[labels] = gp.ArraySpec(cube_roi)
block_pipelines.append((pipelines, request))
return block_pipelines, (raw, labels, ground_truth)
