def __getitem__(self, index):
request = gp.BatchRequest()
# request.add(self.raw_0, (1, ) + tuple(self.shape))
# request.add(self.raw_0, (1, ) + tuple(self.shape))
direction = random.randrange(0, self.max_direction)
offset = offset_from_direction(direction,
max_direction=self.max_direction,
distance=self.distance)
if self.upsample is not None:
ref0, ref1 = self.raw_0_us, self.raw_1_us
else:
ref0, ref1 = self.raw_0, self.raw_1
request[ref0] = gp.Roi((0, 0, 0), (1, 256, 256))
request[ref1] = gp.Roi((0, ) + offset, (1, 256, 256))
batch = self.pipeline.request_batch(request)
out0 = batch[ref0].data
out1 = batch[ref1].data
raw_stack = np.concatenate((out0, out1), axis=0)
shift_direction = np.array(direction, dtype=np.int64)
return raw_stack, shift_direction
def process(self, batch, request):
outputs = gp.Batch()
gt_graph = nx.Graph()
mst_graph = nx.Graph()
for block, block_specs in self.specs.items():
ground_truth_key = block_specs["ground_truth"][0]
mst_key = block_specs["mst_pred"][0]
block_gt_graph = batch[ground_truth_key].to_nx_graph(
).to_undirected()
block_mst_graph = batch[mst_key].to_nx_graph().to_undirected()
gt_graph = nx.disjoint_union(gt_graph, block_gt_graph)
mst_graph = nx.disjoint_union(mst_graph, block_mst_graph)
for node, attrs in gt_graph.nodes.items():
attrs["id"] = node
for node, attrs in mst_graph.nodes.items():
attrs["id"] = node
outputs[self.gt] = gp.Graph.from_nx_graph(
gt_graph,
gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3),
directed=False))
outputs[self.mst] = gp.Graph.from_nx_graph(
mst_graph,
gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3),
directed=False),
)
return outputs
def build_source(self):
data = daisy.open_ds(filename, key)
if self.time_window is None:
source_roi = gp.Roi(data.roi.get_offset(), data.roi.get_shape())
else:
offs = list(data.roi.get_offset())
offs[1] += self.time_window[0]
sh = list(data.roi.get_shape())
offs[1] = self.time_window[1] - self.time_window[0]
source_roi = gp.Roi(tuple(offs), tuple(sh))
voxel_size = gp.Coordinate(data.voxel_size)
return gp.ZarrSource(filename,
{
self.raw_0: key,
self.raw_1: key
},
array_specs={
self.raw_0: gp.ArraySpec(
roi=source_roi,
voxel_size=voxel_size,
interpolatable=True),
self.raw_1: gp.ArraySpec(
roi=source_roi,
voxel_size=voxel_size,
interpolatable=True)
})
def test_shift_points5(self):
data = {
0: gp.Point([3, 0]),
1: gp.Point([3, 2]),
2: gp.Point([3, 4]),
3: gp.Point([3, 6]),
4: gp.Point([3, 8])
}
spec = gp.PointsSpec(gp.Roi(offset=(0, 0), shape=(15, 10)))
points = gp.Points(data, spec)
request_roi = gp.Roi(offset=(3, 0), shape=(9, 10))
shift_array = np.array([[3, 0], [-3, 0], [0, 0], [-3, 0], [3, 0]],
dtype=int)
lcm_voxel_size = gp.Coordinate((3, 2))
shifted_data = {
0: gp.Point([6, 0]),
2: gp.Point([3, 4]),
4: gp.Point([6, 8])
}
result = gp.ShiftAugment.shift_points(points,
request_roi,
shift_array,
shift_axis=1,
lcm_voxel_size=lcm_voxel_size)
# print("test 4", result.data, shifted_data)
self.assertTrue(self.points_equal(result.data, shifted_data))
self.assertTrue(result.spec == gp.PointsSpec(request_roi))
def test_compute_upstream_roi2(self):
request_roi = gp.Roi(offset=(0, 0), shape=(5, 10))
sub_shift_array = np.array([[2, 0], [-1, 0], [5, 0], [-2, 0], [0, 0]],
dtype=int)
upstream_roi = gp.Roi(offset=(-5, 0), shape=(12, 10))
result = gp.ShiftAugment.compute_upstream_roi(request_roi,
sub_shift_array)
self.assertTrue(upstream_roi == result)
def setup(self):
self.provides(
gp.ArrayKeys.M_PRED,
gp.ArraySpec(roi=gp.Roi((0, 0, 0), (200, 200, 200)),
voxel_size=self.voxel_size,
interpolatable=False))
self.provides(
gp.ArrayKeys.D_PRED,
gp.ArraySpec(roi=gp.Roi((0, 0, 0), (200, 200, 200)),
voxel_size=self.voxel_size,
interpolatable=False))
def __init__(self, filename,
key,
density=None,
channels=0,
shape=(16, 256, 256),
time_window=None,
add_sparse_mosaic_channel=True,
random_rot=False):
self.filename = filename
self.key = key
self.shape = shape
self.density = density
self.raw = gp.ArrayKey('RAW_0')
self.add_sparse_mosaic_channel = add_sparse_mosaic_channel
self.random_rot = random_rot
self.channels = channels
data = daisy.open_ds(filename, key)
if time_window is None:
source_roi = gp.Roi(data.roi.get_offset(), data.roi.get_shape())
else:
offs = list(data.roi.get_offset())
offs[1] += time_window[0]
sh = list(data.roi.get_shape())
offs[1] = time_window[1] - time_window[0]
source_roi = gp.Roi(tuple(offs), tuple(sh))
voxel_size = gp.Coordinate(data.voxel_size)
self.pipeline = gp.ZarrSource(
filename,
{
self.raw: key
},
array_specs={
self.raw: gp.ArraySpec(
roi=source_roi,
voxel_size=voxel_size,
interpolatable=True)
}) + gp.RandomLocation() + IntensityDiffFilter(self.raw, 0, min_distance=0.1, channels=Slice(None))
# add augmentations
self.pipeline = self.pipeline + gp.ElasticAugment([40, 40],
[2, 2],
[0, math.pi / 2.0],
prob_slip=-1,
spatial_dims=2)
self.pipeline.setup()
np.random.seed(os.getpid() + int(time.time()))
def test_get_sub_shift_array2(self):
total_roi = gp.Roi(offset=(0, 0), shape=(6, 6))
item_roi = gp.Roi(offset=(1, 2), shape=(3, 3))
shift_array = np.arange(12).reshape(6, 2).astype(int)
shift_axis = 0
lcm_voxel_size = gp.Coordinate((1, 1))
sub_shift_array = np.array([[2, 3], [4, 5], [6, 7]], dtype=int)
result = gp.ShiftAugment.get_sub_shift_array(total_roi, item_roi,
shift_array, shift_axis,
lcm_voxel_size)
self.assertTrue(np.array_equal(result, sub_shift_array))
def setup(self):
self.provides(
self.raw,
gp.ArraySpec(roi=gp.Roi((0, 0), (1000, 1000)),
dtype=np.uint8,
interpolatable=True,
voxel_size=(1, 1)))
self.provides(
self.gt,
gp.ArraySpec(roi=gp.Roi((0, 0), (1000, 1000)),
dtype=np.uint64,
interpolatable=False,
voxel_size=(1, 1)))
def setup(self):
if str(self.snapshot_file).endswith(".h5") or str(self.snapshot_file).endswith(
".hdf"
):
data = h5py.File(self.snapshot_file, "r")
elif str(self.snapshot_file).endswith(".zarr"):
data = zarr.open(self.snapshot_file, "r")
for key, path in self.datasets.items():
if isinstance(key, gp.ArrayKey):
try:
x = data[path]
except KeyError:
raise KeyError(f"Could not find {path}")
spec = self.spec_from_dataset(x)
self.provides(key, spec)
elif isinstance(key, gp.GraphKey):
try:
locations = data[f"{path}-locations"]
except KeyError:
raise KeyError(f"Could not find {path}-locations")
spec = gp.GraphSpec(
gp.Roi((None,) * len(locations[0]), (None,) * len(locations[0])),
directed=self.directed.get(key),
)
self.provides(key, spec)
def setup(self):
# provide points in an infinite ROI
self.graph_spec = gp.GraphSpec(
roi=gp.Roi(offset=(0, ) * self.dims, shape=(None, ) * self.dims))
self.provides(self.graph_key, self.graph_spec)
def setup(self):
self.ndims = self.data.shape[1]
if self.points_spec is not None:
self.provides(self.points, self.points_spec)
elif isinstance(self.points, gp.ArrayKey):
self.provides(self.points, gp.ArraySpec(voxel_size=((1, ))))
elif isinstance(self.points, gp.GraphKey):
print(self.ndims)
min_bb = gp.Coordinate(
np.floor(np.amin(self.data[:, :self.ndims], 0)))
max_bb = gp.Coordinate(
np.ceil(np.amax(self.data[:, :self.ndims], 0)) + 1)
roi = gp.Roi(min_bb, max_bb - min_bb)
logger.debug(f"Bounding Box: {roi}")
self.provides(self.points, gp.GraphSpec(roi=roi))
if self.labels is not None:
assert isinstance(self.labels, gp.ArrayKey), \
f"Label key must be an ArrayKey, \
was given {type(self.labels)}"
if self.labels_spec is not None:
self.provides(self.labels, self.labels_spec)
else:
self.provides(self.labels, gp.ArraySpec(voxel_size=((1, ))))
def __init__(
self,
gt,
mst,
output,
roi=None,
details=None,
location_attr="location",
small_component_threshold=10000,
comatch_threshold=4000,
edit_distance_node_spacing=5000,
edge_threshold_attr="distance",
num_thresholds=5,
threshold_range=(0, 1),
connectivity=None,
output_graph=None,
):
self.mst = mst
self.gt = gt
self.output = output
self.roi = roi
if self.roi is None:
self.roi = gp.Roi((None, ) * 3, (None, ) * 3)
self.location_attr = location_attr
self.small_component_threshold = small_component_threshold
self.comatch_threshold = comatch_threshold
self.edit_distance_node_spacing = edit_distance_node_spacing
self.edge_threshold_attr = edge_threshold_attr
self.num_thresholds = num_thresholds
self.threshold_range = threshold_range
self.details = details
self.connectivity = connectivity
self.output_graph = output_graph
def test_shift_points2(self):
data = {1: gp.Point([0, 1])}
spec = gp.PointsSpec(gp.Roi(offset=(0, 0), shape=(5, 5)))
points = gp.Points(data, spec)
request_roi = gp.Roi(offset=(0, 1), shape=(5, 3))
shift_array = np.array([[0, 0], [0, -1], [0, 0], [0, 0], [0, 1]],
dtype=int)
lcm_voxel_size = gp.Coordinate((1, 1))
result = gp.ShiftAugment.shift_points(points,
request_roi,
shift_array,
shift_axis=0,
lcm_voxel_size=lcm_voxel_size)
# print("test 2", result.data, data)
self.assertTrue(self.points_equal(result.data, data))
self.assertTrue(result.spec == gp.PointsSpec(request_roi))
def setup(self):
self.provides(
self.array_key,
gp.ArraySpec(roi=gp.Roi(offset=gp.Coordinate(
(-10000, -10000, -10000)),
shape=gp.Coordinate(
(20000, 20000, 20000))),
voxel_size=(1, 1, 1)))
def __slice_to_roi(self, slices):
offset = tuple(s.start for s in slices)
shape = tuple(s.stop - s.start for s in slices)
roi = gp.Roi(offset, shape)
roi = roi.snap_to_grid((self.lsd_extractor.downsample, ) * roi.dims())
return roi
def setup(self):
# remove selected dim from provided specs
for key, upstream_spec in self.get_upstream_provider().spec.items():
spec = upstream_spec.copy()
spec.roi = gp.Roi(self.__remove_dim(spec.roi.get_begin()),
self.__remove_dim(spec.roi.get_shape()))
spec.voxel_size = self.__remove_dim(spec.voxel_size)
self.spec[key] = spec
def test_context(self):
d_pred = gp.ArrayKeys.D_PRED
m_pred = gp.ArrayKeys.M_PRED
presyn = gp.PointsKeys.PRESYN
postsyn = gp.PointsKeys.POSTSYN
outdir = tempfile.mkdtemp()
voxel_size = gp.Coordinate((10, 10, 10))
size = ((200, 200, 200))
# Check whether the score of the entire cube is measured, although
# cube of borderpoint partially outside request ROI.
context = 40
shape = gp.Coordinate(size) / voxel_size
m_predar = np.zeros(shape, dtype=np.float32)
outsidepoint = gp.Coordinate((13, 13, 13))
borderpoint = (4, 4, 4)
m_predar[3:5, 3:5, 3:5] = 1
m_predar[outsidepoint] = 1
d_predar = np.ones((3, shape[0], shape[1], shape[2])) * 0
pipeline = (TestSource(m_predar, d_predar, voxel_size=voxel_size) +
ExtractSynapses(m_pred,
d_pred,
presyn,
postsyn,
out_dir=outdir,
settings=parameters,
context=context) +
gp.PrintProfilingStats())
request = gp.BatchRequest()
roi = gp.Roi((40, 40, 40), (80, 80, 80))
request[presyn] = gp.PointsSpec(roi=roi)
request[postsyn] = gp.PointsSpec(roi=roi)
with gp.build(pipeline):
batch = pipeline.request_batch(request)
synapsefile = os.path.join(outdir, "40", "40", "40.npz")
with np.load(synapsefile) as data:
data = dict(data)
self.assertTrue(len(data['ids']) == 1)
self.assertEqual(data['scores'][0], 2.0**3) # Size of the cube.
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][0][ii],
borderpoint[ii] * voxel_size[ii])
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][1][ii],
borderpoint[ii] * voxel_size[ii] + 0)
shutil.rmtree(outdir)
def prepare(self, request):
if self.array not in request:
return
request[self.array].roi = gp.Roi(
self.__remove_dim(request[self.array].roi.get_begin()),
self.__remove_dim(request[self.array].roi.get_shape()))
if request[self.array].voxel_size is not None:
request[self.array].voxel_size = self.__remove_dim(
request[self.array].voxel_size)
def setup(self):
upstream_spec = self.get_upstream_provider().spec[self.array]
spec = upstream_spec.copy()
if spec.roi is not None:
spec.roi = gp.Roi(self.__insert_dim(spec.roi.get_begin(), 0),
self.__insert_dim(spec.roi.get_shape(), 1))
if spec.voxel_size is not None:
spec.voxel_size = self.__insert_dim(spec.voxel_size, 1)
self.spec[self.array] = spec
self.updates(self.array, self.spec[self.array])
def prepare(self, request):
upstream_spec = self.get_upstream_provider().spec
# add a new dim
for key, spec in request.items():
upstream_voxel_size = upstream_spec[key].voxel_size
v = upstream_voxel_size[self.dim]
spec.roi = gp.Roi(self.__insert_dim(spec.roi.get_begin(), 0),
self.__insert_dim(spec.roi.get_shape(), v))
if spec.voxel_size is not None:
spec.voxel_size = self.__insert_dim(spec.voxel_size, v)
def process(self, batch, request):
if self.array not in request:
return
array = batch[self.array]
array.spec.roi = gp.Roi(
self.__insert_dim(array.spec.roi.get_begin(), 0),
self.__insert_dim(array.spec.roi.get_shape(), 1))
array.spec.voxel_size = self.__insert_dim(array.spec.voxel_size, 1)
array.data = array.data[:, :, np.newaxis, :, :]
def test_output_basics(self):
d_pred = gp.ArrayKeys.D_PRED
m_pred = gp.ArrayKeys.M_PRED
presyn = gp.PointsKeys.PRESYN
postsyn = gp.PointsKeys.POSTSYN
voxel_size = gp.Coordinate((10, 10, 10))
size = ((200, 200, 200))
context = 40
shape = gp.Coordinate(size) / voxel_size
m_predar = np.zeros(shape, dtype=np.float32)
insidepoint = gp.Coordinate((10, 10, 10))
outsidepoint = gp.Coordinate((15, 15, 15))
m_predar[insidepoint] = 1
m_predar[outsidepoint] = 1
d_predar = np.ones((3, shape[0], shape[1], shape[2])) * 10
outdir = tempfile.mkdtemp()
pipeline = (TestSource(m_predar, d_predar, voxel_size=voxel_size) +
ExtractSynapses(m_pred,
d_pred,
presyn,
postsyn,
out_dir=outdir,
settings=parameters,
context=context))
request = gp.BatchRequest()
roi = gp.Roi((40, 40, 40), (80, 80, 80))
request[presyn] = gp.PointsSpec(roi=roi)
request[postsyn] = gp.PointsSpec(roi=roi)
with gp.build(pipeline):
batch = pipeline.request_batch(request)
print(outdir, "outdir")
synapsefile = os.path.join(outdir, "40", "40", "40.npz")
with np.load(synapsefile) as data:
data = dict(data)
self.assertTrue(len(data['ids']) == 1)
self.assertEqual(data['scores'][0], 1.0) # Size of the cube.
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][1][ii],
insidepoint[ii] * voxel_size[ii])
for ii in range(len(voxel_size)):
self.assertEqual(data['positions'][0][0][ii],
insidepoint[ii] * voxel_size[ii] + 10)
shutil.rmtree(outdir)
def prepare(self, request):
if self.key not in request:
return
request[self.key].roi = gp.Roi(
self.__insert_dim(request[self.key].roi.get_begin(), 0),
self.__insert_dim(request[self.key].roi.get_shape(), 1))
if isinstance(self.key, gp.ArrayKey):
if request[self.key].voxel_size is not None:
request[self.key].voxel_size = self.__insert_dim(
request[self.key].voxel_size, 1)
def process(self, batch, request):
if self.key not in batch:
return
if isinstance(self.key, gp.ArrayKey):
data = batch[self.key].data
shape = data.shape
roi = batch[self.key].spec.roi
assert shape[-roi.dims()] == 1, "Channel to delete must be size 1," \
"but given shape " + str(shape)
shape = self.__remove_dim(shape, len(shape) - roi.dims())
batch[self.key].data = data.reshape(shape)
batch[self.key].spec.roi = gp.Roi(
self.__remove_dim(roi.get_begin()),
self.__remove_dim(roi.get_shape()))
batch[self.key].spec.voxel_size = \
self.__remove_dim(batch[self.key].spec.voxel_size)
if isinstance(self.key, gp.GraphKey):
roi = batch[self.key].spec.roi
batch[self.key].spec.roi = gp.Roi(
self.__remove_dim(roi.get_begin()),
self.__remove_dim(roi.get_shape()))
graph = gp.Graph([], [], spec=batch[self.key].spec)
for node in batch[self.key].nodes:
print(node)
new_node = gp.Node(node.id,
node.location[1:],
temporary=node.temporary,
attrs=node.attrs)
graph.add_node(new_node)
print(node)
print(graph.spec.roi)
print(list(graph.nodes))
batch[self.key] = graph
def setup(self):
self.enable_autoskip()
all_rois = []
for block, block_specs in self.specs.items():
ground_truth = block_specs["ground_truth"]
mst_pred = block_specs["mst_pred"]
for key, spec in [ground_truth, mst_pred]:
current_spec = self.spec[key].copy()
current_spec.roi = spec.roi
self.updates(key, current_spec)
all_rois.append(current_spec.roi)
self.total_roi = all_rois[0]
for roi in all_rois[1:]:
self.total_roi = self.total_roi.union(roi)
self.provides(
self.mst,
gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3),
directed=False))
self.provides(
self.gt,
gp.GraphSpec(roi=gp.Roi((None, ) * 3, (None, ) * 3),
directed=False))
def setup(self):
upstream_spec = self.get_upstream_provider().spec[self.key]
spec = upstream_spec.copy()
if spec.roi is not None:
spec.roi = gp.Roi(self.__remove_dim(spec.roi.get_begin()),
self.__remove_dim(spec.roi.get_shape()))
if isinstance(self.key, gp.ArrayKey):
if spec.voxel_size is not None:
spec.voxel_size = self.__remove_dim(spec.voxel_size)
self.spec[self.key] = spec
self.updates(self.key, self.spec[self.key])
def __init__(self, voxel_size):
self.voxel_size = gp.Coordinate(voxel_size)
self.roi = gp.Roi((0, 0, 0), (10, 10, 10)) * self.voxel_size
self.raw = gp.ArrayKey("RAW")
self.labels = gp.ArrayKey("LABELS")
self.array_spec_raw = gp.ArraySpec(roi=self.roi,
voxel_size=self.voxel_size,
dtype='uint8',
interpolatable=True)
self.array_spec_labels = gp.ArraySpec(roi=self.roi,
voxel_size=self.voxel_size,
dtype='uint64',
interpolatable=False)
def test_pipeline3(self):
array_key = gp.ArrayKey("TEST_ARRAY")
points_key = gp.PointsKey("TEST_POINTS")
voxel_size = gp.Coordinate((1, 1))
spec = gp.ArraySpec(voxel_size=voxel_size, interpolatable=True)
hdf5_source = gp.Hdf5Source(self.fake_data_file,
{array_key: 'testdata'},
array_specs={array_key: spec})
csv_source = gp.CsvPointsSource(
self.fake_points_file, points_key,
gp.PointsSpec(
roi=gp.Roi(shape=gp.Coordinate((100, 100)), offset=(0, 0))))
request = gp.BatchRequest()
shape = gp.Coordinate((60, 60))
request.add(array_key, shape, voxel_size=gp.Coordinate((1, 1)))
request.add(points_key, shape)
shift_node = gp.ShiftAugment(prob_slip=0.2,
prob_shift=0.2,
sigma=5,
shift_axis=0)
pipeline = ((hdf5_source, csv_source) + gp.MergeProvider() +
gp.RandomLocation(ensure_nonempty=points_key) + shift_node)
with gp.build(pipeline) as b:
request = b.request_batch(request)
# print(request[points_key])
target_vals = [
self.fake_data[point[0]][point[1]] for point in self.fake_points
]
result_data = request[array_key].data
result_points = request[points_key].data
result_vals = [
result_data[int(point.location[0])][int(point.location[1])]
for point in result_points.values()
]
for result_val in result_vals:
self.assertTrue(
result_val in target_vals,
msg=
"result value {} at points {} not in target values {} at points {}"
.format(result_val, list(result_points.values()), target_vals,
self.fake_points))
def process(self, batch, request):
for key, array in batch.arrays.items():
# remove first dim
array.spec.roi = gp.Roi(
self.__remove_dim(array.spec.roi.get_begin()),
self.__remove_dim(array.spec.roi.get_shape()))
array.spec.voxel_size = self.__remove_dim(array.spec.voxel_size)
assert array.data.shape[self.dim] == 1, \
"Squash for dim %d requires that the array %s has size 1 in " \
"that dim, but array shape is %s" % (
self.dim,
key,
array.data.shape)
array.data = array.data.reshape(
self.__remove_dim(array.data.shape))
