def run_inference(input_file, input_key, output_file, output_key, checkpoint,
n_gpus):
out_blocks = (65, 675, 675)
chunks = (1, ) + out_blocks
with h5py.File(input_file) as f:
shape = f[input_key].shape
aff_shape = (3, ) + shape
f = z5py.N5File(output_file)
f.require_dataset(output_key,
shape=aff_shape,
chunks=chunks,
compression='gzip',
dtype='uint8')
get_offset_lists(shape,
list(range(n_gpus)),
'./offsets',
output_shape=out_blocks)
with futures.ProcessPoolExecutor(n_gpus) as pp:
tasks = [
pp.submit(single_inference, input_file, input_key, output_file,
output_key, checkpoint, gpu_id)
for gpu_id in range(n_gpus)
]
[t.result() for t in tasks]
evaluate_bench()
def solve_subproblems(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
# input configs
problem_path = config['problem_path']
scale = config['scale']
block_shape = config['block_shape']
block_list = config['block_list']
n_threads = config['threads_per_job']
agglomerator_key = config['agglomerator']
time_limit = config.get('time_limit_solver', None)
fu.log("reading problem from %s" % problem_path)
problem = z5py.N5File(problem_path)
shape = problem.attrs['shape']
# load the costs
costs_key = 's%i/costs' % scale
fu.log("reading costs from path in problem: %s" % costs_key)
ds = problem[costs_key]
ds.n_threads = n_threads
costs = ds[:]
# load the graph
graph_key = 's%i/graph' % scale
fu.log("reading graph from path in problem: %s" % graph_key)
graph = ndist.Graph(os.path.join(problem_path, graph_key),
numberOfThreads=n_threads)
uv_ids = graph.uvIds()
# check if the problem has an ignore-label
ignore_label = problem[graph_key].attrs['ignoreLabel']
fu.log("ignore label is %s" % ('true' if ignore_label else 'false'))
fu.log("using agglomerator %s" % agglomerator_key)
agglomerator = su.key_to_agglomerator(agglomerator_key)
# the output group
out = problem['s%i/sub_results' % scale]
# TODO this should be a n5 varlen dataset as well and
# then this is just another dataset in problem path
block_prefix = os.path.join(problem_path, 's%i' % scale, 'sub_graphs',
'block_')
blocking = nt.blocking([0, 0, 0], shape, list(block_shape))
with futures.ThreadPoolExecutor(n_threads) as tp:
tasks = [
tp.submit(_solve_block_problem, block_id, graph, uv_ids,
block_prefix, costs, agglomerator, ignore_label,
blocking, out, time_limit) for block_id in block_list
]
[t.result() for t in tasks]
fu.log_job_success(job_id)
def sub_solutions(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
# input configs
problem_path = config['problem_path']
scale = config['scale']
block_shape = config['block_shape']
block_list = config['block_list']
n_threads = config['threads_per_job']
output_path = config['output_path']
output_key = config['output_key']
ws_path = config['ws_path']
ws_key = config['ws_key']
sub_result_identifer = config.get('sub_result_identifier', 'sub_results')
sub_graph_identifer = config.get('sub_graph_identifier', 'sub_graphs')
fu.log("reading problem from %s" % problem_path)
problem = z5py.N5File(problem_path)
shape = problem.attrs['shape']
blocking = nt.blocking([0, 0, 0], list(shape), list(block_shape))
# we need to project the ws labels back to the original labeling
# for this, we first need to load the initial node labeling
if scale > 1:
node_label_key = 's%i/node_labeling' % scale
fu.log("scale %i > 1; reading node labeling from %s" % (scale, node_label_key))
ds_node_labeling = problem[node_label_key]
ds_node_labeling.n_threads = n_threasd
initial_node_labeling = ds_node_labeling[:]
else:
initial_node_labeling = None
# read the sub results
ds_results = problem['s%i/%s/node_result' % (scale, sub_result_identifier)]
# TODO should be varlen dataset
fu.log("reading subresults")
block_node_prefix = os.path.join(problem_path, 's%i' % scale, sub_graph_identifier, 'block_')
block_list, block_results = _read_subresults(ds_results, block_node_prefix, blocking,
block_list, n_threads, initial_node_labeling)
fu.log("writing subresults")
# write the resulting segmentation
with vu.file_reader(output_path) as f_out, vu.file_reader(ws_path, 'r') as f_in:
ds_in = f_in[ws_key]
ds_out = f_out[output_key]
with futures.ThreadPoolExecutor(n_threads) as tp:
tasks = [tp.submit(_write_block_res, ds_in, ds_out,
block_id, blocking, block_res)
for block_id, block_res in zip(block_list, block_results)]
[t.result() for t in tasks]
fu.log_job_success(job_id)
def solve_subproblems(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
# input configs
problem_path = config['problem_path']
scale = config['scale']
block_shape = config['block_shape']
block_list = config['block_list']
n_threads = config['threads_per_job']
agglomerator_key = config['agglomerator']
time_limit = config.get('time_limit_solver', None)
fu.log("reading problem from %s" % problem_path)
problem = z5py.N5File(problem_path)
shape = problem['s0/graph'].attrs['shape']
# load the costs
costs_key = 's%i/costs' % scale
fu.log("reading costs from path in problem: %s" % costs_key)
ds = problem[costs_key]
ds.n_threads = n_threads
costs = ds[:]
# load the graph
graph_key = 's%i/graph' % scale
fu.log("reading graph from path in problem: %s" % graph_key)
graph = ndist.Graph(problem_path, graph_key, numberOfThreads=n_threads)
uv_ids = graph.uvIds()
# check if the problem has an ignore-label
ignore_label = problem[graph_key].attrs['ignore_label']
fu.log("ignore label is %s" % ('true' if ignore_label else 'false'))
fu.log("using solver %s" % agglomerator_key)
solver = get_multicut_solver(agglomerator_key)
# the output group
out = problem['s%i/sub_results' % scale]
node_ds_key = 's%i/sub_graphs/nodes' % scale
ds_nodes = problem[node_ds_key]
blocking = nt.blocking([0, 0, 0], shape, list(block_shape))
with futures.ThreadPoolExecutor(n_threads) as tp:
tasks = [
tp.submit(_solve_block_problem, block_id, graph, uv_ids, ds_nodes,
costs, solver, ignore_label, blocking, out, time_limit)
for block_id in block_list
]
[t.result() for t in tasks]
fu.log_job_success(job_id)
def test_2d_vigra_along_z(self):
"""Test if 2d files generated through vigra are recognized correctly"""
# Prepare some data set for this case
data = numpy.random.randint(0, 255,
(20, 100, 200, 3)).astype(numpy.uint8)
axistags = vigra.defaultAxistags("yxc")
expected_axistags = vigra.defaultAxistags("zyxc")
h5_op = OpStreamingH5N5SequenceReaderM(graph=self.graph)
n5_op = OpStreamingH5N5SequenceReaderM(graph=self.graph)
tempdir = tempfile.TemporaryDirectory()
try:
for sliceIndex, zSlice in enumerate(data):
testDataH5FileName = f"{tempdir.name}/test-{sliceIndex:02d}.h5"
testDataN5FileName = f"{tempdir.name}/test-{sliceIndex:02d}.n5"
# Write the dataset to an hdf5 and a n5 file
# (Note: Don't use vigra to do this, which may reorder the axes)
h5File = h5py.File(testDataH5FileName)
n5File = z5py.N5File(testDataN5FileName)
try:
h5File.create_group("volume")
n5File.create_group("volume")
h5File["volume"].create_dataset("subvolume", data=zSlice)
n5File["volume"].create_dataset("subvolume", data=zSlice)
# Write the axistags attribute
current_path = "volume/subvolume"
h5File[current_path].attrs["axistags"] = axistags.toJSON()
n5File[current_path].attrs["axistags"] = axistags.toJSON()
finally:
h5File.close()
n5File.close()
# Read the data with an operator
hdf5GlobString = f"{tempdir.name}/test-*.h5/volume/subvolume"
n5GlobString = f"{tempdir.name}/test-*.n5/volume/subvolume"
h5_op.SequenceAxis.setValue("z")
n5_op.SequenceAxis.setValue("z")
h5_op.GlobString.setValue(hdf5GlobString)
n5_op.GlobString.setValue(n5GlobString)
assert h5_op.OutputImage.ready()
assert n5_op.OutputImage.ready()
assert h5_op.OutputImage.meta.axistags == expected_axistags
assert n5_op.OutputImage.meta.axistags == expected_axistags
numpy.testing.assert_array_equal(
h5_op.OutputImage.value[5:10, 50:100, 100:150],
data[5:10, 50:100, 100:150])
numpy.testing.assert_array_equal(
n5_op.OutputImage.value[5:10, 50:100, 100:150],
data[5:10, 50:100, 100:150])
finally:
h5_op.cleanUp()
n5_op.cleanUp()
def test_3d_vigra_along_t(self):
"""Test if 3d volumes generated through vigra are recognized correctly"""
# Prepare some data set for this case
data = numpy.random.randint(0, 255,
(10, 15, 50, 100, 3)).astype(numpy.uint8)
axistags = vigra.defaultAxistags("zyxc")
expected_axistags = vigra.defaultAxistags("tzyxc")
h5_op = OpStreamingH5N5SequenceReaderS(graph=self.graph)
n5_op = OpStreamingH5N5SequenceReaderS(graph=self.graph)
try:
testDataH5FileName = f"{self.tempdir_normalized_name}/test.h5"
testDataN5FileName = f"{self.tempdir_normalized_name}/test.n5"
# Write the dataset to an hdf5 file
# (Note: Don't use vigra to do this, which may reorder the axes)
h5File = h5py.File(testDataH5FileName)
n5File = z5py.N5File(testDataN5FileName)
try:
h5File.create_group("volumes")
n5File.create_group("volumes")
internalPathString = "subvolume-{sliceIndex:02d}"
for sliceIndex, tSlice in enumerate(data):
subpath = internalPathString.format(sliceIndex=sliceIndex)
h5File["volumes"].create_dataset(subpath, data=tSlice)
n5File["volumes"].create_dataset(subpath, data=tSlice)
# Write the axistags attribute
current_path = "volumes/{}".format(subpath)
h5File[current_path].attrs["axistags"] = axistags.toJSON()
n5File[current_path].attrs["axistags"] = axistags.toJSON()
finally:
h5File.close()
n5File.close()
# Read the data with an operator
hdf5GlobString = f"{testDataH5FileName}/volumes/subvolume-*"
n5GlobString = f"{testDataN5FileName}/volumes/subvolume-*"
h5_op.SequenceAxis.setValue("t")
n5_op.SequenceAxis.setValue("t")
h5_op.GlobString.setValue(hdf5GlobString)
n5_op.GlobString.setValue(n5GlobString)
assert h5_op.OutputImage.ready()
assert n5_op.OutputImage.ready()
assert h5_op.OutputImage.meta.axistags == expected_axistags
assert n5_op.OutputImage.meta.axistags == expected_axistags
numpy.testing.assert_array_equal(h5_op.OutputImage.value, data)
numpy.testing.assert_array_equal(n5_op.OutputImage.value, data)
finally:
h5_op.cleanUp()
n5_op.cleanUp()
def get_h5_n5_file(filepath, mode="a"):
"""
returns, depending on the file-extension of filepath, either a hdf5 or a N5 file defined by filepath
If the file is created when it does not exist depends on mode and on the function z5py.N5File/h5py.File.
default mode = 'a': Read/write if exists, create otherwise
"""
name, ext = os.path.splitext(filepath)
if ext in OpStreamingH5N5Reader.N5EXTS:
return z5py.N5File(filepath, mode)
elif ext in OpStreamingH5N5Reader.H5EXTS:
return h5py.File(filepath, mode)
def test_Writer(self):
# Create the h5 file
hdf5File = h5py.File(self.testDataH5FileName)
n5File = z5py.N5File(self.testDataN5FileName)
opPiper = OpArrayPiper(graph=self.graph)
opPiper.Input.setValue(self.testData)
h5_opWriter = OpH5N5WriterBigDataset(graph=self.graph)
n5_opWriter = OpH5N5WriterBigDataset(graph=self.graph)
h5_opWriter.h5N5File.setValue(hdf5File)
n5_opWriter.h5N5File.setValue(n5File)
h5_opWriter.h5N5Path.setValue(self.datasetInternalPath)
n5_opWriter.h5N5Path.setValue(self.datasetInternalPath)
h5_opWriter.Image.connect(opPiper.Output)
n5_opWriter.Image.connect(opPiper.Output)
# Force the operator to execute by asking for the output (a bool)
h5_success = h5_opWriter.WriteImage.value
n5_success = n5_opWriter.WriteImage.value
assert h5_success
assert n5_success
hdf5File.close()
n5File.close()
# Check the file.
hdf5File = h5py.File(self.testDataH5FileName, "r")
n5File = z5py.N5File(self.testDataN5FileName, "r")
h5_dataset = hdf5File[self.datasetInternalPath]
n5_dataset = n5File[self.datasetInternalPath]
assert h5_dataset.shape == self.dataShape
assert n5_dataset.shape == self.dataShape
assert (numpy.all(
h5_dataset[...] == self.testData.view(numpy.ndarray)[...])).all()
assert (numpy.all(
n5_dataset[...] == self.testData.view(numpy.ndarray)[...])).all()
hdf5File.close()
n5File.close()
def getPossibleInternalPathsFor(cls, file_path: Path, min_ndim=2, max_ndim=5) -> List[str]:
datasetNames = []
def accumulateInternalPaths(name, val):
if isinstance(val, (h5py.Dataset, z5py.dataset.Dataset)) and min_ndim <= len(val.shape) <= max_ndim:
datasetNames.append("/" + name)
if cls.pathIsHdf5(file_path):
with h5py.File(file_path, "r") as f:
f.visititems(accumulateInternalPaths)
elif cls.pathIsN5(file_path):
with z5py.N5File(file_path, mode="r+") as f:
f.visititems(accumulateInternalPaths)
return datasetNames
def test_Writer(self):
# Create the h5 file
hdf5File = h5py.File(self.testDataH5FileName)
n5File = z5py.N5File(self.testDataN5FileName)
opPiper = OpArrayPiper(graph=self.graph)
opPiper.Input.setValue(self.testData)
# Force extra metadata onto the output
opPiper.Output.meta.ideal_blockshape = (1, 1, 0, 0, 1)
# Pretend the RAM usage will be really high to force lots of tiny blocks
opPiper.Output.meta.ram_usage_per_requested_pixel = 1000000.0
h5_opWriter = OpH5N5WriterBigDataset(graph=self.graph)
n5_opWriter = OpH5N5WriterBigDataset(graph=self.graph)
# This checks that you can give a preexisting group as the file
h5_g = hdf5File.create_group("volume")
n5_g = n5File.create_group("volume")
h5_opWriter.h5N5File.setValue(h5_g)
n5_opWriter.h5N5File.setValue(n5_g)
h5_opWriter.h5N5Path.setValue("data")
n5_opWriter.h5N5Path.setValue("data")
h5_opWriter.Image.connect(opPiper.Output)
n5_opWriter.Image.connect(opPiper.Output)
# Force the operator to execute by asking for the output (a bool)
h5_success = h5_opWriter.WriteImage.value
n5_success = n5_opWriter.WriteImage.value
assert h5_success
assert n5_success
hdf5File.close()
n5File.close()
# Check the file.
hdf5File = h5py.File(self.testDataH5FileName, "r")
n5File = h5py.File(self.testDataH5FileName, "r")
h5_dataset = hdf5File[self.datasetInternalPath]
n5_dataset = n5File[self.datasetInternalPath]
assert h5_dataset.shape == self.dataShape
assert n5_dataset.shape == self.dataShape
assert (numpy.all(
h5_dataset[...] == self.testData.view(numpy.ndarray)[...])).all()
assert (numpy.all(
n5_dataset[...] == self.testData.view(numpy.ndarray)[...])).all()
hdf5File.close()
n5File.close()
def getPossibleN5InternalPaths(cls, absPath, min_ndim=2, max_ndim=5):
"""
Returns the name of all datasets in the file with at least 2 axes.
"""
datasetNames = []
# Open the file as a read-only so we can get a list of the internal paths
with z5py.N5File(absPath, mode='r+') as f:
def accumulate_names(path, val):
if isinstance(val, z5py.dataset.Dataset) and min_ndim <= len(
val.shape) <= max_ndim:
name = path.replace(absPath,
'') # Need only the internal path here
datasetNames.append(name)
f.visititems(accumulate_names)
return datasetNames
def __init__(self, n5_path, ds_path, offset_px=N5_OFFSET):
"""
Parameters
----------
n5_path
ds_path
offset_px
N5 dataset offset compared to the canonical JPEG image stack. Default (0, -1, 0)
"""
self.offset_px = offset_px or (0, 0, 0)
self.n5_path = n5_path
self.ds_path = ds_path
self.n5_file = z5py.N5File(self.n5_path, mode='r')
self.n5_ds = self.n5_file[self.ds_path]
def globInternalPaths(cls, file_path: str, glob_str: str, cwd: str = None) -> List[str]:
glob_str = glob_str.lstrip("/")
internal_paths = set()
for path in cls.expand_path(file_path, cwd=cwd):
f = None
try:
if cls.pathIsNpz(path):
internal_paths |= set(globNpz(path, glob_str))
continue
elif cls.pathIsHdf5(path):
f = h5py.File(path, "r")
elif cls.pathIsN5(path):
f = z5py.N5File(path) # FIXME
else:
raise Exception(f"{path} is not an 'n5' or 'h5' file")
internal_paths |= set(globH5N5(f, glob_str))
finally:
if f is not None:
f.close()
return sorted(internal_paths)
def test_expandGlobStrings(self):
expected_datasets = ["g1/g2/data2", "g1/g2/data3"]
h5_file_name = f"{self.tempdir_normalized_name}/test.h5"
n5_file_name = f"{self.tempdir_normalized_name}/test.n5"
try:
h5_file = h5py.File(h5_file_name, mode="w")
n5_file = z5py.N5File(n5_file_name, mode="w")
h5_g1 = h5_file.create_group("g1")
n5_g1 = n5_file.create_group("g1")
h5_g2 = h5_g1.create_group("g2")
n5_g2 = n5_g1.create_group("g2")
h5_g3 = h5_file.create_group("g3")
n5_g3 = n5_file.create_group("g3")
h5_g1.create_dataset("data1", data=numpy.ones((10, 10)))
n5_g1.create_dataset("data1", data=numpy.ones((10, 10)))
h5_g2.create_dataset("data2", data=numpy.ones((10, 10)))
n5_g2.create_dataset("data2", data=numpy.ones((10, 10)))
h5_g2.create_dataset("data3", data=numpy.ones((10, 10)))
n5_g2.create_dataset("data3", data=numpy.ones((10, 10)))
h5_g3.create_dataset("data4", data=numpy.ones((10, 10)))
n5_g3.create_dataset("data4", data=numpy.ones((10, 10)))
h5_file.flush()
h5_glob_res1 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
h5_file, f"{h5_file_name}/g1/g2/data*")
n5_glob_res1 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
n5_file, f"{n5_file_name}/g1/g2/data*")
self.assertEqual(h5_glob_res1, expected_datasets)
self.assertEqual(n5_glob_res1, expected_datasets)
finally:
h5_file.close()
n5_file.close()
h5_glob_res2 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
h5_file_name, f"{h5_file_name}/g1/g2/data*")
n5_glob_res2 = OpStreamingH5N5SequenceReaderS.expandGlobStrings(
n5_file_name, f"{n5_file_name}/g1/g2/data*")
self.assertEqual(h5_glob_res2, expected_datasets)
self.assertEqual(n5_glob_res2, expected_datasets)
def test_direct_constructor(self):
self.assertFalse(os.path.exists(self.path))
f = z5py.N5File(self.path)
self.assertFalse(f.is_zarr)
def solve_lifted_subproblems(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
# input configs
problem_path = config['problem_path']
scale = config['scale']
block_shape = config['block_shape']
block_list = config['block_list']
lifted_prefix = config['lifted_prefix']
agglomerator_key = config['agglomerator']
time_limit = config.get('time_limit_solver', None)
n_threads = config.get('threads_per_job', 1)
fu.log("reading problem from %s" % problem_path)
problem = z5py.N5File(problem_path)
shape = problem.attrs['shape']
# load the costs
# NOTE we use different cost identifiers for multicut and lifted multicut
# in order to run both in the same n5-container.
# However, for scale level 0 the costs come from the CostsWorkflow and
# hence the identifier is identical
costs_key = 's%i/costs_lmc' % scale if scale > 0 else 's0/costs'
fu.log("reading costs from path in problem: %s" % costs_key)
ds = problem[costs_key]
ds.n_threads = n_threads
costs = ds[:]
# load the graph
# NOTE we use different graph identifiers for multicut and lifted multicut
# in order to run both in the same n5-container.
# However, for scale level 0 the graph comes from the GraphWorkflow and
# hence the identifier is identical
graph_key = 's%i/graph_lmc' % scale if scale > 0 else 's0/graph'
fu.log("reading graph from path in problem: %s" % graph_key)
graph = ndist.Graph(os.path.join(problem_path, graph_key),
numberOfThreads=n_threads)
uv_ids = graph.uvIds()
# check if the problem has an ignore-label
ignore_label = problem[graph_key].attrs['ignoreLabel']
fu.log("ignore label is %s" % ('true' if ignore_label else 'false'))
fu.log("using agglomerator %s" % agglomerator_key)
lifted_agglomerator = su.key_to_lifted_agglomerator(agglomerator_key)
# TODO enable different multicut agglomerator
agglomerator = su.key_to_agglomerator(agglomerator_key)
# load the lifted edges and costs
nh_key = 's%i/lifted_nh_%s' % (scale, lifted_prefix)
lifted_costs_key = 's%i/lifted_costs_%s' % (scale, lifted_prefix)
ds = problem[nh_key]
fu.log("reading lifted uvs")
ds.n_threads = n_threads
lifted_uvs = ds[:]
fu.log("reading lifted costs")
ds = problem[lifted_costs_key]
ds.n_threads = n_threads
lifted_costs = ds[:]
# the output group
out = problem['s%i/sub_results_lmc' % scale]
# NOTE we use different sub-graph identifiers for multicut and lifted multicut
# in order to run both in the same n5-container.
# However, for scale level 0 the sub-graphs come from the GraphWorkflow and
# are hence identical
sub_graph_identifier = 'sub_graphs' if scale == 0 else 'sub_graphs_lmc'
block_prefix = os.path.join(problem_path, 's%i' % scale,
sub_graph_identifier, 'block_')
blocking = nt.blocking([0, 0, 0], shape, list(block_shape))
fu.log("start processsing %i blocks" % len(block_list))
with futures.ThreadPoolExecutor(n_threads) as tp:
tasks = [
tp.submit(_solve_block_problem, block_id, graph, uv_ids,
block_prefix, costs, lifted_uvs, lifted_costs,
lifted_agglomerator, agglomerator, ignore_label,
blocking, out, time_limit) for block_id in block_list
]
[t.result() for t in tasks]
fu.log_job_success(job_id)
def _accumulate_block(block_id, blocking, ds_in, ds_labels, out_prefix,
graph_block_prefix, filters, sigmas, halo, ignore_label,
apply_in_2d, channel_agglomeration):
fu.log("start processing block %i" % block_id)
# load graph and check if this block has edges
graph = ndist.Graph(graph_block_prefix + str(block_id))
if graph.numberOfEdges == 0:
fu.log("block %i has no edges" % block_id)
fu.log_block_success(block_id)
return
shape = ds_labels.shape
# get the bounding
if sum(halo) > 0:
block = blocking.getBlockWithHalo(block_id, halo)
block_shape = block.outerBlock.shape
bb_in = vu.block_to_bb(block.outerBlock)
bb = vu.block_to_bb(block.innerBlock)
bb_local = vu.block_to_bb(block.innerBlockLocal)
# increase inner bounding box by 1 in posirive direction
# in accordance with the graph extraction
bb = tuple(
slice(b.start, min(b.stop + 1, sh)) for b, sh in zip(bb, shape))
bb_local = tuple(
slice(b.start, min(b.stop + 1, bsh))
for b, bsh in zip(bb_local, block_shape))
else:
block = blocking.getBlock(block_id)
bb = vu.block_to_bb(block)
bb = tuple(
slice(b.start, min(b.stop + 1, sh)) for b, sh in zip(bb, shape))
bb_in = bb
bb_local = slice(None)
input_dim = ds_in.ndim
# TODO make choice of channels optional
if input_dim == 4:
bb_in = (slice(0, 3), ) + bb_in
input_ = vu.normalize(ds_in[bb_in])
if input_dim == 4:
assert channel_agglomeration is not None
input_ = getattr(np, channel_agglomeration)(input_, axis=0)
# load labels
labels = ds_labels[bb]
# TODO pre-smoothing ?!
# accumulate the edge features
edge_features = [
_accumulate_filter(input_, graph, labels, bb_local, filter_name, sigma,
ignore_label, filter_name == filters[-1]
and sigma == sigmas[-1], apply_in_2d)
for filter_name in filters for sigma in sigmas
]
edge_features = np.concatenate(edge_features, axis=1)
# save the features
save_path = out_prefix + str(block_id)
fu.log("saving feature result of shape %s to %s" %
(str(edge_features.shape), save_path))
save_root, save_key = os.path.split(save_path)
with z5py.N5File(save_root) as f:
f.create_dataset(save_key,
data=edge_features,
chunks=edge_features.shape)
fu.log_block_success(block_id)
