def test_affmap_features(self):
print("Test affmap features")
features_out = './tmpdir/features_affmap.n5'
ffeats = z5py.File(features_out, use_zarr_format=False)
ffeats.create_group('blocks')
features_tmp = os.path.join(features_out, 'blocks')
def extract_block(block_id):
ndist.extractBlockFeaturesFromAffinityMaps(
self.graph_path, 'sub_graphs/s0/block_', self.path,
self.full_key, self.path, self.labels_key, [block_id],
features_tmp, self.offsets)
n_blocks = self.blocking.numberOfBlocks
n_threads = 8
with futures.ThreadPoolExecutor(n_threads) as tp:
tasks = [
tp.submit(extract_block, block_id)
for block_id in range(n_blocks)
]
[t.result() for t in tasks]
subgraph_path = os.path.join(self.graph_path, 'sub_graphs', 's0')
for block_id in range(n_blocks):
# TODO compare to nifty features
key = 'block_%i' % block_id
self.check_features(features_tmp, key, subgraph_path, key)
n_edges = z5py.File(self.graph_path)['graph'].attrs['numberOfEdges']
chunk_size = min(2097152, n_edges)
if 'features' not in ffeats:
ffeats.create_dataset('features',
dtype='float64',
shape=(n_edges, 10),
chunks=(chunk_size, 1),
compression='gzip')
graph_block_prefix = os.path.join(self.graph_path, 'sub_graphs', 's0',
'block_')
features_tmp_prefix = os.path.join(features_out, 'blocks/block_')
edge_offset = 0
ndist.mergeFeatureBlocks(graph_block_prefix,
features_tmp_prefix,
os.path.join(features_out, 'features'),
n_blocks,
edge_offset,
n_edges,
numberOfThreads=n_threads)
print("Checking merged features")
self.check_features(features_out, 'features', self.graph_path, 'graph')
def features_step2(graph_block_prefix, features_tmp_prefix, out_path, out_key,
input_file, n_threads):
t0 = time.time()
edge_begin, edge_end, n_blocks = np.load(input_file)
ndist.mergeFeatureBlocks(graph_block_prefix,
features_tmp_prefix,
os.path.join(out_path, out_key),
numberOfBlocks=n_blocks,
edgeIdBegin=edge_begin,
edgeIdEnd=edge_end,
numberOfThreads=n_threads)
job_id = int(os.path.split(input_file)[1].split('_')[2][:-4])
print("Success job %i" % job_id)
print("In %f s" % (time.time() - t0, ))
def merge_edge_features(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path, 'r') as f:
config = json.load(f)
graph_path = config['graph_path']
subgraph_key = config['subgraph_key']
in_path = config['in_path']
output_path = config['output_path']
subfeat_key = config['subfeat_key']
output_key = config['output_key']
n_threads = config['threads_per_job']
edge_block_list = config['block_list']
edge_chunk_size = config['edge_chunk_size']
block_ids = config['block_ids']
# assert that the edge block list is consecutive
diff_list = np.diff(edge_block_list)
assert (diff_list == 1).all()
n_edges = config['n_edges']
edge_blocking = nt.blocking([0], [n_edges], [edge_chunk_size])
edge_begin = edge_blocking.getBlock(edge_block_list[0]).begin[0]
edge_end = edge_blocking.getBlock(edge_block_list[-1]).end[0]
# the block list might either be the number of blocks or a list of blocks
block_ids = list(range(block_ids)) if isinstance(block_ids,
int) else block_ids
ndist.mergeFeatureBlocks(graph_path,
subgraph_key,
in_path,
subfeat_key,
output_path,
output_key,
blockIds=block_ids,
edgeIdBegin=edge_begin,
edgeIdEnd=edge_end,
numberOfThreads=n_threads)
fu.log_job_success(job_id)
def merge_features(graph_path, out_path, config_path, tmp_folder):
t0 = time.time()
edge_begin = 0
out_key = 'features'
edge_end = z5py.File(out_path)[out_key].shape[0]
with open(config_path) as f:
config = json.load(f)
block_shape = config['block_shape']
n_threads = config['n_threads']
roi = config.get('roi', None)
shape = z5py.File(graph_path).attrs['shape']
blocking = nifty.tools.blocking([0, 0, 0], shape, block_shape)
subgraph_prefix = os.path.join(graph_path, 'sub_graphs', 's0', 'block_')
features_tmp_prefix = os.path.join(out_path, 'blocks', 'block_')
# adjust to roi if necessary
if roi is None:
n_blocks = blocking.numberOfBlocks
ndist.mergeAllFeatureBlocks(subgraph_prefix,
features_tmp_prefix,
os.path.join(out_path, out_key),
numberOfBlocks=n_blocks,
edgeIdBegin=edge_begin,
edgeIdEnd=edge_end,
numberOfThreads=n_threads)
else:
block_list = blocking.getBlockIdsOverlappingBoundingBox(
roi[0], roi[1], [0, 0, 0]).tolist()
ndist.mergeFeatureBlocks(subgraph_prefix,
features_tmp_prefix,
os.path.join(out_path, out_key),
blockIds=block_list,
edgeIdBegin=edge_begin,
edgeIdEnd=edge_end,
numberOfThreads=n_threads)
res_file = os.path.join(tmp_folder, 'merge_features.log')
with open(res_file, 'w') as f:
json.dump({'t': time.time() - t0}, f)
def merge_features(graph_path, n_blocks, features_out):
ffeats = z5py.File(features_out)
n_edges = z5py.File(graph_path)['graph'].attrs['numberOfEdges']
features_tmp_prefix = os.path.join(features_out, 'blocks/block_')
# chunk size = 64**3
chunk_size = min(262144, n_edges)
if 'features' not in ffeats:
ffeats.create_dataset('features',
dtype='float64',
shape=(n_edges, 10),
chunks=(chunk_size, 1),
compression='gzip')
graph_block_prefix = os.path.join(graph_path, 'sub_graphs', 's0', 'block_')
n_threads = 8
edge_offset = 0
ndist.mergeFeatureBlocks(graph_block_prefix,
features_tmp_prefix,
os.path.join(features_out, 'features'),
n_blocks,
edge_offset,
n_edges,
numberOfThreads=n_threads)