def test_graph_label_multiset(self):
from cluster_tools.graph import GraphWorkflow
task = GraphWorkflow
task_config = GraphWorkflow.get_config()['initial_sub_graphs']
task_config['ignore_label'] = False
with open(
os.path.join(self.config_folder, 'initial_sub_graphs.config'),
'w') as f:
json.dump(task_config, f)
ret = luigi.build([
task(input_path=self.input_path,
input_key=self.label_multiset_key,
graph_path=self.output_path,
output_key=self.output_key,
n_scales=1,
config_dir=self.config_folder,
tmp_folder=self.tmp_folder,
target=self.target,
max_jobs=self.max_jobs)
],
local_scheduler=True)
self.assertTrue(ret)
self.check_subresults(self.label_multiset_key_ref)
self.check_result(self.label_multiset_key_ref)
def graph_example(shebang):
input_path = '/home/cpape/Work/data/isbi2012/cluster_example/isbi_train.n5'
input_key = 'volumes/watersheds'
output_path = '/home/cpape/Work/data/isbi2012/cluster_example/graph.n5'
tmp_folder = './tmp'
config_folder = './configs'
max_jobs = 8
global_conf = WatershedLocal.default_global_config()
global_conf.update({'shebang': shebang, 'block_shape': [10, 256, 256]})
with open('./configs/global.config', 'w') as f:
json.dump(global_conf, f)
ret = luigi.build([
GraphWorkflow(input_path=input_path,
input_key=input_key,
graph_path=output_path,
n_scales=1,
config_dir=config_folder,
tmp_folder=tmp_folder,
target='local',
max_jobs=max_jobs)
],
local_scheduler=True)
def test_lifted_nh(self):
graph_path = os.path.join(self.tmp_folder, 'graph.n5')
graph_key = 'graph'
graph_config = GraphWorkflow.get_config()['initial_sub_graphs']
graph_config["ignore_label"] = False
with open(os.path.join(self.config_folder, 'initial_sub_graphs.config'), 'w') as f:
json.dump(graph_config, f)
task_graph = GraphWorkflow(tmp_folder=self.tmp_folder, config_dir=self.config_folder,
max_jobs=8, target='local',
input_path=self.input_path, input_key=self.ws_key,
graph_path=graph_path, output_key=graph_key)
ret = luigi.build([task_graph],
local_scheduler=True)
self.assertTrue(ret)
graph = ndist.loadAsUndirectedGraph(os.path.join(graph_path, graph_key))
n_nodes = graph.numberOfNodes
node_labels = np.ones(n_nodes, dtype='uint64')
node_label_path = os.path.join(self.tmp_folder, 'node_labels.n5')
node_label_key = 'node_labels'
with z5py.File(node_label_path) as f:
ds = f.require_dataset(node_label_key, shape=node_labels.shape, dtype=node_labels.dtype,
chunks=(1000,), compression='gzip')
ds[:] = node_labels
# TODO try different graph depth and different number of threads !
graph_depth = 3
out_path = os.path.join(self.tmp_folder, 'lifted_nh.h5')
out_key = 'lifted_nh'
task_nh = SparseLiftedNeighborhoodLocal(tmp_folder=self.tmp_folder, config_dir=self.config_folder,
max_jobs=1, dependency=task_graph,
graph_path=graph_path, graph_key=graph_key,
node_label_path=node_label_path, node_label_key=node_label_key,
output_path=out_path, output_key=out_key,
prefix='', nh_graph_depth=graph_depth)
ret = luigi.build([task_nh], local_scheduler=True)
self.assertTrue(ret)
self._check_result(graph_depth)
def test_lifted_nh_with_labels(self):
node_label_path = os.path.join(self.tmp_folder, 'node_labels.n5')
node_label_key = 'node_labels'
task_labels = NodeLabelWorkflow(tmp_folder=self.tmp_folder, config_dir=self.config_folder,
max_jobs=8, target='local',
ws_path=self.input_path, ws_key=self.ws_key,
input_path=self.input_path, input_key=self.labels_key,
output_path=node_label_path, output_key=node_label_key,
max_overlap=True)
graph_path = os.path.join(self.tmp_folder, 'graph.n5')
graph_key = 'graph'
graph_config = GraphWorkflow.get_config()['initial_sub_graphs']
graph_config["ignore_label"] = False
with open(os.path.join(self.config_folder, 'initial_sub_graphs.config'), 'w') as f:
json.dump(graph_config, f)
task_graph = GraphWorkflow(tmp_folder=self.tmp_folder, config_dir=self.config_folder,
max_jobs=8, target='local',
input_path=self.input_path, input_key=self.ws_key,
graph_path=graph_path, output_key=graph_key)
ret = luigi.build([task_labels, task_graph],
local_scheduler=True)
self.assertTrue(ret)
# TODO try different graph depth and different number of threads !
graph_depth = 3
out_path = os.path.join(self.tmp_folder, 'lifted_nh.h5')
out_key = 'lifted_nh'
task_nh = SparseLiftedNeighborhoodLocal(tmp_folder=self.tmp_folder, config_dir=self.config_folder,
max_jobs=1, dependency=task_graph,
graph_path=graph_path, graph_key=graph_key,
node_label_path=node_label_path, node_label_key=node_label_key,
output_path=out_path, output_key=out_key,
prefix='', nh_graph_depth=graph_depth)
ret = luigi.build([task_nh], local_scheduler=True)
self.assertTrue(ret)
self._check_result(graph_depth)
def test_graph(self):
max_jobs = 8
ret = luigi.build([
GraphWorkflow(input_path=self.input_path,
input_key=self.input_key,
graph_path=self.output_path,
n_scales=1,
config_dir=self.config_folder,
tmp_folder=self.tmp_folder,
target=self.target,
max_jobs=max_jobs)
],
local_scheduler=True)
self.assertTrue(ret)
self._check_subresults()
self._check_result()
def setUp(self):
self._mkdir(self.tmp_folder)
self._mkdir(self.config_folder)
configs = GraphWorkflow.get_config()
global_config = configs['global']
# global_config['shebang'] = '#! /home/cpape/Work/software/conda/miniconda3/envs/affogato/bin/python'
global_config[
'shebang'] = '#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python'
global_config['block_shape'] = self.block_shape
with open(os.path.join(self.config_folder, 'global.config'), 'w') as f:
json.dump(global_config, f)
task_config = configs['initial_sub_graphs']
task_config['ignore_label'] = False
with open(
os.path.join(self.config_folder, 'initial_sub_graphs.config'),
'w') as f:
json.dump(task_config, f)
def merge_boutons(target, max_jobs):
qos = 'normal'
path = '/g/rompani/pape/lgn/data.n5/predictions'
tmp_folder = '/g/rompani/pape/lgn/tmp_pp'
offsets = [[-1, 0, 0], [0, -1, 0], [0, 0, -1],
[-2, 0, 0], [0, -3, 0], [0, 0, -3]]
fg_key = 'predictions/foreground'
seg_key = 'predictions/mws-seg'
aff_key = 'predictions/affinities'
conf_dir = os.path.join(tmp_folder, 'configs')
os.makedirs(conf_dir, exist_ok=True)
problem_path = os.path.join(tmp_folder, 'data.n5')
graph_key = 'graph'
feat_key = 'feats'
size_key = 'region_sizes'
pp_node_label_key = 'node_labels/gws'
configs = GraphWorkflow.get_config()
conf = configs['global']
shebang = '/g/kreshuk/pape/Work/software/conda/miniconda3/envs/torch13/bin/python'
block_shape = [32, 256, 256]
conf.update({'shebang': shebang, 'block_shape': block_shape})
with open(os.path.join(conf_dir, 'global.config'), 'w') as f:
json.dump(conf, f)
conf = configs['initial_sub_graphs']
conf.update({'ignore_label': True, 'mem_limit': 2, 'qos': qos})
with open(os.path.join(conf_dir, 'initial_sub_graphs.config'), 'w') as f:
json.dump(conf, f)
conf = configs['merge_sub_graphs']
conf.update({'mem_limit': 64, 'threads_per_job': 16, 'qos': qos})
with open(os.path.join(conf_dir, 'merge_sub_graphs.config'), 'w') as f:
json.dump(conf, f)
conf = configs['map_edge_ids']
conf.update({'mem_limit': 64, 'threads_per_job': 8, 'qos': qos})
with open(os.path.join(conf_dir, 'map_edge_ids.config'), 'w') as f:
json.dump(conf, f)
task = GraphWorkflow(tmp_folder=tmp_folder, config_dir=conf_dir,
max_jobs=max_jobs, target=target,
input_path=path, input_key=seg_key,
graph_path=problem_path, output_key=graph_key)
ret = luigi.build([task], local_scheduler=True)
assert ret, "Graph extraction failed"
# write config for edge feature task
configs = EdgeFeaturesWorkflow.get_config()
conf = configs['block_edge_features']
conf.update({'offsets': offsets, 'qos': qos})
with open(os.path.join(conf_dir, 'block_edge_features.config'), 'w') as f:
json.dump(conf, f)
conf = configs['merge_edge_features']
conf.update({'mem_limit': 64, 'threads_per_job': 12, 'qos': qos,
'time_limit': 360})
with open(os.path.join(conf_dir, 'merge_edge_features.config'), 'w') as f:
json.dump(conf, f)
task = EdgeFeaturesWorkflow(tmp_folder=tmp_folder, config_dir=conf_dir,
max_jobs=max_jobs, target=target,
input_path=path, input_key=aff_key,
labels_path=path, labels_key=seg_key,
graph_path=problem_path, graph_key=graph_key,
output_path=problem_path, output_key=feat_key)
ret = luigi.build([task], local_scheduler=True)
assert ret, "Edge feature extraction failed"
configs = RegionFeaturesWorkflow.get_config()
conf = configs['region_features']
conf.update({'mem_limit': 2, 'time_limit': 360, 'qos': qos, 'threads_per_job': 1})
with open(os.path.join(conf_dir, 'region_features.config'), 'w') as f:
json.dump(conf, f)
conf = configs['merge_region_features']
conf.update({'mem_limit': 2, 'time_limit': 360, 'qos': qos})
with open(os.path.join(conf_dir, 'merge_region_features.config'), 'w') as f:
json.dump(conf, f)
task = RegionFeaturesWorkflow(tmp_folder=tmp_folder, config_dir=conf_dir,
max_jobs=max_jobs, target=target,
input_path=path, input_key=fg_key,
labels_path=path, labels_key=seg_key,
output_path=problem_path, output_key=size_key,
max_jobs_merge=16)
ret = luigi.build([task], local_scheduler=True)
assert ret, "Region feature extraction failed"
# TODO proper size filter ?
min_component_size = 2000
graph_merging(problem_path, graph_key, feat_key, size_key,
pp_node_label_key, min_component_size=min_component_size)
out_key = 'predictions/pp-seg'
task = WriteSlurm if target == 'slurm' else WriteLocal
conf = task.default_task_config()
conf.update({'mem_limit': 8})
with open(os.path.join(conf_dir, 'write.config'), 'w') as f:
json.dump(conf, f)
t = task(tmp_folder=tmp_folder, config_dir=conf_dir, max_jobs=max_jobs,
input_path=path, input_key=seg_key,
output_path=path, output_key=out_key,
assignment_path=problem_path,
assignment_key=pp_node_label_key,
identifier='write-pp')
luigi.build([t], local_scheduler=True)