def block_components(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
output_path = config['output_path']
output_key = config['output_key']
block_list = config['block_list']
tmp_folder = config['tmp_folder']
block_shape = config['block_shape']
threshold = config['threshold']
threshold_mode = config['threshold_mode']
mask_path = config.get('mask_path', '')
mask_key = config.get('mask_key', '')
channel = config.get('channel', None)
fu.log("Applying threshold %f with mode %s" % (threshold, threshold_mode))
with vu.file_reader(input_path, 'r') as f_in,\
vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = f_out[output_key]
shape = ds_in.shape
if channel is not None:
shape = shape[1:]
assert len(shape) == 3
blocking = nt.blocking([0, 0, 0], list(shape), block_shape)
if mask_path != '':
# note that the mask is usually small enough to keep it
# in memory (and we interpolate to get to the full volume)
# if this does not hold need to change this code!
mask = vu.load_mask(mask_path, mask_key, shape)
offsets = [
_cc_block_with_mask(block_id, blocking, ds_in, ds_out,
threshold, threshold_mode, mask, channel)
for block_id in block_list
]
else:
offsets = [
_cc_block(block_id, blocking, ds_in, ds_out, threshold,
threshold_mode, channel) for block_id in block_list
]
offset_dict = {block_id: off for block_id, off in zip(block_list, offsets)}
save_path = os.path.join(tmp_folder,
'connected_components_offsets_%i.json' % job_id)
with open(save_path, 'w') as f:
json.dump(offset_dict, f)
fu.log_job_success(job_id)
def watershed_from_seeds(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
# read the input cofig
input_path = config['input_path']
input_key = config['input_key']
shape = list(vu.get_shape(input_path, input_key))
if len(shape) == 4:
shape = shape[1:]
block_shape = list(config['block_shape'])
block_list = config['block_list']
# TODO seeds and output might be identical
# in that case we would need in-place logic if we
# want to support h5 (it's fine with n5 as is)
# read the seed and output config
seeds_path = config['seeds_path']
seeds_key = config['seeds_key']
output_path = config['output_path']
output_key = config['output_key']
# check if we have a mask
with_mask = 'mask_path' in config
if with_mask:
mask_path = config['mask_path']
mask_key = config['mask_key']
# get the blocking
blocking = nt.blocking([0, 0, 0], shape, block_shape)
# submit blocks
with vu.file_reader(input_path, 'r') as f_in,\
vu.file_reader(seeds_path, 'r') as f_seeds,\
vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
assert ds_in.ndim in (3, 4)
ds_seeds = f_out[seeds_key]
assert ds_seeds.ndim == 3
ds_out = f_out[output_key]
assert ds_out.ndim == 3
# note that the mask is usually small enough to keep it
# in memory (and we interpolate to get to the full volume)
# if this does not hold need to change this code!
if with_mask:
mask = vu.load_mask(mask_path, mask_key, shape)
for block_id in block_list:
_ws_block_masked(blocking, block_id, ds_in, ds_seeds, ds_out,
mask, config)
else:
for block_id in block_list:
_ws_block(blocking, block_id, ds_in, ds_seeds, ds_out, config)
# log success
fu.log_job_success(job_id)
def block_components(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
output_path = config['output_path']
output_key = config['output_key']
block_list = config['block_list']
tmp_folder = config['tmp_folder']
block_shape = config['block_shape']
threshold = config['threshold']
threshold_mode = config['threshold_mode']
sigma = config.get('sigma_prefilter', 0)
mask_path = config.get('mask_path', '')
mask_key = config.get('mask_key', '')
channel = config.get('channel', None)
fu.log("Applying threshold %f with mode %s" % (threshold, threshold_mode))
with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = f_out[output_key]
shape = ds_in.shape
if channel is not None:
shape = shape[1:]
assert len(shape) == 3
blocking = nt.blocking([0, 0, 0], list(shape), block_shape)
if mask_path != '':
mask = vu.load_mask(mask_path, mask_key, shape)
offsets = [_cc_block_with_mask(block_id, blocking,
ds_in, ds_out, threshold,
threshold_mode, mask, channel,
sigma) for block_id in block_list]
else:
offsets = [_cc_block(block_id, blocking,
ds_in, ds_out, threshold,
threshold_mode, channel, sigma) for block_id in block_list]
offset_dict = {block_id: off for block_id, off in zip(block_list, offsets)}
save_path = os.path.join(tmp_folder,
'connected_components_offsets_%i.json' % job_id)
with open(save_path, 'w') as f:
json.dump(offset_dict, f)
fu.log_job_success(job_id)
def linear(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
# read the input cofig
input_path = config['input_path']
input_key = config['input_key']
block_shape = list(config['block_shape'])
block_list = config['block_list']
# read the output config and path to transformation
output_path = config['output_path']
output_key = config['output_key']
trafo_file = config['transformation']
mask_path = config['mask_path']
mask_key = config['mask_key']
if mask_path != '':
assert mask_key != ''
with vu.file_reader(input_path, 'r') as f:
in_shape = f[input_key].shape
mask = vu.load_mask(mask_path, mask_key, in_shape)
same_file = input_path == output_path
in_place = same_file and (input_key == output_key)
# submit blocks
if same_file:
with vu.file_reader(input_path) as f:
ds_in = f[input_key]
ds_out = ds_in if in_place else f[output_key]
shape = list(ds_in.shape)
trafo = _load_transformation(trafo_file, shape)
blocking = nt.blocking([0, 0, 0], shape, block_shape)
_transform_linear(ds_in, ds_out, trafo, blocking, block_list, mask)
else:
with vu.file_reader(input_path,
'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = f_out[output_key]
shape = list(ds_in.shape)
trafo = _load_transformation(trafo_file, shape)
blocking = nt.blocking([0, 0, 0], shape, block_shape)
_transform_linear(ds_in, ds_out, trafo, blocking, block_list, mask)
# log success
fu.log_job_success(job_id)
def watershed(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
# check if we are running two-pass watershed and if
# so, if we are first or second pass
pass_ = config.get('pass', None)
assert pass_ in (None, 1, 2)
# read the input cofig
input_path = config['input_path']
input_key = config['input_key']
shape = list(vu.get_shape(input_path, input_key))
if len(shape) == 4:
shape = shape[1:]
block_shape = list(config['block_shape'])
block_list = config['block_list']
# read the output config
output_path = config['output_path']
output_key = config['output_key']
# check if we have a mask
with_mask = 'mask_path' in config
if with_mask:
mask_path = config['mask_path']
mask_key = config['mask_key']
# get the blocking
blocking = nt.blocking([0, 0, 0], shape, block_shape)
# submit blocks
with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
assert ds_in.ndim in (3, 4)
ds_out = f_out[output_key]
assert ds_out.ndim == 3
# note that the mask is usually small enough to keep it
# in memory (and we interpolate to get to the full volume)
# if this does not hold need to change this code!
if with_mask:
mask = vu.load_mask(mask_path, mask_key, shape)
for block_id in block_list:
_ws_block_masked(blocking, block_id,
ds_in, ds_out, mask, config, pass_)
else:
for block_id in block_list:
_ws_block(blocking, block_id, ds_in, ds_out, config, pass_)
# log success
fu.log_job_success(job_id)
def two_pass_mws(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
output_path = config['output_path']
output_key = config['output_key']
block_shape = config['block_shape']
block_list = config['block_list']
offsets = config['offsets']
strides = config['strides']
assert len(strides) == 3
assert all(isinstance(stride, int) for stride in strides)
randomize_strides = config['randomize_strides']
assert isinstance(randomize_strides, bool)
noise_level = config['noise_level']
halo = config['halo']
mask_path = config.get('mask_path', '')
mask_key = config.get('mask_key', '')
pass_id = config['pass']
tmp_folder = config['tmp_folder']
max_block_id = config['max_block_id']
with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = f_out[output_key]
shape = ds_in.shape[1:]
blocking = nt.blocking([0, 0, 0], list(shape), block_shape)
if mask_path != '':
mask = vu.load_mask(mask_path, mask_key, shape)
else:
mask = None
mws_fu = _mws_block_pass1 if pass_id == 0 else _mws_block_pass2
[mws_fu(block_id, blocking,
ds_in, ds_out,
mask, offsets,
strides, randomize_strides,
halo, noise_level, max_block_id,
tmp_folder)
for block_id in block_list]
fu.log_job_success(job_id)
def inference(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
output_path = config['output_path']
checkpoint_path = config['checkpoint_path']
block_shape = config['block_shape']
block_list = config['block_list']
halo = config['halo']
framework = config['framework']
n_threads = config['threads_per_job']
fu.log("run iference with framework %s, with %i threads" %
(framework, n_threads))
output_keys = config['output_keys']
channel_mapping = config['channel_mapping']
if config.get('set_visible_device', False):
os.environ['CUDA_VISIBLE_DEVICES'] = str(job_id)
fu.log("setting cuda visible devices to %i" % job_id)
gpu = 0
fu.log("Loading model from %s" % checkpoint_path)
predict = get_predictor(framework)(checkpoint_path, halo, gpu=gpu)
fu.log("Have model")
preprocess = get_preprocessor(framework)
shape = vu.get_shape(input_path, input_key)
blocking = nt.blocking(roiBegin=[0, 0, 0],
roiEnd=list(shape),
blockShape=list(block_shape))
with vu.file_reader(input_path,
'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = [f_out[key] for key in output_keys]
if 'mask_path' in config:
mask = vu.load_mask(config['mask_path'], config['mask_key'], shape)
else:
mask = None
_run_inference(blocking, block_list, halo, ds_in, ds_out, mask,
preprocess, predict, channel_mapping, n_threads)
fu.log_job_success(job_id)
def two_pass_watershed(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
with open(config_path, 'r') as f:
config = json.load(f)
# read the input cofig
input_path = config['input_path']
input_key = config['input_key']
shape = list(vu.get_shape(input_path, input_key))
if len(shape) == 4:
shape = shape[1:]
block_shape = list(config['block_shape'])
block_list = config['block_list']
# read the output config
output_path = config['output_path']
output_key = config['output_key']
# get the blocking
blocking = nt.blocking([0, 0, 0], shape, block_shape)
pass_id = config['pass']
# submit blocks
with vu.file_reader(input_path,
'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
assert ds_in.ndim in (3, 4)
ds_out = f_out[output_key]
assert ds_out.ndim == 3
if 'mask_path' in config:
mask_path = config['mask_path']
mask_key = config['mask_key']
mask = vu.load_mask(mask_path, mask_key, shape)
else:
mask = None
ws_fu = _ws_block if pass_id == 0 else _ws_pass2
for block_id in block_list:
ws_fu(blocking, block_id, ds_in, ds_out, mask, config)
# log success
fu.log_job_success(job_id)
def inference(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
output_path = config['output_path']
output_key = config['output_key']
block_shape = config['block_shape']
block_list = config['block_list']
dtype = config.get('dtype', 'uint8')
n_channels = config.get('n_channels', 1)
halo = config.get('halo', None)
framework = config.get('framework', 'pytorch')
n_threads = config['n_threads']
predict = get_predictor(framework)
preprocess = get_preprocessor(framework)
shape = vu.get_shape(input_path, input_key)
blocking = nt.blocking(roiBegin=[0, 0, 0],
roiEnd=list(shape),
blockShape=list(block_shape))
with vu.file_reader(input_path,
'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = f_in[outputt_key]
if mask_path in config:
mask = vu.load_mask(config['mask_path'], config['mask_key'], shape)
_run_inference_with_mask(blocking, block_list, halo, ds_in, ds_out,
mask, predict, dtype, n_channels)
else:
_run_inference(blocking, block_list, halo, ds_in, ds_out,
preprocess, predict, dtype, n_channels, n_threads)
fu.log_job_success(job_id)
def multiscale_inference(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
input_scales = config['input_scales']
scale_factors = config['scale_factors']
# add a trivial scale factor for the zeroth scale
scale_factors = [[1, 1, 1]] + scale_factors
output_path = config['output_path']
checkpoint_path = config['checkpoint_path']
block_shape = config['block_shape']
block_list = config['block_list']
halos = config['halos']
framework = config['framework']
n_threads = config['threads_per_job']
use_best = config.get('use_best', True)
multiscale_output = config.get('multiscale_output', False)
channel_accumulation = config.get('channel_accumulation', None)
if channel_accumulation is not None:
fu.log("Accumulating channels with %s" % channel_accumulation)
channel_accumulation = getattr(np, channel_accumulation)
fu.log("run inference with framework %s, with %i threads" % (framework, n_threads))
output_keys = config['output_keys']
channel_mapping = config['channel_mapping']
device_mapping = config.get('device_mapping', None)
if device_mapping is not None:
device_id = device_mapping[str(job_id)]
os.environ['CUDA_VISIBLE_DEVICES'] = str(device_id)
fu.log("setting cuda visible devices to %i" % device_id)
gpu = 0
fu.log("Loading model from %s" % checkpoint_path)
prep_model = config.get('prep_model', None)
if prep_model is not None:
prep_model = get_prep_model(prep_model)
predict = get_predictor(framework)(checkpoint_path, halos, gpu=gpu, prep_model=prep_model,
use_best=use_best)
fu.log("Have model")
preprocess = get_preprocessor(framework)
with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out:
g_in = f_in[input_key]
ds_in = [g_in[in_scale] for in_scale in input_scales]
if multiscale_output:
ds_out = [[f_out[os.path.join(key, scale)] for key in output_keys]
for scale in input_scales]
else:
ds_out = [f_out[key] for key in output_keys]
shape = ds_in[0].shape
blocking = nt.blocking(roiBegin=[0, 0, 0],
roiEnd=list(shape),
blockShape=list(block_shape))
if 'mask_path' in config:
mask = vu.load_mask(config['mask_path'], config['mask_key'], shape)
else:
mask = None
_run_inference(blocking, block_list, halos, ds_in, ds_out, mask,
scale_factors, preprocess,
predict, channel_mapping,
channel_accumulation, n_threads,
multiscale_output)
fu.log_job_success(job_id)
def inference(job_id, config_path):
fu.log("start processing job %i" % job_id)
fu.log("reading config from %s" % config_path)
# get the config
with open(config_path) as f:
config = json.load(f)
input_path = config['input_path']
input_key = config['input_key']
output_path = config['output_path']
checkpoint_path = config['checkpoint_path']
block_shape = config['block_shape']
block_list = config['block_list']
halo = config['halo']
framework = config['framework']
n_threads = config['threads_per_job']
use_best = config.get('use_best', True)
fu.log("run iference with framework %s, with %i threads" %
(framework, n_threads))
output_keys = config['output_keys']
channel_mapping = config['channel_mapping']
device_mapping = config.get('device_mapping', None)
if device_mapping is not None:
device_id = device_mapping[str(job_id)]
os.environ['CUDA_VISIBLE_DEVICES'] = str(device_id)
fu.log("setting cuda visible devices to %i" % device_id)
gpu = 0
tda_config = config.get('tda_config', {})
if tda_config:
fu.log("Using test-time-data-augmentation with config:")
fu.log(str(tda_config))
fu.log("Loading model from %s" % checkpoint_path)
prep_model = config.get('prep_model', None)
if prep_model is not None:
prep_model = get_prep_model(prep_model)
predict = get_predictor(framework)(checkpoint_path,
halo,
gpu=gpu,
prep_model=prep_model,
use_best=use_best,
**tda_config)
fu.log("Have model")
preprocess = get_preprocessor(framework)
shape = vu.get_shape(input_path, input_key)
blocking = nt.blocking(roiBegin=[0, 0, 0],
roiEnd=list(shape),
blockShape=list(block_shape))
with vu.file_reader(input_path,
'r') as f_in, vu.file_reader(output_path) as f_out:
ds_in = f_in[input_key]
ds_out = [f_out[key] for key in output_keys]
if 'mask_path' in config:
mask = vu.load_mask(config['mask_path'], config['mask_key'], shape)
else:
mask = None
_run_inference(blocking, block_list, halo, ds_in, ds_out, mask,
preprocess, predict, channel_mapping, n_threads)
fu.log_job_success(job_id)