def block_components(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] output_path = config['output_path'] output_key = config['output_key'] block_list = config['block_list'] tmp_folder = config['tmp_folder'] block_shape = config['block_shape'] threshold = config['threshold'] threshold_mode = config['threshold_mode'] mask_path = config.get('mask_path', '') mask_key = config.get('mask_key', '') channel = config.get('channel', None) fu.log("Applying threshold %f with mode %s" % (threshold, threshold_mode)) with vu.file_reader(input_path, 'r') as f_in,\ vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = f_out[output_key] shape = ds_in.shape if channel is not None: shape = shape[1:] assert len(shape) == 3 blocking = nt.blocking([0, 0, 0], list(shape), block_shape) if mask_path != '': # note that the mask is usually small enough to keep it # in memory (and we interpolate to get to the full volume) # if this does not hold need to change this code! mask = vu.load_mask(mask_path, mask_key, shape) offsets = [ _cc_block_with_mask(block_id, blocking, ds_in, ds_out, threshold, threshold_mode, mask, channel) for block_id in block_list ] else: offsets = [ _cc_block(block_id, blocking, ds_in, ds_out, threshold, threshold_mode, channel) for block_id in block_list ] offset_dict = {block_id: off for block_id, off in zip(block_list, offsets)} save_path = os.path.join(tmp_folder, 'connected_components_offsets_%i.json' % job_id) with open(save_path, 'w') as f: json.dump(offset_dict, f) fu.log_job_success(job_id)
def watershed_from_seeds(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) # read the input cofig input_path = config['input_path'] input_key = config['input_key'] shape = list(vu.get_shape(input_path, input_key)) if len(shape) == 4: shape = shape[1:] block_shape = list(config['block_shape']) block_list = config['block_list'] # TODO seeds and output might be identical # in that case we would need in-place logic if we # want to support h5 (it's fine with n5 as is) # read the seed and output config seeds_path = config['seeds_path'] seeds_key = config['seeds_key'] output_path = config['output_path'] output_key = config['output_key'] # check if we have a mask with_mask = 'mask_path' in config if with_mask: mask_path = config['mask_path'] mask_key = config['mask_key'] # get the blocking blocking = nt.blocking([0, 0, 0], shape, block_shape) # submit blocks with vu.file_reader(input_path, 'r') as f_in,\ vu.file_reader(seeds_path, 'r') as f_seeds,\ vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] assert ds_in.ndim in (3, 4) ds_seeds = f_out[seeds_key] assert ds_seeds.ndim == 3 ds_out = f_out[output_key] assert ds_out.ndim == 3 # note that the mask is usually small enough to keep it # in memory (and we interpolate to get to the full volume) # if this does not hold need to change this code! if with_mask: mask = vu.load_mask(mask_path, mask_key, shape) for block_id in block_list: _ws_block_masked(blocking, block_id, ds_in, ds_seeds, ds_out, mask, config) else: for block_id in block_list: _ws_block(blocking, block_id, ds_in, ds_seeds, ds_out, config) # log success fu.log_job_success(job_id)
def block_components(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] output_path = config['output_path'] output_key = config['output_key'] block_list = config['block_list'] tmp_folder = config['tmp_folder'] block_shape = config['block_shape'] threshold = config['threshold'] threshold_mode = config['threshold_mode'] sigma = config.get('sigma_prefilter', 0) mask_path = config.get('mask_path', '') mask_key = config.get('mask_key', '') channel = config.get('channel', None) fu.log("Applying threshold %f with mode %s" % (threshold, threshold_mode)) with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = f_out[output_key] shape = ds_in.shape if channel is not None: shape = shape[1:] assert len(shape) == 3 blocking = nt.blocking([0, 0, 0], list(shape), block_shape) if mask_path != '': mask = vu.load_mask(mask_path, mask_key, shape) offsets = [_cc_block_with_mask(block_id, blocking, ds_in, ds_out, threshold, threshold_mode, mask, channel, sigma) for block_id in block_list] else: offsets = [_cc_block(block_id, blocking, ds_in, ds_out, threshold, threshold_mode, channel, sigma) for block_id in block_list] offset_dict = {block_id: off for block_id, off in zip(block_list, offsets)} save_path = os.path.join(tmp_folder, 'connected_components_offsets_%i.json' % job_id) with open(save_path, 'w') as f: json.dump(offset_dict, f) fu.log_job_success(job_id)
def linear(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) # read the input cofig input_path = config['input_path'] input_key = config['input_key'] block_shape = list(config['block_shape']) block_list = config['block_list'] # read the output config and path to transformation output_path = config['output_path'] output_key = config['output_key'] trafo_file = config['transformation'] mask_path = config['mask_path'] mask_key = config['mask_key'] if mask_path != '': assert mask_key != '' with vu.file_reader(input_path, 'r') as f: in_shape = f[input_key].shape mask = vu.load_mask(mask_path, mask_key, in_shape) same_file = input_path == output_path in_place = same_file and (input_key == output_key) # submit blocks if same_file: with vu.file_reader(input_path) as f: ds_in = f[input_key] ds_out = ds_in if in_place else f[output_key] shape = list(ds_in.shape) trafo = _load_transformation(trafo_file, shape) blocking = nt.blocking([0, 0, 0], shape, block_shape) _transform_linear(ds_in, ds_out, trafo, blocking, block_list, mask) else: with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = f_out[output_key] shape = list(ds_in.shape) trafo = _load_transformation(trafo_file, shape) blocking = nt.blocking([0, 0, 0], shape, block_shape) _transform_linear(ds_in, ds_out, trafo, blocking, block_list, mask) # log success fu.log_job_success(job_id)
def watershed(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) # check if we are running two-pass watershed and if # so, if we are first or second pass pass_ = config.get('pass', None) assert pass_ in (None, 1, 2) # read the input cofig input_path = config['input_path'] input_key = config['input_key'] shape = list(vu.get_shape(input_path, input_key)) if len(shape) == 4: shape = shape[1:] block_shape = list(config['block_shape']) block_list = config['block_list'] # read the output config output_path = config['output_path'] output_key = config['output_key'] # check if we have a mask with_mask = 'mask_path' in config if with_mask: mask_path = config['mask_path'] mask_key = config['mask_key'] # get the blocking blocking = nt.blocking([0, 0, 0], shape, block_shape) # submit blocks with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] assert ds_in.ndim in (3, 4) ds_out = f_out[output_key] assert ds_out.ndim == 3 # note that the mask is usually small enough to keep it # in memory (and we interpolate to get to the full volume) # if this does not hold need to change this code! if with_mask: mask = vu.load_mask(mask_path, mask_key, shape) for block_id in block_list: _ws_block_masked(blocking, block_id, ds_in, ds_out, mask, config, pass_) else: for block_id in block_list: _ws_block(blocking, block_id, ds_in, ds_out, config, pass_) # log success fu.log_job_success(job_id)
def two_pass_mws(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] output_path = config['output_path'] output_key = config['output_key'] block_shape = config['block_shape'] block_list = config['block_list'] offsets = config['offsets'] strides = config['strides'] assert len(strides) == 3 assert all(isinstance(stride, int) for stride in strides) randomize_strides = config['randomize_strides'] assert isinstance(randomize_strides, bool) noise_level = config['noise_level'] halo = config['halo'] mask_path = config.get('mask_path', '') mask_key = config.get('mask_key', '') pass_id = config['pass'] tmp_folder = config['tmp_folder'] max_block_id = config['max_block_id'] with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = f_out[output_key] shape = ds_in.shape[1:] blocking = nt.blocking([0, 0, 0], list(shape), block_shape) if mask_path != '': mask = vu.load_mask(mask_path, mask_key, shape) else: mask = None mws_fu = _mws_block_pass1 if pass_id == 0 else _mws_block_pass2 [mws_fu(block_id, blocking, ds_in, ds_out, mask, offsets, strides, randomize_strides, halo, noise_level, max_block_id, tmp_folder) for block_id in block_list] fu.log_job_success(job_id)
def inference(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) # get the config with open(config_path) as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] output_path = config['output_path'] checkpoint_path = config['checkpoint_path'] block_shape = config['block_shape'] block_list = config['block_list'] halo = config['halo'] framework = config['framework'] n_threads = config['threads_per_job'] fu.log("run iference with framework %s, with %i threads" % (framework, n_threads)) output_keys = config['output_keys'] channel_mapping = config['channel_mapping'] if config.get('set_visible_device', False): os.environ['CUDA_VISIBLE_DEVICES'] = str(job_id) fu.log("setting cuda visible devices to %i" % job_id) gpu = 0 fu.log("Loading model from %s" % checkpoint_path) predict = get_predictor(framework)(checkpoint_path, halo, gpu=gpu) fu.log("Have model") preprocess = get_preprocessor(framework) shape = vu.get_shape(input_path, input_key) blocking = nt.blocking(roiBegin=[0, 0, 0], roiEnd=list(shape), blockShape=list(block_shape)) with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = [f_out[key] for key in output_keys] if 'mask_path' in config: mask = vu.load_mask(config['mask_path'], config['mask_key'], shape) else: mask = None _run_inference(blocking, block_list, halo, ds_in, ds_out, mask, preprocess, predict, channel_mapping, n_threads) fu.log_job_success(job_id)
def two_pass_watershed(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) with open(config_path, 'r') as f: config = json.load(f) # read the input cofig input_path = config['input_path'] input_key = config['input_key'] shape = list(vu.get_shape(input_path, input_key)) if len(shape) == 4: shape = shape[1:] block_shape = list(config['block_shape']) block_list = config['block_list'] # read the output config output_path = config['output_path'] output_key = config['output_key'] # get the blocking blocking = nt.blocking([0, 0, 0], shape, block_shape) pass_id = config['pass'] # submit blocks with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] assert ds_in.ndim in (3, 4) ds_out = f_out[output_key] assert ds_out.ndim == 3 if 'mask_path' in config: mask_path = config['mask_path'] mask_key = config['mask_key'] mask = vu.load_mask(mask_path, mask_key, shape) else: mask = None ws_fu = _ws_block if pass_id == 0 else _ws_pass2 for block_id in block_list: ws_fu(blocking, block_id, ds_in, ds_out, mask, config) # log success fu.log_job_success(job_id)
def inference(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) # get the config with open(config_path) as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] output_path = config['output_path'] output_key = config['output_key'] block_shape = config['block_shape'] block_list = config['block_list'] dtype = config.get('dtype', 'uint8') n_channels = config.get('n_channels', 1) halo = config.get('halo', None) framework = config.get('framework', 'pytorch') n_threads = config['n_threads'] predict = get_predictor(framework) preprocess = get_preprocessor(framework) shape = vu.get_shape(input_path, input_key) blocking = nt.blocking(roiBegin=[0, 0, 0], roiEnd=list(shape), blockShape=list(block_shape)) with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = f_in[outputt_key] if mask_path in config: mask = vu.load_mask(config['mask_path'], config['mask_key'], shape) _run_inference_with_mask(blocking, block_list, halo, ds_in, ds_out, mask, predict, dtype, n_channels) else: _run_inference(blocking, block_list, halo, ds_in, ds_out, preprocess, predict, dtype, n_channels, n_threads) fu.log_job_success(job_id)
def multiscale_inference(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) # get the config with open(config_path) as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] input_scales = config['input_scales'] scale_factors = config['scale_factors'] # add a trivial scale factor for the zeroth scale scale_factors = [[1, 1, 1]] + scale_factors output_path = config['output_path'] checkpoint_path = config['checkpoint_path'] block_shape = config['block_shape'] block_list = config['block_list'] halos = config['halos'] framework = config['framework'] n_threads = config['threads_per_job'] use_best = config.get('use_best', True) multiscale_output = config.get('multiscale_output', False) channel_accumulation = config.get('channel_accumulation', None) if channel_accumulation is not None: fu.log("Accumulating channels with %s" % channel_accumulation) channel_accumulation = getattr(np, channel_accumulation) fu.log("run inference with framework %s, with %i threads" % (framework, n_threads)) output_keys = config['output_keys'] channel_mapping = config['channel_mapping'] device_mapping = config.get('device_mapping', None) if device_mapping is not None: device_id = device_mapping[str(job_id)] os.environ['CUDA_VISIBLE_DEVICES'] = str(device_id) fu.log("setting cuda visible devices to %i" % device_id) gpu = 0 fu.log("Loading model from %s" % checkpoint_path) prep_model = config.get('prep_model', None) if prep_model is not None: prep_model = get_prep_model(prep_model) predict = get_predictor(framework)(checkpoint_path, halos, gpu=gpu, prep_model=prep_model, use_best=use_best) fu.log("Have model") preprocess = get_preprocessor(framework) with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: g_in = f_in[input_key] ds_in = [g_in[in_scale] for in_scale in input_scales] if multiscale_output: ds_out = [[f_out[os.path.join(key, scale)] for key in output_keys] for scale in input_scales] else: ds_out = [f_out[key] for key in output_keys] shape = ds_in[0].shape blocking = nt.blocking(roiBegin=[0, 0, 0], roiEnd=list(shape), blockShape=list(block_shape)) if 'mask_path' in config: mask = vu.load_mask(config['mask_path'], config['mask_key'], shape) else: mask = None _run_inference(blocking, block_list, halos, ds_in, ds_out, mask, scale_factors, preprocess, predict, channel_mapping, channel_accumulation, n_threads, multiscale_output) fu.log_job_success(job_id)
def inference(job_id, config_path): fu.log("start processing job %i" % job_id) fu.log("reading config from %s" % config_path) # get the config with open(config_path) as f: config = json.load(f) input_path = config['input_path'] input_key = config['input_key'] output_path = config['output_path'] checkpoint_path = config['checkpoint_path'] block_shape = config['block_shape'] block_list = config['block_list'] halo = config['halo'] framework = config['framework'] n_threads = config['threads_per_job'] use_best = config.get('use_best', True) fu.log("run iference with framework %s, with %i threads" % (framework, n_threads)) output_keys = config['output_keys'] channel_mapping = config['channel_mapping'] device_mapping = config.get('device_mapping', None) if device_mapping is not None: device_id = device_mapping[str(job_id)] os.environ['CUDA_VISIBLE_DEVICES'] = str(device_id) fu.log("setting cuda visible devices to %i" % device_id) gpu = 0 tda_config = config.get('tda_config', {}) if tda_config: fu.log("Using test-time-data-augmentation with config:") fu.log(str(tda_config)) fu.log("Loading model from %s" % checkpoint_path) prep_model = config.get('prep_model', None) if prep_model is not None: prep_model = get_prep_model(prep_model) predict = get_predictor(framework)(checkpoint_path, halo, gpu=gpu, prep_model=prep_model, use_best=use_best, **tda_config) fu.log("Have model") preprocess = get_preprocessor(framework) shape = vu.get_shape(input_path, input_key) blocking = nt.blocking(roiBegin=[0, 0, 0], roiEnd=list(shape), blockShape=list(block_shape)) with vu.file_reader(input_path, 'r') as f_in, vu.file_reader(output_path) as f_out: ds_in = f_in[input_key] ds_out = [f_out[key] for key in output_keys] if 'mask_path' in config: mask = vu.load_mask(config['mask_path'], config['mask_key'], shape) else: mask = None _run_inference(blocking, block_list, halo, ds_in, ds_out, mask, preprocess, predict, channel_mapping, n_threads) fu.log_job_success(job_id)