def preprocess():
path = './data.n5'
in_key = 'labels_for_subdivision_mip/s0'
root_key = 'labels_for_subdivision_mip'
n_scales = 4
scale_factors = n_scales * [[2, 2, 2]]
halos = scale_factors
tmp_folder = './tmp_subdivision_labels/tmp2'
config_dir = os.path.join(tmp_folder, 'configs')
os.makedirs(config_dir, exist_ok=True)
conf = DownscalingWorkflow.get_config()['downscaling']
conf.update({'library_kwargs': {'order': 0}})
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(conf, f)
target = 'local'
max_jobs = 16
task = DownscalingWorkflow(tmp_folder=tmp_folder,
config_dir=config_dir,
target=target,
max_jobs=max_jobs,
input_path=path,
input_key=in_key,
scale_factors=scale_factors,
halos=halos,
output_path=path,
output_key_prefix=root_key)
luigi.build([task], local_scheduler=True)
def downscale_predictions(max_jobs=16, target='slurm'):
path = '/g/arendt/EM_6dpf_segmentation/EM-Prospr/em-segmented-membranes-parapodium.h5'
tmp_folder = './tmp_bdv_parapodium'
config_dir = './configs_bdv_parapodium'
try:
os.mkdir(config_dir)
except OSError:
pass
config = DownscalingWorkflow.get_config()
global_config = config['global']
global_config.update({
'shebang':
'#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python'
})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
ds_config = config['downscaling']
ds_config.update({'threads_per_job': max_jobs})
ds_config.update({'mem_limit': 32})
ds_config.update({'time_limit': 360})
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(ds_config, f)
n_scales = 5
scale_factors = n_scales * [[2, 2, 2]]
halos = n_scales * [[10, 10, 10]]
resolution = (.025, .02, .02)
offsets = (5084, 0, 0)
# transfer the offsets from measure in pixels to measure in
# micrometer
offsets = tuple(off * res for off, res in zip(offsets, resolution))
metadata = {
'unit': 'micrometer',
'resolution': resolution,
'offsets': offsets
}
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=1,
config_dir=config_dir,
target=target,
input_path=path,
input_key='boundary_channel',
scale_factors=scale_factors,
halos=halos,
metadata_format='bdv',
metadata_dict=metadata)
success = luigi.build([task], local_scheduler=True)
def downscale_test():
tmp_folder = './tmp_bdv'
config_dir = './configs_bdv'
try:
os.mkdir(config_dir)
except OSError:
pass
config = DownscalingWorkflow.get_config()
global_config = config['global']
global_config.update({
'shebang':
'#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python'
})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
ds_config = config['downscaling']
ds_config.update({'threads_per_job': 8, 'chunks': (25, 64, 64)})
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(ds_config, f)
scale_factors = [[1, 2, 2], 2, 2]
halos = [[10, 10, 10], [10, 10, 10], [10, 10, 10]]
metadata = {
'unit': 'micrometer',
'resolution': (.25, .2, .2),
'offsets': (0., 0., 0.)
}
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=1,
config_dir=config_dir,
target='local',
input_path='./bdv_mipmap-raw.h5',
input_key='raw',
scale_factors=scale_factors,
halos=halos,
metadata_format='bdv',
metadata_dict=metadata)
success = luigi.build([task], local_scheduler=True)
if success:
with h5py.File('./bdv_mipmap-raw.h5') as f:
ds = f['t00000/s00/3/cells']
raw_ds = ds[:]
print(raw_ds.shape)
view([raw_ds])
def downscale_raw(path, max_jobs=8, target='local'):
""" Downscale raw data.
Arguments:
path [str] - path to raw data
max_jobs [int] - maximum number of jobs
target [str] - target of computation: local, slurm or lsf
"""
# input and output keys
input_key = 'raw'
output_key = 'volumes/raw'
# temporary directories
config_dir = './configs'
tmp_folder = './tmp_downscaling'
os.makedirs(config_dir, exist_ok=True)
# write the global configiration with shebang of python env with
# all necessary dependencies
config = DownscalingWorkflow.get_config()
global_config = config['global']
shebang = '#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env37/bin/python'
global_config.update({'shebang': shebang})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
# write the task specific config
# here, we configure the downscaling task to use skimage
task_config = config['downscaling']
task_config.update({'library': 'skimage'})
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(task_config, f)
scale_factors = [[1, 2, 2], [1, 2, 2], [1, 2, 2], 2]
halos = [[0, 10, 10], [0, 10, 10], [0, 10, 10], [10, 10, 10]]
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=max_jobs,
config_dir=config_dir,
target=target,
input_path=path,
input_key=input_key,
output_key_prefix=output_key,
scale_factors=scale_factors,
halos=halos)
success = luigi.build([task], local_scheduler=True)
assert success, "Dowscaling failed"
def downscale(path,
input_key,
output_key,
scale_factors,
halos,
tmp_folder,
target,
max_jobs,
resolution=None,
library='skimage',
**library_kwargs):
""" Downscale input data to obtain pyramid.
Arguments:
path [str] - path to n5 container with input data and for output data.
input_key [str] - name of input data.
output_key [str] - name of output multi-scale group.
scale_factors [listlike] - factors used for downsampling.
halos [listlike] - halo values used for downsampling.
tmp_folder [str] - folder to store tempory data.
target [str] - computation target, can be 'local', 'slurm' or 'lsf'.
max_jobs [int] - maximum number of jobs used in computation.
resolution [tuple] - resolution of the input data. (default: None)
library [str] - library used for down-sampling (default: 'skimage')
library_kwargs [kwargs] - keyword arguments for the down-scaling function.
"""
assert len(scale_factors) == len(halos)
config_folder = os.path.join(tmp_folder, 'configs')
write_global_config(config_folder)
task = DownscalingWorkflow
configs = task.get_config()
config = configs['downscaling']
config.update({"mem_limit": 12, "time_limit": 120, "library": library})
if library_kwargs:
config.update({"library_kwargs": library_kwargs})
with open(os.path.join(config_folder, "downscaling.config"), 'w') as f:
json.dump(config, f)
metadata = {}
if resolution:
metadata.update({'resolution': resolution})
t = DownscalingWorkflow(tmp_folder=tmp_folder,
config_dir=config_folder,
max_jobs=max_jobs,
target=target,
input_path=path,
input_key=input_key,
output_key_prefix=output_key,
scale_factors=scale_factors,
halos=halos,
metadata_format='paintera',
metadata_dict=metadata)
ret = luigi.build([t], local_scheduler=True)
assert ret, "Conversion to paintera format failed"
def downscale_raw(sample, max_jobs=8, target='local'):
input_path = '/g/kreshuk/data/cremi/realigned/sample%s_small.n5' % sample
input_key = 'raw'
config_dir = './config_ds_raw'
tmp_folder = './tmp_ds_raw_%s' % sample
try:
os.mkdir(config_dir)
except OSError:
pass
config = DownscalingWorkflow.get_config()
global_config = config['global']
shebang = '#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python'
global_config.update({'shebang': shebang})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
task_config = config['downscaling']
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(task_config, f)
scale_factors = [[1, 2, 2], [1, 2, 2], [1, 2, 2], 2]
halos = [[0, 10, 10], [0, 10, 10], [0, 10, 10], [10, 10, 10]]
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=max_jobs,
config_dir=config_dir,
target=target,
input_path=input_path,
input_key='raw/s0',
output_key_prefix='raw',
scale_factors=scale_factors,
halos=halos)
success = luigi.build([task], local_scheduler=True)
#
if success and target == 'local':
with z5py.File(input_path) as f:
ds = f['volumes/raw/s2']
ds.n_threads = 8
data = ds[:]
print(data.shape)
view([data])
def downscale_test():
tmp_folder = './tmp_test'
config_dir = './configs_test'
try:
os.mkdir(config_dir)
except OSError:
pass
config = DownscalingWorkflow.get_config()
global_config = config['global']
global_config.update({
'shebang':
'#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python'
})
global_config.update({
'roi_begin': [50, 500, 500],
'roi_end': [150, 1500, 1500]
})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
scale_factors = [[1, 2, 2], 2, 2]
halos = [[0, 10, 10], [10, 10, 10], [10, 10, 10]]
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=8,
config_dir=config_dir,
target='local',
input_path='./test.n5',
input_key='volumes/raw/s0',
output_key_prefix='volumes/raw',
scale_factors=scale_factors,
halos=halos)
success = luigi.build([task], local_scheduler=True)
if success:
with z5py.File('./test.n5') as f:
ds = f['volumes/raw/s3']
ds.n_threads = 8
raw_ds = ds[:]
print(raw_ds.shape)
view([raw_ds])
def downscale_volume(roi_name, max_jobs=250, target='slurm'):
config_dir = './configs'
if roi_name is None:
roi_begin, roi_end = None, None
tmp_folder = './tmp'
else:
roi_begin, roi_end = get_roi(roi_name)
tmp_folder = './tmp_%s' % roi_name
try:
os.mkdir(config_dir)
except OSError:
pass
config = DownscalingWorkflow.get_config()
global_config = config['global']
global_config.update({
'shebang':
'#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python',
'roi_begin': roi_begin,
'roi_end': roi_end
})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
task_config = config['downscaling']
task_config.update({'time_limit': 120, 'mem_limit': 3})
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(task_config, f)
scale_factors = [[1, 2, 2], 2, 2, 2, 2]
halos = [[0, 10, 10], [10, 10, 10], [10, 10, 10], [10, 10, 10],
[10, 10, 10]]
# scale_factors = [[1, 2, 2]]
# halos = [[0, 10, 10]]
path = '/g/kreshuk/data/arendt/platyneris_v1/data.n5'
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=max_jobs,
config_dir=config_dir,
target=target,
input_path=path,
input_key='volumes/raw/s0',
output_key_prefix='volumes/raw',
scale_factors=scale_factors,
halos=halos)
success = luigi.build([task], local_scheduler=True)
view_ = False
if view_ and success and target == 'local':
sfa = [2, 4, 4]
roi_begin = tuple(roib // sf for roib, sf in zip(roi_begin, sfa))
roi_end = tuple(roie // sf for roie, sf in zip(roi_end, sfa))
bb = tuple(slice(roib, roie) for roib, roie in zip(roi_begin, roi_end))
print(bb)
with z5py.File(path) as f:
ds = f['volumes/raw/s2']
ds.n_threads = 8
data = ds[bb]
print(data.shape)
view([data])
def downscale_seg(sample, max_jobs=8, target='local'):
input_path = '/g/kreshuk/data/cremi/realigned/sample%s_small.n5' % sample
input_key = 'segmentation/multicut/s0'
config_dir = './config_ds_seg'
tmp_folder = './tmp_ds_seg_%s' % sample
try:
os.mkdir(config_dir)
except OSError:
pass
config = DownscalingWorkflow.get_config()
global_config = config['global']
shebang = '#! /g/kreshuk/pape/Work/software/conda/miniconda3/envs/cluster_env/bin/python'
global_config.update({'shebang': shebang})
with open(os.path.join(config_dir, 'global.config'), 'w') as f:
json.dump(global_config, f)
ds_config = config['downscaling']
# FIXME majority vote downscaling is broken
# ds_config.update({'library': 'skimage', 'threads_per_job': 8})
ds_config.update({
'library': 'vigra',
'library_kwargs': {
'order': 0
},
'threads_per_job': 8
})
with open(os.path.join(config_dir, 'downscaling.config'), 'w') as f:
json.dump(ds_config, f)
scale_factors = [[1, 2, 2], [1, 2, 2], [1, 2, 2], 2]
halos = [[0, 10, 10], [0, 10, 10], [0, 10, 10], [10, 10, 10]]
task = DownscalingWorkflow(tmp_folder=tmp_folder,
max_jobs=1,
config_dir=config_dir,
target='local',
input_path=input_path,
input_key='segmentation/multicut/s0',
output_key_prefix='segmentation/multicut',
scale_factors=scale_factors,
halos=halos)
success = luigi.build([task], local_scheduler=True)
#
if success and target == 'local':
with z5py.File(input_path) as f:
#
ds = f['raw/s2']
ds.n_threads = 8
raw = ds[:]
rshape = raw.shape
#
ds = f['segmentation/multicut/s2']
ds.n_threads = 8
seg = ds[:]
mshape = seg.shape
assert mshape == rshape, "%s %s" % (str(mshape), str(rshape))
view([raw, seg])
def downscale(in_path,
in_key,
out_path,
resolution,
scale_factors,
chunks,
tmp_folder,
target,
max_jobs,
block_shape,
library="vigra",
library_kwargs=None,
metadata_format="bdv.n5",
out_key="",
unit="micrometer",
source_name=None,
roi_begin=None,
roi_end=None,
int_to_uint=False,
channel=None):
task = DownscalingWorkflow
block_shape = chunks if block_shape is None else block_shape
config_dir = os.path.join(tmp_folder, "configs")
# ome.zarr can also be written in 2d, all other formats require 3d
require3d = metadata_format != "ome.zarr"
check_input_data(in_path, in_key, resolution, require3d, channel)
write_global_config(config_dir,
block_shape=block_shape,
require3d=require3d,
roi_begin=roi_begin,
roi_end=roi_end)
configs = DownscalingWorkflow.get_config()
conf = configs["copy_volume"]
conf.update({"chunks": chunks, "time_limit": 600})
with open(os.path.join(config_dir, "copy_volume.config"), "w") as f:
json.dump(conf, f)
ds_conf = configs["downscaling"]
ds_conf.update({"chunks": chunks, "library": library, "time_limit": 600})
if library_kwargs is not None:
ds_conf.update({"library_kwargs": library_kwargs})
with open(os.path.join(config_dir, "downscaling.config"), "w") as f:
json.dump(ds_conf, f)
halos = scale_factors
metadata_dict = {
"resolution": resolution,
"unit": unit,
"setup_name": source_name
}
t = task(tmp_folder=tmp_folder,
config_dir=config_dir,
target=target,
max_jobs=max_jobs,
input_path=in_path,
input_key=in_key,
scale_factors=scale_factors,
halos=halos,
metadata_format=metadata_format,
metadata_dict=metadata_dict,
output_path=out_path,
output_key_prefix=out_key,
int_to_uint=int_to_uint)
ret = luigi.build([t], local_scheduler=True)
if not ret:
raise RuntimeError("Downscaling failed")