def process_block(image_in, ndim, blockreduce, func, blockoffset, blocksize,
margin, fullsize, mo, is_labelimage, relabel, neighbourmerge,
save_fwmap, maxlabel, mpi):
"""Write a block of data into a hdf5 file."""
# open data for reading
im = Image(image_in, permission='r')
im.load(mpi.comm, load_data=False)
# get the indices into the input and output datasets
# TODO: get indices from attributes
# TODO: get from mpi.get_blocks
set_slices_in_and_out(im, mo, blocksize, margin, fullsize)
# simply copy the data from input to output
"""NOTE:
it is assumed that the inputs are not 4D labelimages
"""
if ndim == 4:
mo.write(im.slice_dataset())
im.close()
return
if ((not is_labelimage)
or ((not relabel) and (not neighbourmerge) and (not blockreduce))):
data = im.slice_dataset()
#datatype = 'uint16'
#from skimage.util.dtype import convert
#data = convert(data, np.dtype(datatype), force_copy=False)
mo.write(data)
im.close()
return
# forward map to relabel the blocks in the output
if relabel:
# FIXME: make sure to get all data in the block
fw, maxlabel = relabel_block(im.ds[:], maxlabel, mpi)
if save_fwmap:
comps = im.split_path()
fpath = '{}_{}.npy'.format(comps['base'], comps['int'][1:])
np.save(fpath, fw)
if (not neighbourmerge) and (not blockreduce):
data = im.slice_dataset()
mo.write(fw[data])
im.close()
return
else:
ulabels = np.unique(im.ds[:])
fw = [l for l in range(0, np.amax(ulabels) + 1)]
fw = np.array(fw)
# blockwise reduction of input datasets
if blockreduce is not None:
pass
else:
data = im.slice_dataset()
# merge overlapping labels
fw = merge_overlap(fw, im, mo, data, margin)
mo.write(fw[data])
im.close()
def add_features(df, image_in='', origin=[0, 0, 0], dsfacs=[1, 16, 16]):
if 'centroid-0' in df.columns:
cens = ['centroid-{}'.format(i) for i in [0, 1, 2]]
coms = ['com_{}'.format(d) for d in 'zyx']
df[coms] = df[cens] + origin
if image_in:
dt_im = Image(image_in, permission='r')
dt_im.load(load_data=False)
data = dt_im.slice_dataset()
dt_im.close()
ds_centroid = np.array(df[coms] / dsfacs, dtype='int')
ds_centroid = [data[p[0], p[1], p[2]] for p in ds_centroid]
df['dist_to_edge'] = np.array(ds_centroid)
if 'inertia_tensor_eigvals-0' in df.columns:
ites = ['inertia_tensor_eigvals-{}'.format(i) for i in [0, 1, 2]]
eigvals = np.clip(np.array(df[ites]), 0, np.inf)
df['fractional_anisotropy'] = fractional_anisotropy(eigvals)
df['major_axis_length'] = get_ellips_axis_lengths(eigvals[:, 0])
df['minor_axis_length'] = get_ellips_axis_lengths(eigvals[:, -1])
# TODO: range, variance, ...
return df
def get_centreslice(image_in, ids, dim='z', ch=0):
"""Return an image's centreslice for a dimension."""
if isinstance(image_in, Image):
im = image_in
else:
im = Image('{}/{}'.format(image_in, ids))
try:
im.load(load_data=False)
except KeyError:
print('dataset {} not found'.format(ids))
return None
if len(im.dims) > 3:
ch_idx = im.axlab.index('c')
im.slices[ch_idx] = slice(ch, ch + 1, 1)
dim_idx = im.axlab.index(dim)
cslc = int(im.dims[dim_idx] / 2)
slcs = [slc for slc in im.slices]
im.slices[dim_idx] = slice(cslc, cslc + 1, 1)
data = im.slice_dataset()
im.slices = slcs
return data
def downsample_channel(image_in,
ch,
resolution_level=-1,
dsfacs=[1, 4, 4, 1, 1],
ismask=False,
output='',
report={}):
"""Downsample an image."""
ods = 'data' if not ismask else 'mask'
# return in case no mask provided
if not image_in:
return None, report, ''
if resolution_level != -1 and not ismask: # we should have an Imaris pyramid
image_in = '{}/DataSet/ResolutionLevel {}'.format(
image_in, resolution_level)
# load data
im = Image(image_in, permission='r')
im.load(load_data=False)
props = im.get_props()
if len(im.dims) > 4:
im.slices[im.axlab.index('t')] = slice(0, 1, 1)
props = im.squeeze_props(props, dim=4)
if len(im.dims) > 3:
im.slices[im.axlab.index('c')] = slice(ch, ch + 1, 1)
props = im.squeeze_props(props, dim=3)
data = im.slice_dataset()
im.close()
# downsample
dsfac = tuple(dsfacs[:len(data.shape)])
if not ismask:
data = downscale_local_mean(data, dsfac).astype('float32')
else:
data = block_reduce(data, dsfac, np.max)
# generate output
props['axlab'] = 'zyx' # FIXME: axlab returns as string-list
props['shape'] = data.shape
props['elsize'] = [es * ds for es, ds in zip(im.elsize[:3], dsfac)]
props['slices'] = None
mo = write_data(data, props, output, ods)
# report data
thr = 1000
meds_mask = data < thr
report['medians'][ods] = get_zyx_medians(data, meds_mask)
c_slcs = {dim: get_centreslice(mo, '', dim) for dim in 'zyx'}
report['centreslices'][ods] = c_slcs
return mo, report, meds_mask
def apply_field(image_in, bias_in, outputpath):
im = Image(image_in, permission='r+')
im.load(load_data=True)
bf = Image(bias_in, permission='r+')
bf.load(load_data=True)
corr = divide_bias_field(im, bf, outputpath)
im.close()
bf.close()
corr.close()
def extract_resolution_level(image_in, resolution_level, output='', report={}):
"""Extract data from a 5D Imaris image pyramid and create 4D image."""
if resolution_level != -1:
image_in = '{}/DataSet/ResolutionLevel {}'.format(image_in, resolution_level)
mo = Image(image_in, permission='r')
mo.load(load_data=False)
report['parameters']['n_channels'] = mo.dims[mo.axlab.index('c')]
return mo, report
def get_maxlabels_from_attribute(filelist, ids, maxlabelfile):
maxlabels = []
for datafile in filelist:
image_in = '{}/{}'.format(datafile, ids)
im = Image(image_in, permission='r')
im.load(load_data=False)
maxlabels.append(im.ds.attrs['maxlabel'])
im.close()
if maxlabelfile:
np.savetxt(maxlabelfile, maxlabels, fmt='%d')
return maxlabels
def get_data(h5_path, ids, ch=0, dim=''):
im = Image('{}/{}'.format(h5_path, ids))
im.load(load_data=False)
if dim:
dim_idx = im.axlab.index(dim)
cslc = int(im.dims[dim_idx] / 2)
im.slices[dim_idx] = slice(cslc, cslc + 1, 1)
if len(im.dims) > 3:
im.slices[im.axlab.index('c')] = slice(ch, ch + 1, 1)
data = im.slice_dataset()
return data
def get_centreslices(info_dict, idss=[], ch=0):
"""Get the centreslices for all dims and steps."""
h5_path = info_dict['paths']['file']
image_in = h5_path + info_dict['paths']['int']
im = Image(image_in)
im.load(load_data=False)
if not idss:
idss = [k for k in im.file.keys()]
im.close()
centreslices = {
ids: {dim: get_centreslice(h5_path, ids, dim, ch)
for dim in 'zyx'}
for ids in idss
}
return centreslices
def get_info_dict(image_in, info_dict={}, report_type='bfc', channel=0):
im = Image(image_in)
im.load(load_data=False)
info_dict['elsize'] = {dim: im.elsize[i] for i, dim in enumerate(im.axlab)}
info_dict['paths'] = im.split_path() # FIXME: out_base
info_dict['paths']['out_base'] = info_dict['paths']['base']
im.close()
ppath = '{}.pickle'.format(info_dict['paths']['base'])
with open(ppath, 'rb') as f:
info_dict['parameters'] = pickle.load(f)
info_dict['centreslices'] = get_centreslices(info_dict)
# info_dict['medians'] = get_medians(info_dict)
info_dict['medians'], info_dict['means'], info_dict['stds'], info_dict[
'n_samples'] = get_means_and_stds(info_dict, ch=channel, thr=1000)
return info_dict
def biasfield_correction(
image_in,
parameter_file='',
outputdir='',
channels=[],
mask_in='',
resolution_level=-1,
downsample_factors=[1, 1, 1, 1, 1],
n_iterations=50,
n_fitlevels=4,
n_bspline_cps=[5, 5, 5],
):
"""Perform N4 bias field correction."""
params = get_params(locals(), parameter_file, 'biasfield_correction')
if not params['channels']:
im = Image(image_in, permission='r')
im.load()
n_channels = im.dims[im.axlab.index('c')]
params['channels'] = list(range(n_channels))
im.close()
n_workers = get_n_workers(len(params['channels']), params)
arglist = [(
image_in,
ch,
params['mask_in'],
params['resolution_level'],
params['downsample_factors'],
params['n_iterations'],
params['n_fitlevels'],
params['n_bspline_cps'],
outputdir,
) for ch in params['channels']]
with multiprocessing.Pool(processes=n_workers) as pool:
pool.starmap(estimate_biasfield, arglist)
def stack_channels(images_in, outputpath=''):
channels = []
for image_in in images_in:
im = Image(image_in, permission='r')
im.load(load_data=True)
channels.append(im.ds[:])
data = np.stack(channels, axis=3)
mo = Image(outputpath)
mo.elsize = list(im.elsize) + [1]
mo.axlab = im.axlab + 'c'
mo.dims = data.shape
mo.chunks = list(im.chunks) + [1]
mo.dtype = im.dtype
mo.set_slices()
if outputpath:
mo.create()
mo.write(data)
mo.close()
return mo
def get_paths(image_in,
resolution_level=-1,
channel=0,
outputstem='',
step='',
save_steps=False):
"""Get the parameters for the preprocessing step."""
# get paths from input file
if resolution_level != -1: # we should have an Imaris pyramid
image_in = '{}/DataSet/ResolutionLevel {}'.format(
image_in, resolution_level)
im = Image(image_in, permission='r')
im.load(load_data=False)
paths = im.split_path()
im.close()
# define output basename
paths['out_base'] = outputstem
# define h5 output template
paths['out_h5'] = '{}.h5/{}'.format(paths['out_base'], '{}')
# define output for main results and intermediate steps
if not outputstem:
paths['main'] = paths['steps'] = ''
else:
### FIXME ????????????? what is this?
if save_steps:
paths['main'] = paths['steps'] = paths['out_h5']
else:
paths['main'] = paths['out_h5']
paths['steps'] = paths['out_h5']
# define output for parameters
paths['params'] = '{}.pickle'.format(paths['out_base'], step)
return paths
def apply_bias_field_full(image_in,
bias_in,
dsfacs=[1, 64, 64, 1],
in_place=False,
write_to_single_file=False,
blocksize_xy=1280,
outputpath='',
channel=None):
"""single-core in ~200 blocks"""
perm = 'r+' if in_place else 'r'
im = Image(image_in, permission=perm)
im.load(load_data=False)
bf = Image(bias_in, permission='r')
bf.load(load_data=False)
if channel is not None:
im.slices[3] = slice(channel, channel + 1)
if write_to_single_file: # assuming single-channel copied file here
mo = Image(outputpath)
mo.load()
mo.slices[3] = slice(0, 1, 1)
mpi = wmeMPI(usempi=False)
mpi_nm = wmeMPI(usempi=False)
if blocksize_xy:
blocksize = [im.dims[0], blocksize_xy, blocksize_xy, 1, 1]
blockmargin = [0, im.chunks[1], im.chunks[2], 0, 0]
else:
blocksize = im.dims[:3] + [1, 1]
blockmargin = [0] * len(im.dims)
mpi.set_blocks(im, blocksize, blockmargin)
mpi_nm.set_blocks(im, blocksize)
mpi.scatter_series()
for i in mpi.series:
print(i)
block = mpi.blocks[i]
data_shape = list(im.slices2shape(block['slices']))
block_nm = mpi_nm.blocks[i]
it = zip(block['slices'], block_nm['slices'], blocksize, data_shape)
data_shape = list(im.slices2shape(block_nm['slices']))
data_slices = []
for b_slc, n_slc, bs, ds in it:
m_start = n_slc.start - b_slc.start
m_stop = m_start + bs
m_stop = min(m_stop, ds)
data_slices.append(slice(m_start, m_stop, None))
data_slices[3] = block['slices'][3]
data_shape = list(im.slices2shape(data_slices))
# get the fullres image block
im.slices = block['slices']
data = im.slice_dataset().astype('float')
# get the upsampled bias field
bias = get_bias_field_block(bf, im.slices, data.shape)
data /= bias
data = np.nan_to_num(data, copy=False)
if in_place:
im.slices = block_nm['slices']
data = data[tuple(data_slices[:3])].astype(im.dtype)
im.write(data)
elif write_to_single_file:
mo.slices = block_nm['slices']
mo.slices[3] = slice(0, 1, 1)
data = data[tuple(data_slices[:3])].astype(mo.dtype)
mo.write(data)
else:
props = im.get_props()
if len(im.dims) > 4:
props = im.squeeze_props(props, dim=4)
if len(im.dims) > 3:
props = im.squeeze_props(props, dim=3)
props['axlab'] = 'zyx' # FIXME: axlab return as string-list
props['shape'] = bias.shape
props['slices'] = None
props['dtype'] = bias.dtype
mo = Image(block['path'], **props) # FIXME: needs channel
mo.create(comm=mpi.comm)
mo.slices = None
mo.set_slices()
mo.write(data=bias)
mo.close()
im.close()
bf.close()
def csv_to_im(
image_in,
csv_path,
labelkey='label',
key='dapi',
name='',
maxlabel=0,
normalize=False,
scale_uint16=False,
replace_nan=False,
channel=-1,
outpath='',
):
"""Write segment backprojection."""
if isinstance(image_in, Image):
labels = image_in
else:
labels = LabelImage(image_in)
labels.load(load_data=False)
if not maxlabel:
labels.set_maxlabel()
maxlabel = labels.maxlabel
if csv_path.endswith('.csv'):
df = pd.read_csv(csv_path)
df = df.astype({labelkey: int})
elif csv_path.endswith('.h5ad'):
import scanpy as sc
adata = sc.read(csv_path)
if not csv_path.endswith('_nofilter.h5ad'):
adata.X = adata.raw.X
df = adata.obs[labelkey].astype(int)
df = pd.concat([df, adata[:, key].to_df()], axis=1)
# for key in keys: # TODO
fw = np.zeros(maxlabel + 1, dtype='float')
for index, row in df.iterrows():
fw[int(row[labelkey])] = row[key]
if replace_nan:
fw = np.nan_to_num(fw)
if normalize:
def normalize_data(data):
"""Normalize data between 0 and 1."""
data = data.astype('float64')
datamin = np.amin(data)
datamax = np.amax(data)
data -= datamin
data *= 1 / (datamax - datamin)
return data, [datamin, datamax]
fw_n, fw_minmax = normalize_data(fw)
fw_n *= 65535
fw = fw_n
elif scale_uint16: # for e.g. pseudotime / FA / etc / any [0, 1] vars
fw *= 65535
out = labels.forward_map(list(fw))
if outpath.endswith('.ims'):
mo = Image(outpath, permission='r+')
mo.load(load_data=False)
if channel >= 0 and channel < mo.dims[3]:
ch = channel
else:
mo.create()
ch = mo.dims[3] - 1
mo.slices[3] = slice(ch, ch + 1, 1)
mo.write(out.astype(mo.dtype)) # FIXME: >65535 wraps around
cpath = 'DataSetInfo/Channel {}'.format(ch)
name = name or key
mo.file[cpath].attrs['Name'] = np.array([c for c in name], dtype='|S1')
mo.close()
elif outpath.endswith('.nii.gz'):
props = labels.get_props()
if not labels.path.endswith('.nii.gz'):
props = transpose_props(props, outlayout='xyz')
out = out.transpose()
mo = write_output(outpath, out, props)
else:
outpath = outpath or gen_outpath(labels, key)
mo = write_output(outpath, out, labels.get_props())
return mo
def cell_segmentation(
plan_path,
memb_path,
dapi_path,
mean_path,
dapi_shift_planes=0,
nucl_opening_footprint=[3, 7, 7],
dapi_filter='median',
dapi_sigma=1,
dapi_dog_sigma1=2,
dapi_dog_sigma2=4,
dapi_thr=0,
sauvola_window_size=[19, 75, 75],
sauvola_k=0.2,
dapi_absmin=500,
dapi_erodisk=0,
dist_max=5,
peaks_size=[11, 19, 19],
peaks_thr=1.0,
peaks_dil_footprint=[3, 7, 7],
compactness=0.80,
memb_filter='median',
memb_sigma=3,
planarity_thr=0.0005,
dset_mask_filter='gaussian',
dset_mask_sigma=50,
dset_mask_thr=1000,
steps=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
outputstem='',
save_steps=False,
):
step = 'segment'
paths = get_paths(plan_path, -1, 0, outputstem, step, save_steps)
report = {
'parameters': locals(),
'paths': paths,
'medians': {},
'centreslices': {}
}
# load images
im_dapi = Image(dapi_path)
im_dapi.load()
nucl_props = im_dapi.get_props()
im_memb = MaskImage(memb_path)
im_memb.load()
memb_props = im_memb.get_props()
im_plan = MaskImage(plan_path)
im_plan.load()
# im_dset_mask = Image(dset_mask_path, permission='r')
# im_dset_mask.load(load_data=False)
im_mean = Image(mean_path)
im_mean.load()
# preprocess dapi channel
# .h5/nucl/dapi<_shifted><_opened><_preprocess>
stage = 'nucleus channel'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/nucl/dapi')
if 0 not in steps:
op = 'reading'
im_dapi_pp = get_image('{}{}'.format(outstem, '_preprocess'))
else:
op = 'processing'
im_dapi_pp = preprocess_nucl(
im_dapi,
dapi_shift_planes,
dapi_filter,
dapi_sigma,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# create a nuclear mask from the dapi channel
# .h5/nucl/dapi<_mask_thr><_sauvola><_mask><_mask_ero>
stage = 'nucleus mask'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/nucl/dapi')
if 1 not in steps:
op = 'reading'
im_dapi_mask = get_image('{}{}'.format(outstem, '_mask_ero'))
else:
op = 'processing'
im_dapi_mask = create_nuclear_mask(
im_dapi_pp,
dapi_thr,
sauvola_window_size,
sauvola_k,
dapi_absmin,
dapi_erodisk,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# create a membrane mask from the membrane mean
# .h5/memb/planarity<_mask>
stage = 'membrane mask'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/memb/planarity')
if 2 not in steps:
op = 'reading'
im_memb_mask = get_image('{}{}'.format(outstem, '_mask'))
else:
op = 'processing'
im_memb_mask = create_membrane_mask(
im_plan,
planarity_thr,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# combine nuclear and membrane mask
# .h5/segm/seeds<_mask>
stage = 'mask combination'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/segm/seeds')
if 3 not in steps:
op = 'reading'
im_nucl_mask = get_image('{}{}'.format(outstem, '_mask'))
else:
op = 'processing'
im_nucl_mask = combine_nucl_and_memb_masks(
im_memb_mask,
im_dapi_mask,
nucl_opening_footprint,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# find seeds for watershed
stage = 'nucleus detection'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/segm/seeds')
if 4 not in steps:
op = 'reading'
im_dt = get_image('{}{}'.format(outstem, '_edt'))
im_peaks = get_image('{}{}'.format(outstem, '_peaks'))
# .h5/segm/seeds<_edt><_mask_distmax><_peaks><_peaks_dil>
else:
op = 'processing'
im_dt, im_peaks = define_seeds(
im_nucl_mask,
im_memb_mask,
im_dapi_pp,
dapi_dog_sigma1,
dapi_dog_sigma2,
dist_max,
peaks_size,
peaks_thr,
peaks_dil_footprint,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# preprocess membrane mean channel
# .h5/memb/preprocess<_smooth>
stage = 'membrane channel'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/memb/mean')
if 5 not in steps:
op = 'reading'
im_memb_pp = get_image('{}{}'.format(outstem, '_smooth'))
else:
op = 'processing'
im_memb_pp = preprocess_memb(
im_memb,
memb_filter,
memb_sigma,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# perform watershed from the peaks to fill the nuclei
# .h5/segm/labels<_edt><_memb>
stage = 'watershed'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/segm/labels')
if 6 not in steps:
op = 'reading'
im_ws = get_image('{}{}'.format(outstem, '_memb'), imtype='Label')
else:
op = 'processing'
im_ws = perform_watershed(
im_peaks,
im_memb_pp,
im_dt,
peaks_thr,
memb_sigma,
memb_filter,
compactness,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# generate a dataset mask from the mean of all channels
# .h5/mean<_smooth><_mask>
stage = 'dataset mask'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/mean')
if 7 not in steps:
op = 'reading'
im_dset_mask = get_image('{}{}'.format(outstem, '_mask'),
imtype='Mask')
else:
op = 'processing'
im_dset_mask = create_dataset_mask(
im_mean,
filter=dset_mask_filter,
sigma=dset_mask_sigma,
threshold=dset_mask_thr,
outstem=outstem,
save_steps=save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# filter the segments with the dataset mask
# .h5/segm/labels<_memb_del>
# .h5/mask
stage = 'segment filter'
t = time.time()
outstem = '{}.h5{}'.format(outputstem, '/segm/labels')
if 8 not in steps:
im_ws_pp = get_image('{}{}'.format(outstem, '_memb_del'),
imtype='Label')
else:
op = 'processing'
im_ws_pp = segmentation_postprocessing(
im_dset_mask,
im_ws,
outstem,
save_steps,
)
elapsed = time.time() - t
print('{} ({}) took {:1f} s'.format(stage, op, elapsed))
# write report
generate_report('{}.h5/{}'.format(outputstem, 'mean_mask'))
return im_ws_pp
def export_regionprops(
seg_paths,
seg_names=['full', 'memb', 'nucl'],
data_paths=[],
data_names=[],
aux_data_path=[],
downsample_factors=[1, 1, 1],
outputstem='',
blocksize=[],
blockmargin=[],
blockrange=[],
channels=[],
filter_borderlabels=False,
min_labelsize=0,
split_features=False,
fset_morph=['label'],
fset_intens=['mean_intensity'],
fset_addit=['com_z', 'com_y', 'com_x'],
):
# load the segments: ['full'] or ['full', 'memb', 'nucl']
label_ims = {}
pfs = seg_names[:len(seg_paths)]
for pf, seg_path in zip(pfs, seg_paths):
im = LabelImage(seg_path, permission='r')
im.load(load_data=False)
label_ims[pf] = im
comps = label_ims['full'].split_path()
# prepare parallel processing
mpi_label = wmeMPI(usempi=False)
blocksize = blocksize or label_ims['full'].dims
mpi_label.set_blocks(label_ims['full'], blocksize, blockmargin, blockrange)
mpi_label.scatter_series()
# load the data
data_ims = {}
mpi_data = wmeMPI(usempi=False)
for i, data_path in enumerate(data_paths):
pf = 'im{:02d}'.format(i)
data = Image(data_path, permission='r')
data.load(load_data=False)
ch_idx = data.axlab.index('c')
# FIXME channels for multiple data_paths
chs = channels or [ch for ch in range(data.dims[ch_idx])]
names = [data_names.pop(0) for _ in range(len(chs))]
data_ims[pf] = {'im': data, 'ch': chs, 'names': names}
""" TODO
try:
mpi_data.blocks = [
{'id': split_filename(comps['file'])[0]['postfix'],
'slices': dset_name2slices(comps['file'], axlab=data.axlab, shape=data.dims),
'path': '',},
]
except:
"""
mpi_data.set_blocks(data, blocksize, blockmargin, blockrange)
border_labelset = set([])
# if filter_borderlabels:
# outstem = outputstem or label_ims['full'].split_path()['base']
# outstem += '_dataset'
# border_labelset |= filter_borders(label_ims['full'], outstem)
dfs = []
for i in mpi_label.series:
print('processing block {:03d} with id: {}'.format(
i, mpi_label.blocks[i]['id']))
dfs.append(
process_block(
mpi_label.blocks[i],
mpi_data.blocks[i],
label_ims,
split_features,
data_ims,
min_labelsize,
channels,
filter_borderlabels,
fset_morph,
fset_intens,
fset_addit,
border_labelset,
outputstem,
aux_data_path,
downsample_factors,
))
return dfs
def mergeblocks(
images_in,
dataslices=None,
blocksize=[],
blockmargin=[],
blockrange=[],
blockoffset=[0, 0, 0],
fullsize=[],
is_labelimage=False,
relabel=False,
neighbourmerge=False,
save_fwmap=False,
blockreduce=[],
func='np.amax',
datatype='',
usempi=False,
outputpath='',
save_steps=False,
protective=False,
):
"""Merge blocks of data into a single hdf5 file."""
if blockrange:
images_in = images_in[blockrange[0]:blockrange[1]]
mpi = wmeMPI(usempi)
im = Image(images_in[0], permission='r')
im.load(mpi.comm, load_data=False)
props = im.get_props(protective=protective, squeeze=True)
ndim = im.get_ndim()
props['dtype'] = datatype or props['dtype']
props['chunks'] = props['chunks'] or None
# get the size of the outputfile
# TODO: option to derive fullsize from dset_names?
if blockreduce:
datasize = np.subtract(fullsize, blockoffset)
outsize = [
int(np.ceil(d / np.float(b)))
for d, b in zip(datasize, blockreduce)
]
props['elsize'] = [e * b for e, b in zip(im.elsize, blockreduce)]
else: # FIXME: 'zyx(c)' stack assumed
outsize = np.subtract(fullsize, blockoffset)
if ndim == 4:
outsize = list(outsize) + [im.ds.shape[3]] # TODO: flexible insert
if outputpath.endswith('.ims'):
mo = LabelImage(outputpath)
mo.create(comm=mpi.comm)
else:
props['shape'] = outsize
mo = LabelImage(outputpath, **props)
mo.create(comm=mpi.comm)
mpi.blocks = [{'path': image_in} for image_in in images_in]
mpi.nblocks = len(images_in)
mpi.scatter_series()
# merge the datasets
maxlabel = 0
for i in mpi.series:
block = mpi.blocks[i]
# try:
maxlabel = process_block(block['path'], ndim, blockreduce, func,
blockoffset, blocksize, blockmargin, fullsize,
mo, is_labelimage, relabel, neighbourmerge,
save_fwmap, maxlabel, mpi)
print('processed block {:03d}: {}'.format(i, block['path']))
# except Exception as e:
# print('failed block {:03d}: {}'.format(i, block['path']))
# print(e)
im.close()
mo.close()
return mo