def add_features(df, image_in='', origin=[0, 0, 0], dsfacs=[1, 16, 16]):
if 'centroid-0' in df.columns:
cens = ['centroid-{}'.format(i) for i in [0, 1, 2]]
coms = ['com_{}'.format(d) for d in 'zyx']
df[coms] = df[cens] + origin
if image_in:
dt_im = Image(image_in, permission='r')
dt_im.load(load_data=False)
data = dt_im.slice_dataset()
dt_im.close()
ds_centroid = np.array(df[coms] / dsfacs, dtype='int')
ds_centroid = [data[p[0], p[1], p[2]] for p in ds_centroid]
df['dist_to_edge'] = np.array(ds_centroid)
if 'inertia_tensor_eigvals-0' in df.columns:
ites = ['inertia_tensor_eigvals-{}'.format(i) for i in [0, 1, 2]]
eigvals = np.clip(np.array(df[ites]), 0, np.inf)
df['fractional_anisotropy'] = fractional_anisotropy(eigvals)
df['major_axis_length'] = get_ellips_axis_lengths(eigvals[:, 0])
df['minor_axis_length'] = get_ellips_axis_lengths(eigvals[:, -1])
# TODO: range, variance, ...
return df
def generate_report(image_in, info_dict={}, ioff=True):
"""Generate a QC report of the segmentation process."""
report_type = 'reseg'
# Turn interactive plotting on/off.
if ioff:
plt.ioff()
else:
plt.ion()
# Get paths from image if info_dict not provided.
if not info_dict:
im = Image(image_in)
info_dict['paths'] = im.split_path() # FIXME: out_base
info_dict['paths']['out_base'] = info_dict['paths']['base']
im.close()
info_dict['parameters'] = get_parameters(info_dict, report_type)
# info_dict['centreslices'] = get_centreslices(info_dict, idss=[])
info_dict['medians'] = {}
# Create the axes.
figsize = (18, 9)
gridsize = (1, 4)
f = plt.figure(figsize=figsize, constrained_layout=False)
gs0 = gridspec.GridSpec(gridsize[0], gridsize[1], figure=f)
# axs = [gen_orthoplot(f, gs0[j, i]) for j in range(0, 2) for i in range(0, 4)]
axs = [
gen_orthoplot(f, gs0[j, i]) for j in range(0, gridsize[0])
for i in range(0, gridsize[1])
]
# Plot the images and graphs. vmaxs = [15000] + [5000] * 7
info_dict['plotinfo'] = {'vmax': 10000}
plot_images(axs, info_dict)
# Add annotations and save as pdf.
reseg_id = 'axis{:01d}-seam{:02d}-j{:03d}'.format(info_dict['axis'],
info_dict['seam'],
info_dict['j'])
header = 'mLSR-3D Quality Control'
figtitle = '{}: {} \n {}'.format(
header,
report_type,
reseg_id,
)
figpath = '{}_{}_{}-report.pdf'.format(
info_dict['paths']['out_base'],
report_type,
reseg_id,
)
print('writing report to {}'.format(figpath))
f.suptitle(figtitle, fontsize=14, fontweight='bold')
add_titles(axs, info_dict)
f.savefig(figpath)
plt.close(f)
def downsample_channel(image_in,
ch,
resolution_level=-1,
dsfacs=[1, 4, 4, 1, 1],
ismask=False,
output='',
report={}):
"""Downsample an image."""
ods = 'data' if not ismask else 'mask'
# return in case no mask provided
if not image_in:
return None, report, ''
if resolution_level != -1 and not ismask: # we should have an Imaris pyramid
image_in = '{}/DataSet/ResolutionLevel {}'.format(
image_in, resolution_level)
# load data
im = Image(image_in, permission='r')
im.load(load_data=False)
props = im.get_props()
if len(im.dims) > 4:
im.slices[im.axlab.index('t')] = slice(0, 1, 1)
props = im.squeeze_props(props, dim=4)
if len(im.dims) > 3:
im.slices[im.axlab.index('c')] = slice(ch, ch + 1, 1)
props = im.squeeze_props(props, dim=3)
data = im.slice_dataset()
im.close()
# downsample
dsfac = tuple(dsfacs[:len(data.shape)])
if not ismask:
data = downscale_local_mean(data, dsfac).astype('float32')
else:
data = block_reduce(data, dsfac, np.max)
# generate output
props['axlab'] = 'zyx' # FIXME: axlab returns as string-list
props['shape'] = data.shape
props['elsize'] = [es * ds for es, ds in zip(im.elsize[:3], dsfac)]
props['slices'] = None
mo = write_data(data, props, output, ods)
# report data
thr = 1000
meds_mask = data < thr
report['medians'][ods] = get_zyx_medians(data, meds_mask)
c_slcs = {dim: get_centreslice(mo, '', dim) for dim in 'zyx'}
report['centreslices'][ods] = c_slcs
return mo, report, meds_mask
def apply_field(image_in, bias_in, outputpath):
im = Image(image_in, permission='r+')
im.load(load_data=True)
bf = Image(bias_in, permission='r+')
bf.load(load_data=True)
corr = divide_bias_field(im, bf, outputpath)
im.close()
bf.close()
corr.close()
def process_block(image_in, ndim, blockreduce, func, blockoffset, blocksize,
margin, fullsize, mo, is_labelimage, relabel, neighbourmerge,
save_fwmap, maxlabel, mpi):
"""Write a block of data into a hdf5 file."""
# open data for reading
im = Image(image_in, permission='r')
im.load(mpi.comm, load_data=False)
# get the indices into the input and output datasets
# TODO: get indices from attributes
# TODO: get from mpi.get_blocks
set_slices_in_and_out(im, mo, blocksize, margin, fullsize)
# simply copy the data from input to output
"""NOTE:
it is assumed that the inputs are not 4D labelimages
"""
if ndim == 4:
mo.write(im.slice_dataset())
im.close()
return
if ((not is_labelimage)
or ((not relabel) and (not neighbourmerge) and (not blockreduce))):
data = im.slice_dataset()
#datatype = 'uint16'
#from skimage.util.dtype import convert
#data = convert(data, np.dtype(datatype), force_copy=False)
mo.write(data)
im.close()
return
# forward map to relabel the blocks in the output
if relabel:
# FIXME: make sure to get all data in the block
fw, maxlabel = relabel_block(im.ds[:], maxlabel, mpi)
if save_fwmap:
comps = im.split_path()
fpath = '{}_{}.npy'.format(comps['base'], comps['int'][1:])
np.save(fpath, fw)
if (not neighbourmerge) and (not blockreduce):
data = im.slice_dataset()
mo.write(fw[data])
im.close()
return
else:
ulabels = np.unique(im.ds[:])
fw = [l for l in range(0, np.amax(ulabels) + 1)]
fw = np.array(fw)
# blockwise reduction of input datasets
if blockreduce is not None:
pass
else:
data = im.slice_dataset()
# merge overlapping labels
fw = merge_overlap(fw, im, mo, data, margin)
mo.write(fw[data])
im.close()
def get_maxlabels_from_attribute(filelist, ids, maxlabelfile):
maxlabels = []
for datafile in filelist:
image_in = '{}/{}'.format(datafile, ids)
im = Image(image_in, permission='r')
im.load(load_data=False)
maxlabels.append(im.ds.attrs['maxlabel'])
im.close()
if maxlabelfile:
np.savetxt(maxlabelfile, maxlabels, fmt='%d')
return maxlabels
def generate_report(image_in, info_dict={}, ioff=True):
"""Generate a QC report of the mask creation process."""
report_type = 'mask'
# Turn interactive plotting on/off.
if ioff:
plt.ioff()
else:
plt.ion()
# Get paths from image if info_dict not provided.
if not info_dict:
im = Image(image_in)
info_dict['paths'] = im.split_path()
im.close()
info_dict['parameters'] = get_parameters(info_dict, report_type)
info_dict['centreslices'] = get_centreslices(info_dict)
# info_dict['medians'] = get_medians(info_dict)
# Create the axes.
figsize = (18, 9)
gridsize = (1, 4)
f = plt.figure(figsize=figsize, constrained_layout=False)
gs0 = gridspec.GridSpec(gridsize[0], gridsize[1], figure=f)
axs = [gen_orthoplot(f, gs0[0, i]) for i in range(0, 4)]
# Plot the images and graphs.
info_dict['plotinfo'] = {'vmax': 10000}
plot_images(axs, info_dict)
# Add annotations and save as pdf.
header = 'mLSR-3D Quality Control'
figtitle = '{}: {} \n {}'.format(
header,
report_type,
info_dict['paths']['fname']
)
figpath = '{}_{}-report.pdf'.format(
info_dict['paths']['base'],
report_type
)
f.suptitle(figtitle, fontsize=14, fontweight='bold')
add_titles(axs, info_dict)
f.savefig(figpath)
def get_centreslices(info_dict, idss=[], ch=0):
"""Get the centreslices for all dims and steps."""
h5_path = info_dict['paths']['file']
image_in = h5_path + info_dict['paths']['int']
im = Image(image_in)
im.load(load_data=False)
if not idss:
idss = [k for k in im.file.keys()]
im.close()
centreslices = {
ids: {dim: get_centreslice(h5_path, ids, dim, ch)
for dim in 'zyx'}
for ids in idss
}
return centreslices
def get_info_dict(image_in, info_dict={}, report_type='bfc', channel=0):
im = Image(image_in)
im.load(load_data=False)
info_dict['elsize'] = {dim: im.elsize[i] for i, dim in enumerate(im.axlab)}
info_dict['paths'] = im.split_path() # FIXME: out_base
info_dict['paths']['out_base'] = info_dict['paths']['base']
im.close()
ppath = '{}.pickle'.format(info_dict['paths']['base'])
with open(ppath, 'rb') as f:
info_dict['parameters'] = pickle.load(f)
info_dict['centreslices'] = get_centreslices(info_dict)
# info_dict['medians'] = get_medians(info_dict)
info_dict['medians'], info_dict['means'], info_dict['stds'], info_dict[
'n_samples'] = get_means_and_stds(info_dict, ch=channel, thr=1000)
return info_dict
def biasfield_correction(
image_in,
parameter_file='',
outputdir='',
channels=[],
mask_in='',
resolution_level=-1,
downsample_factors=[1, 1, 1, 1, 1],
n_iterations=50,
n_fitlevels=4,
n_bspline_cps=[5, 5, 5],
):
"""Perform N4 bias field correction."""
params = get_params(locals(), parameter_file, 'biasfield_correction')
if not params['channels']:
im = Image(image_in, permission='r')
im.load()
n_channels = im.dims[im.axlab.index('c')]
params['channels'] = list(range(n_channels))
im.close()
n_workers = get_n_workers(len(params['channels']), params)
arglist = [(
image_in,
ch,
params['mask_in'],
params['resolution_level'],
params['downsample_factors'],
params['n_iterations'],
params['n_fitlevels'],
params['n_bspline_cps'],
outputdir,
) for ch in params['channels']]
with multiprocessing.Pool(processes=n_workers) as pool:
pool.starmap(estimate_biasfield, arglist)
def stack_channels(images_in, outputpath=''):
channels = []
for image_in in images_in:
im = Image(image_in, permission='r')
im.load(load_data=True)
channels.append(im.ds[:])
data = np.stack(channels, axis=3)
mo = Image(outputpath)
mo.elsize = list(im.elsize) + [1]
mo.axlab = im.axlab + 'c'
mo.dims = data.shape
mo.chunks = list(im.chunks) + [1]
mo.dtype = im.dtype
mo.set_slices()
if outputpath:
mo.create()
mo.write(data)
mo.close()
return mo
def get_paths(image_in,
resolution_level=-1,
channel=0,
outputstem='',
step='',
save_steps=False):
"""Get the parameters for the preprocessing step."""
# get paths from input file
if resolution_level != -1: # we should have an Imaris pyramid
image_in = '{}/DataSet/ResolutionLevel {}'.format(
image_in, resolution_level)
im = Image(image_in, permission='r')
im.load(load_data=False)
paths = im.split_path()
im.close()
# define output basename
paths['out_base'] = outputstem
# define h5 output template
paths['out_h5'] = '{}.h5/{}'.format(paths['out_base'], '{}')
# define output for main results and intermediate steps
if not outputstem:
paths['main'] = paths['steps'] = ''
else:
### FIXME ????????????? what is this?
if save_steps:
paths['main'] = paths['steps'] = paths['out_h5']
else:
paths['main'] = paths['out_h5']
paths['steps'] = paths['out_h5']
# define output for parameters
paths['params'] = '{}.pickle'.format(paths['out_base'], step)
return paths
def mergeblocks(
images_in,
dataslices=None,
blocksize=[],
blockmargin=[],
blockrange=[],
blockoffset=[0, 0, 0],
fullsize=[],
is_labelimage=False,
relabel=False,
neighbourmerge=False,
save_fwmap=False,
blockreduce=[],
func='np.amax',
datatype='',
usempi=False,
outputpath='',
save_steps=False,
protective=False,
):
"""Merge blocks of data into a single hdf5 file."""
if blockrange:
images_in = images_in[blockrange[0]:blockrange[1]]
mpi = wmeMPI(usempi)
im = Image(images_in[0], permission='r')
im.load(mpi.comm, load_data=False)
props = im.get_props(protective=protective, squeeze=True)
ndim = im.get_ndim()
props['dtype'] = datatype or props['dtype']
props['chunks'] = props['chunks'] or None
# get the size of the outputfile
# TODO: option to derive fullsize from dset_names?
if blockreduce:
datasize = np.subtract(fullsize, blockoffset)
outsize = [
int(np.ceil(d / np.float(b)))
for d, b in zip(datasize, blockreduce)
]
props['elsize'] = [e * b for e, b in zip(im.elsize, blockreduce)]
else: # FIXME: 'zyx(c)' stack assumed
outsize = np.subtract(fullsize, blockoffset)
if ndim == 4:
outsize = list(outsize) + [im.ds.shape[3]] # TODO: flexible insert
if outputpath.endswith('.ims'):
mo = LabelImage(outputpath)
mo.create(comm=mpi.comm)
else:
props['shape'] = outsize
mo = LabelImage(outputpath, **props)
mo.create(comm=mpi.comm)
mpi.blocks = [{'path': image_in} for image_in in images_in]
mpi.nblocks = len(images_in)
mpi.scatter_series()
# merge the datasets
maxlabel = 0
for i in mpi.series:
block = mpi.blocks[i]
# try:
maxlabel = process_block(block['path'], ndim, blockreduce, func,
blockoffset, blocksize, blockmargin, fullsize,
mo, is_labelimage, relabel, neighbourmerge,
save_fwmap, maxlabel, mpi)
print('processed block {:03d}: {}'.format(i, block['path']))
# except Exception as e:
# print('failed block {:03d}: {}'.format(i, block['path']))
# print(e)
im.close()
mo.close()
return mo
def apply_bias_field_full(image_in,
bias_in,
dsfacs=[1, 64, 64, 1],
in_place=False,
write_to_single_file=False,
blocksize_xy=1280,
outputpath='',
channel=None):
"""single-core in ~200 blocks"""
perm = 'r+' if in_place else 'r'
im = Image(image_in, permission=perm)
im.load(load_data=False)
bf = Image(bias_in, permission='r')
bf.load(load_data=False)
if channel is not None:
im.slices[3] = slice(channel, channel + 1)
if write_to_single_file: # assuming single-channel copied file here
mo = Image(outputpath)
mo.load()
mo.slices[3] = slice(0, 1, 1)
mpi = wmeMPI(usempi=False)
mpi_nm = wmeMPI(usempi=False)
if blocksize_xy:
blocksize = [im.dims[0], blocksize_xy, blocksize_xy, 1, 1]
blockmargin = [0, im.chunks[1], im.chunks[2], 0, 0]
else:
blocksize = im.dims[:3] + [1, 1]
blockmargin = [0] * len(im.dims)
mpi.set_blocks(im, blocksize, blockmargin)
mpi_nm.set_blocks(im, blocksize)
mpi.scatter_series()
for i in mpi.series:
print(i)
block = mpi.blocks[i]
data_shape = list(im.slices2shape(block['slices']))
block_nm = mpi_nm.blocks[i]
it = zip(block['slices'], block_nm['slices'], blocksize, data_shape)
data_shape = list(im.slices2shape(block_nm['slices']))
data_slices = []
for b_slc, n_slc, bs, ds in it:
m_start = n_slc.start - b_slc.start
m_stop = m_start + bs
m_stop = min(m_stop, ds)
data_slices.append(slice(m_start, m_stop, None))
data_slices[3] = block['slices'][3]
data_shape = list(im.slices2shape(data_slices))
# get the fullres image block
im.slices = block['slices']
data = im.slice_dataset().astype('float')
# get the upsampled bias field
bias = get_bias_field_block(bf, im.slices, data.shape)
data /= bias
data = np.nan_to_num(data, copy=False)
if in_place:
im.slices = block_nm['slices']
data = data[tuple(data_slices[:3])].astype(im.dtype)
im.write(data)
elif write_to_single_file:
mo.slices = block_nm['slices']
mo.slices[3] = slice(0, 1, 1)
data = data[tuple(data_slices[:3])].astype(mo.dtype)
mo.write(data)
else:
props = im.get_props()
if len(im.dims) > 4:
props = im.squeeze_props(props, dim=4)
if len(im.dims) > 3:
props = im.squeeze_props(props, dim=3)
props['axlab'] = 'zyx' # FIXME: axlab return as string-list
props['shape'] = bias.shape
props['slices'] = None
props['dtype'] = bias.dtype
mo = Image(block['path'], **props) # FIXME: needs channel
mo.create(comm=mpi.comm)
mo.slices = None
mo.set_slices()
mo.write(data=bias)
mo.close()
im.close()
bf.close()
def generate_report(image_in, info_dict={}, ioff=True):
"""Generate a QC report of the segmentation process."""
report_type = 'seg'
# Turn interactive plotting on/off.
if ioff:
plt.ioff()
else:
plt.ion()
# Get paths from image if info_dict not provided.
if not info_dict:
im = Image(image_in)
info_dict['paths'] = im.split_path() # FIXME: out_base
info_dict['paths']['out_base'] = info_dict['paths']['base']
im.close()
info_dict['parameters'] = load_parameters(
info_dict['paths']['out_base'])
info_dict['centreslices'] = get_centreslices(
info_dict,
idss=[
'mean_mask',
'memb/planarity_mask',
'memb/mean',
'memb/mean_smooth',
'chan/ch00',
'nucl/dapi_mask',
'nucl/dapi_preprocess',
'segm/labels_edt',
'segm/labels_memb',
'segm/labels_memb_del',
'segm/seeds_edt',
'segm/seeds_mask',
'segm/seeds_peaks_dil',
])
info_dict['medians'] = {}
# Create the axes.
figsize = (18, 9)
gridsize = (2, 4)
f = plt.figure(figsize=figsize, constrained_layout=False)
gs0 = gridspec.GridSpec(gridsize[0], gridsize[1], figure=f)
axs = [
gen_orthoplot(f, gs0[j, i]) for j in range(0, 2) for i in range(0, 4)
]
# Plot the images and graphs. vmaxs = [15000] + [5000] * 7
info_dict['plotinfo'] = {'vmax': 10000}
plot_images(axs, info_dict)
# Add annotations and save as pdf.
header = 'mLSR-3D Quality Control'
figtitle = '{}: {} \n {}'.format(header, report_type,
info_dict['paths']['fname'])
figpath = '{}_{}-report.pdf'.format(info_dict['paths']['out_base'],
report_type)
f.suptitle(figtitle, fontsize=14, fontweight='bold')
add_titles(axs, info_dict)
f.savefig(figpath)
info_dict.clear()
def csv_to_im(
image_in,
csv_path,
labelkey='label',
key='dapi',
name='',
maxlabel=0,
normalize=False,
scale_uint16=False,
replace_nan=False,
channel=-1,
outpath='',
):
"""Write segment backprojection."""
if isinstance(image_in, Image):
labels = image_in
else:
labels = LabelImage(image_in)
labels.load(load_data=False)
if not maxlabel:
labels.set_maxlabel()
maxlabel = labels.maxlabel
if csv_path.endswith('.csv'):
df = pd.read_csv(csv_path)
df = df.astype({labelkey: int})
elif csv_path.endswith('.h5ad'):
import scanpy as sc
adata = sc.read(csv_path)
if not csv_path.endswith('_nofilter.h5ad'):
adata.X = adata.raw.X
df = adata.obs[labelkey].astype(int)
df = pd.concat([df, adata[:, key].to_df()], axis=1)
# for key in keys: # TODO
fw = np.zeros(maxlabel + 1, dtype='float')
for index, row in df.iterrows():
fw[int(row[labelkey])] = row[key]
if replace_nan:
fw = np.nan_to_num(fw)
if normalize:
def normalize_data(data):
"""Normalize data between 0 and 1."""
data = data.astype('float64')
datamin = np.amin(data)
datamax = np.amax(data)
data -= datamin
data *= 1 / (datamax - datamin)
return data, [datamin, datamax]
fw_n, fw_minmax = normalize_data(fw)
fw_n *= 65535
fw = fw_n
elif scale_uint16: # for e.g. pseudotime / FA / etc / any [0, 1] vars
fw *= 65535
out = labels.forward_map(list(fw))
if outpath.endswith('.ims'):
mo = Image(outpath, permission='r+')
mo.load(load_data=False)
if channel >= 0 and channel < mo.dims[3]:
ch = channel
else:
mo.create()
ch = mo.dims[3] - 1
mo.slices[3] = slice(ch, ch + 1, 1)
mo.write(out.astype(mo.dtype)) # FIXME: >65535 wraps around
cpath = 'DataSetInfo/Channel {}'.format(ch)
name = name or key
mo.file[cpath].attrs['Name'] = np.array([c for c in name], dtype='|S1')
mo.close()
elif outpath.endswith('.nii.gz'):
props = labels.get_props()
if not labels.path.endswith('.nii.gz'):
props = transpose_props(props, outlayout='xyz')
out = out.transpose()
mo = write_output(outpath, out, props)
else:
outpath = outpath or gen_outpath(labels, key)
mo = write_output(outpath, out, labels.get_props())
return mo