def copyData(source, sink, x=None, y=None, z=None, returnMemmap=False):
"""Copy a data file from source to sink
Arguments:
source (str): file name pattern of source
sink (str): file name pattern of sink
returnMemmap (bool): returns the result as an array
Returns:
str: file name of the copy
"""
out_type = io.dataFileNameToType(sink)
if out_type == 'TIF':
if isinstance(source, np.memmap) and x == y == y == z == None:
shutil.copyfile(source.filename, sink)
else:
Xsize, Ysize, Zsize = io.dataSize(source)
# cropped size
Xsize = io.toDataSize(Xsize, r=x)
Ysize = io.toDataSize(Ysize, r=y)
Zsize = io.toDataSize(Zsize, r=z)
im = io.readData(source, x=x, y=y, z=z)
out = io.writeData(sink, im, returnMemmap=returnMemmap)
if returnMemmap:
return io.readData(sink)
else:
return sink
else:
raise RuntimeError(
'copying from TIF to {} not yet supported.'.format(out_type))
def voxelizePixel(points, dataSize=None, weights=None):
"""Mark pixels/voxels of each point in an image array
Arguments:
points (array): point data array
dataSize (tuple or None): size of the final output data, if None size is determined by maximal point coordinates
weights (array or None): weights for each points, if None weights are all 1s.
Returns:
(array): volumetric data with with points marked in voxels
"""
if dataSize is None:
dataSize = tuple(
int(math.ceil(points[:, i].max())) for i in range(points.shape[1]))
elif isinstance(dataSize, str):
dataSize = io.dataSize(dataSize)
points = np.rint(points).astype('int64')
if weights is None:
vox = np.zeros(dataSize, dtype=np.int16)
for i in range(points.shape[0]):
if 0 < points[i, 0] < dataSize[0] and 0 < points[
i, 1] < dataSize[1] and 0 < points[i, 2] < dataSize[2]:
vox[points[i, 0], points[i, 1], points[i, 2]] += 1
elif isinstance(weights, int):
vox = np.zeros(
dataSize,
dtype=np.int16) # TODO: dtype should depend on weight value passed
for i in range(points.shape[0]):
if 0 < points[i, 0] < dataSize[0] and 0 < points[
i, 1] < dataSize[1] and 0 < points[i, 2] < dataSize[2]:
vox[points[i, 0], points[i, 1], points[i, 2]] += weights
elif isinstance(weights, np.ndarray):
vox = np.zeros(dataSize, dtype=weights.dtype)
for i in range(points.shape[0]):
if 0 < points[i, 0] < dataSize[0] and 0 < points[
i, 1] < dataSize[1] and 0 < points[i, 2] < dataSize[2]:
vox[points[i, 0], points[i, 1], points[i, 2]] += weights[i]
else:
RuntimeError(
'VoxelizePixel: only Bool, Int, and arrays are valid weight values.'
)
return vox
def dataSize(filename, **args):
"""Returns size of data stored as a file list
Arguments:
filename (str): file name as regular expression
x,y,z (tuple): data range specifications
Returns:
tuple: data size
"""
fp, fl = readFileList(filename)
nz = len(fl)
d2 = io.dataSize(os.path.join(fp, fl[0]))
if not len(d2) == 2:
raise RuntimeError(
"FileList: importing multiple files of dim %d not supported!" %
len(d2))
dims = (nz, ) + d2
return io.dataSizeFromDataRange(dims, **args)
def voxelize(points,
dataSize=None,
sink=None,
method='Spherical',
size=(5, 5, 5),
weights=None):
"""Converts a list of points into an volumetric image array
Arguments:
points (array): point data array
dataSize (tuple or str): size of final image in xyz. If str, will use the size of the passed image.
sink (str, array or None): the location to write or return the resulting voxelization image, if None return array
method (str or None): method for voxelization: 'Spherical', 'Rectangular' or 'Pixel'
size (tuple): size parameter for the voxelization
weights (array or None): weights for each point, None is uniform weights
Returns:
(array): volumetric data of smeared out points
"""
log.verbose('voxelizing points')
points = io.readPoints(points)
if dataSize is None:
dataSize = tuple(
int(math.ceil(points[:, i].max())) for i in range(points.shape[1]))
elif isinstance(dataSize, str):
dataSize = io.dataSize(dataSize)
if method.lower() == 'spherical':
if weights is None:
data = vox.voxelizeSphere(points.astype('float'), dataSize[0],
dataSize[1], dataSize[2], size[0],
size[1], size[2])
else:
data = vox.voxelizeSphereWithWeights(points.astype('float'),
dataSize[0], dataSize[1],
dataSize[2], size[0], size[1],
size[2], weights)
elif method.lower() == 'rectangular':
if weights is None:
data = vox.voxelizeRectangle(points.astype('float'), dataSize[0],
dataSize[1], dataSize[2], size[0],
size[1], size[2])
else:
data = vox.voxelizeRectangleWithWeights(points.astype('float'),
dataSize[0], dataSize[1],
dataSize[2], size[0],
size[1], size[2], weights)
elif method.lower() == 'pixel':
data = voxelizePixel(points, dataSize, weights)
else:
raise RuntimeError('voxelize: mode: %s not supported!' % method)
if data.dtype == np.float64:
log.warning(
'Converting dtype float64 to int32 for output. This may result in loss of info.'
)
data = data.astype('int32')
if sink:
return io.writeData(sink, data, returnMemmap=True)
else:
return data
def overlay_points(pointSource,
dataSource,
output=None,
overlay=True,
x=all,
y=all,
z=all):
"""Overlay points on 3D data and return as color image
Arguments:
pointSource (str or array): point data to be overlayed on the image data
dataSource (str or array): volumetric image data. if None just output points as an image
overlay (bool): if False will not overlay and just output points as an image.
x, y, z (all or tuple): sub-range specification
Returns:
(str or array): image overlayed with points
See Also:
:func:`overlayLabel`
"""
points = (io.readPoints(pointSource, x=x, y=y, z=z,
shift=True)).astype(int)
if overlay:
X, Y, Z = io.dataSize(dataSource)
datatype = io.readData(dataSource, x=x, y=y, z=0).dtype
if io.isMappable(output):
output = tif.tifffile.memmap(output,
dtype=datatype,
shape=(Z, 2, Y, X),
imagej=True) # TZCYXS FIJI
elif io.isFileExpression(output):
output = [
tif.tifffile.memmap(output,
dtype=datatype,
shape=(2, Y, X),
imagej=True) for z in range(Z)
] # sequence of memmap files
elif output is None:
output = numpy.zeros((Z, 2, Y, X), dtype=datatype)
else:
RuntimeError('output format not compatable with overlayPoints: ' +
output)
for z in range(Z):
print(('Overlaying {}...'.format(z)))
output[z][0][:] = io.readData(dataSource, x=x, y=y,
z=z).squeeze().T
z_points = points[points[:, -1] == z][:, :2]
output[z][1][[*z_points[:, ::-1].T]] = 1
return output
else:
shape = io.dataSize(dataSource)
cimage = vox.voxelize(
points, shape, output=output, method='Pixel', weights=65535
) # TODO: weight should depend on bit depth of dataSource
return cimage
def processSubStack(flow, output_properties, source, overlap_indices, unique_indices,
temp_dir_root):
""" Helper to process stack in parallel
Args:
flow (tuple): images filters to run in sequential order.
Entries should be a dict and will be passed to *bq3d.image_filters.filter_image*.
The input image to each filter will the be output of the pevious filter.
output_properties: (list): properties to include in output. See
label_properties.region_props for more info
source (str): path to image file to analyse.
overlap_indices (tuple or list): list of indices as [start,stop] along each axis to analyse.
unique_indices (tuple or list): list of indices as [start,stop] along each axis
corresponding
to the non-overlapping portion of the image being analyzed.
temp_dir (str): temp dir to be used for processing.
Returns:
"""
timer = Timer()
zRng, yRng, xRng = overlap_indices
log.info(f'chunk ranges: z= {zRng}, y= {yRng}, x = {xRng}')
#memMap routine
temp_dir = unique_temp_dir('run', path = temp_dir_root)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
mmapFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif'
log.info('Creating memory mapped substack at: {}'.format(mmapFile))
img = io.copyData(source, mmapFile, x=xRng, y=yRng, z=zRng, returnMemmap=True)
rawFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif'
log.info('Creating raw substack at: {}'.format(rawFile))
raw = io.copyData(img.filename, rawFile, returnMemmap=True)
# if a flow
filtered_im = img
for p in flow:
params = dict(p)
filter = params.pop('filter')
if 'save' in params:
save = params.pop('save')
else:
save = False
filtered_im = filter_image(filter, filtered_im, temp_dir_root = temp_dir, **params)
# save intermediate output
if save:
log.info(f'Saving output to {save}')
h, ext, dfmt = splitFileExpression(save)
for z in range(*zRng):
fname = h + (dfmt % z) + ext
if not os.path.isfile(fname):
io.empty(fname, io.dataSize(source)[1:], filtered_im.dtype)
unique = filtered_im[unique_slice(overlap_indices, unique_indices)]
io.writeData(save, unique, substack=unique_indices)
# get label properties and return
if output_properties:
props = label_props(raw, filtered_im, output_properties)
else:
props = []
shutil.rmtree(temp_dir, ignore_errors=True)
timer.log_elapsed(prefix='Processed chunk')
return props
def resamplePoints(source,
sink=None,
dataSizeSource=None,
dataSizeSink=None,
orientation=None,
resolutionSource=(4.0625, 4.0625, 3),
resolutionSink=(25, 25, 25),
**args):
"""Resample Points to map from original data to the coordinates of the resampled image
The resampling of points here corresponds to he resampling of an image in :func:`resampleData`
Arguments:
pointSource (str or array): image to be resampled
pointSink (str or None): destination of resampled image
orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
dataSizeSource (str, tuple or None): size of the data source
dataSizeSink (tuple or None): target size of the resampled image
resolutionSource (tuple): resolution of the source image (in length per pixel)
resolutionSink (tuple): resolution of the resampled image (in length per pixel)
Returns:
(array or str): data or file name of resampled points
Notes:
* resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference as when viewed by matplotlib or ImageJ
* orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes
* only a minimal set of information to detremine the resampling parameter
has to be given, e.g. dataSizeSource and dataSizeSink
"""
log.info('resampling points')
# size of data source
if isinstance(dataSizeSource, str):
dataSizeSource = io.dataSize(dataSizeSource)
if isinstance(dataSizeSink, str):
dataSizeSink = io.dataSize(dataSizeSink)
dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(
dataSizeSource=dataSizeSource,
dataSizeSink=dataSizeSink,
resolutionSource=resolutionSource,
resolutionSink=resolutionSink,
orientation=orientation)
points = io.readPoints(source)
dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation)
# scaling factors
scale = [
float(dataSizeSource[i]) / float(dataSizeSinkI[i]) for i in range(3)
]
repoints = points.copy()
for i in range(3):
repoints[:, i] = repoints[:, i] / scale[i]
# permute for non trivial orientation
if not orientation is None:
per = orientationToPermuation(orientation)
repoints = repoints[:, per]
for i in range(3):
if orientation[i] < 0:
repoints[:, i] = dataSizeSink[i] - repoints[:, i]
if sink:
return io.writePoints(sink, repoints)
else:
return repoints
def resampleDataInverse(sink,
source=None,
dataSizeSource=None,
orientation=None,
resolutionSource=(4.0625, 4.0625, 3),
resolutionSink=(25, 25, 25),
processingDirectory=None,
processes=bq3d.config.processes,
cleanup=True,
interpolation='linear',
**args):
"""Resample data inversely to :func:`resampleData` routine
Arguments:
sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`)
source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`)
dataSizeSource (tuple or None): target size of the resampled image
orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
resolutionSource (tuple): resolution of the source image (in length per pixel)
resolutionSink (tuple): resolution of the resampled image (in length per pixel)
processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created
processes (int): number of processes to use for parallel resampling
cleanup (bool): remove temporary files
interpolation (str): method to use for interpolating to the resmapled image
Returns:
(array or str): data or file name of resampled image
Notes:
* resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference as when viewed by matplotlib or ImageJ
* orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes
* only a minimal set of information to detremine the resampling parameter
has to be given, e.g. dataSizeSource and dataSizeSink
"""
# assume we can read data fully into memory
resampledData = io.readData(sink)
dataSizeSink = resampledData.shape
if isinstance(dataSizeSource, str):
dataSizeSource = io.dataSize(dataSizeSource)
dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(
dataSizeSource=dataSizeSource,
dataSizeSink=dataSizeSink,
resolutionSource=resolutionSource,
resolutionSink=resolutionSink,
orientation=orientation)
dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation)
# flip axes back and permute inversely
if not orientation is None:
if orientation[0] < 0:
resampledData = resampledData[::-1, :, :]
if orientation[1] < 0:
resampledData = resampledData[:, ::-1, :]
if orientation[2] < 0:
resampledData = resampledData[:, :, ::-1]
# reorient
peri = invert_orientation(orientation)
peri = orientationToPermuation(peri)
resampledData = resampledData.transpose(peri)
# upscale in z
interpolation = parse_interpolation(interpolation)
resampledDataXY = numpy.zeros(
(dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]),
dtype=resampledData.dtype)
for i in range(dataSizeSinkI[0]):
if i % 25 == 0:
log.vebose("resampleDataInverse: processing %d/%d" %
(i, dataSizeSinkI[0]))
# cv2.resize takes reverse order of sizes !
resampledDataXY[i] = cv2.resize(resampledData[i],
(dataSizeSource[2], dataSizeSinkI[1]),
interpolation=interpolation)
# upscale x, y in parallel
if io.isFileExpression(source):
files = source
else:
if processingDirectory is None:
processingDirectory = tempfile.mkdtemp()
files = os.path.join(sink[0], 'resample_\d{4}.tif')
io.writeData(files, resampledDataXY)
nZ = dataSizeSource[0]
pool = multiprocessing.Pool(processes=processes)
argdata = []
for i in range(nZ):
argdata.append((source, fl.fileExpressionToFileName(files, i),
dataSizeSource, interpolation, i, nZ))
pool.map(_resampleXYParallel, argdata)
if io.isFileExpression(source):
return source
else:
data = io.convertData(files, source)
if cleanup:
shutil.rmtree(processingDirectory)
return data
def resampleData(source,
sink=None,
orientation=None,
dataSizeSink=None,
resolutionSource=(.91, .91, 8.3),
resolutionSink=(25, 25, 25),
processingDirectory=bq3d.config.temp_dir,
processes=bq3d.config.processes,
cleanup=True,
interpolation='linear',
**kwargs):
"""Resample data of source in resolution and orientation
Arguments:
source (str or array): image to be resampled
sink (str or None): destination of resampled image
orientation (tuple): orientation specified by permuation and change in sign of (1,2,3)
dataSizeSink (tuple or None): target size of the resampled image
resolutionSource (tuple): resolution of the source image (in length per pixel)
resolutionSink (tuple): resolution of the resampled image (in length per pixel)
processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created
processes (int): number of processes to use for parallel resampling
cleanup (bool): remove temporary files
interpolation (str): method to use for interpolating to the resmapled image
Returns:
(array or str): data or file name of resampled image
Notes:
* resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference as when viewed by matplotlib or ImageJ
* orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes
* only a minimal set of information to detremine the resampling parameter
has to be given, e.g. dataSizeSource and dataSizeSink
"""
log.info(f'interpolation method: {interpolation}')
log.info(f'Number of processes: {processes}')
interpolation = parse_interpolation(interpolation)
dataSizeSource = io.dataSize(source)
if isinstance(dataSizeSink, str):
dataSizeSink = io.dataSize(dataSizeSink)
# orient actual resolutions onto reference resolution
dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize(
dataSizeSource=dataSizeSource,
dataSizeSink=dataSizeSink,
resolutionSource=resolutionSource,
resolutionSink=resolutionSink,
orientation=orientation)
dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation)
# setup intermediate output
if processingDirectory is None:
processingDirectory = tempfile.mkdtemp()
else:
io.createDirectory(processingDirectory)
resampledXYFile = os.path.join(processingDirectory, 'resampleXY.tif')
data_type = io.getDataType(source)
resampledXY = io.empty(resampledXYFile,
dtype=data_type,
shape=(dataSizeSource[0], dataSizeSinkI[1],
dataSizeSinkI[2]),
imagej=True)
nZ = dataSizeSource[0]
# resample in XY
# chunk for each process
Zlist = list(range(nZ))
chunks = [Zlist[i::processes] for i in range(processes)]
argdata = [(source, resampledXYFile, dataSizeSinkI, interpolation, chunk)
for chunk in chunks]
if processes == 1:
_resampleXYParallel(argdata[0])
else:
pool = multiprocessing.Pool(processes=processes)
pool.map(_resampleXYParallel, argdata)
pool.close()
# rescale in z
resampledXY = io.readData(resampledXYFile)
resampledData = numpy.zeros(
(dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSinkI[2]),
dtype=data_type)
for i in range(dataSizeSinkI[1]): # faster if iterate over y
if i % 50 == 0:
log.verbose(
("resampleData: Z: Resampling %d/%d" % (i, dataSizeSinkI[0])))
resampledData[:, i] = cv2.resize(resampledXY[:, i],
(dataSizeSinkI[2], dataSizeSinkI[0]),
interpolation=interpolation)
if cleanup:
shutil.rmtree(processingDirectory)
if not orientation is None:
# reorient
per = orientationToPermuation(orientation)
resampledData = resampledData.transpose(per)
# reverse orientation after permuting e.g. (-2,1) brings axis 2 to first axis and we can reorder there
if orientation[0] < 0:
resampledData = resampledData[::-1, :, :]
if orientation[1] < 0:
resampledData = resampledData[:, ::-1, :]
if orientation[2] < 0:
resampledData = resampledData[:, :, ::-1]
log.verbose("resampleData: resampled data size: " +
str(resampledData.shape))
return io.writeData(sink, resampledData)
def copyData(source, sink, processes=1, x=None, y=None, z=None, **kwargs):
"""Copy a data file from source to sink when for entire list of files
Arguments:
source (str): file name pattern of source
sink (str): file name pattern of sink
processes (int): number of processes to be used when writing files in parallel
Returns:
str: file name pattern of the copy
Notes:
TODO: replace cropData with this. currently still using because cropData is more flexible
TODO: could simplify by not splitting up the regex files
"""
if isinstance(source, Path):
source = source.as_posix()
if isinstance(sink, Path):
sink = sink.as_posix()
fp, fl = readFileList(source,
z=z) # crops is z by only reading files in range
out_type = io.dataFileNameToType(sink)
if out_type == 'FileList':
# setup inputs for pool
files = []
z_idx = []
for i, fn in enumerate(fl):
files.append(os.path.join(fp, fn))
z_idx.append(i)
f_chunks = [files[i::processes] for i in range(processes)]
z_chunks = [z_idx[i::processes] for i in range(processes)]
# setup pool
args = [(sources, z_chunks[i], sink, x, y)
for i, sources in enumerate(f_chunks)]
if processes == 1:
_parallelCopyToFileList(*args)
else:
pool = multiprocessing.Pool(processes)
pool.map(_parallelCopyToFileList, args)
pool.close()
elif out_type == 'TIF':
# get datasize
Zsize, Ysize, Xsize = io.dataSize(source)
# cropped size
Xsize = io.toDataSize(Xsize, r=x)
Ysize = io.toDataSize(Ysize, r=y)
Zsize = io.toDataSize(Zsize, r=z)
# setup inputs for pool
data_type = getDataType(os.path.join(fp, fl[0]))
files = [os.path.join(fp, i) for i in fl]
idxs = list(range(len(files)))
z_f_chunks = [files[i::processes] for i in range(processes)]
z_i_chunks = [idxs[i::processes] for i in range(processes)]
im = io.empty(sink, dtype=data_type, shape=(Zsize, Ysize, Xsize))
args = [(z_f_chunks[i], idxs, sink, x, y)
for i, idxs in enumerate(z_i_chunks)]
# setup pool
if processes == 1:
_parallelCopyToTif(*args)
else:
pool = multiprocessing.Pool(processes)
pool.map(_parallelCopyToTif, args)
pool.close()
return sink