def copyData(source, sink, x=None, y=None, z=None, returnMemmap=False): """Copy a data file from source to sink Arguments: source (str): file name pattern of source sink (str): file name pattern of sink returnMemmap (bool): returns the result as an array Returns: str: file name of the copy """ out_type = io.dataFileNameToType(sink) if out_type == 'TIF': if isinstance(source, np.memmap) and x == y == y == z == None: shutil.copyfile(source.filename, sink) else: Xsize, Ysize, Zsize = io.dataSize(source) # cropped size Xsize = io.toDataSize(Xsize, r=x) Ysize = io.toDataSize(Ysize, r=y) Zsize = io.toDataSize(Zsize, r=z) im = io.readData(source, x=x, y=y, z=z) out = io.writeData(sink, im, returnMemmap=returnMemmap) if returnMemmap: return io.readData(sink) else: return sink else: raise RuntimeError( 'copying from TIF to {} not yet supported.'.format(out_type))
def voxelizePixel(points, dataSize=None, weights=None): """Mark pixels/voxels of each point in an image array Arguments: points (array): point data array dataSize (tuple or None): size of the final output data, if None size is determined by maximal point coordinates weights (array or None): weights for each points, if None weights are all 1s. Returns: (array): volumetric data with with points marked in voxels """ if dataSize is None: dataSize = tuple( int(math.ceil(points[:, i].max())) for i in range(points.shape[1])) elif isinstance(dataSize, str): dataSize = io.dataSize(dataSize) points = np.rint(points).astype('int64') if weights is None: vox = np.zeros(dataSize, dtype=np.int16) for i in range(points.shape[0]): if 0 < points[i, 0] < dataSize[0] and 0 < points[ i, 1] < dataSize[1] and 0 < points[i, 2] < dataSize[2]: vox[points[i, 0], points[i, 1], points[i, 2]] += 1 elif isinstance(weights, int): vox = np.zeros( dataSize, dtype=np.int16) # TODO: dtype should depend on weight value passed for i in range(points.shape[0]): if 0 < points[i, 0] < dataSize[0] and 0 < points[ i, 1] < dataSize[1] and 0 < points[i, 2] < dataSize[2]: vox[points[i, 0], points[i, 1], points[i, 2]] += weights elif isinstance(weights, np.ndarray): vox = np.zeros(dataSize, dtype=weights.dtype) for i in range(points.shape[0]): if 0 < points[i, 0] < dataSize[0] and 0 < points[ i, 1] < dataSize[1] and 0 < points[i, 2] < dataSize[2]: vox[points[i, 0], points[i, 1], points[i, 2]] += weights[i] else: RuntimeError( 'VoxelizePixel: only Bool, Int, and arrays are valid weight values.' ) return vox
def dataSize(filename, **args): """Returns size of data stored as a file list Arguments: filename (str): file name as regular expression x,y,z (tuple): data range specifications Returns: tuple: data size """ fp, fl = readFileList(filename) nz = len(fl) d2 = io.dataSize(os.path.join(fp, fl[0])) if not len(d2) == 2: raise RuntimeError( "FileList: importing multiple files of dim %d not supported!" % len(d2)) dims = (nz, ) + d2 return io.dataSizeFromDataRange(dims, **args)
def voxelize(points, dataSize=None, sink=None, method='Spherical', size=(5, 5, 5), weights=None): """Converts a list of points into an volumetric image array Arguments: points (array): point data array dataSize (tuple or str): size of final image in xyz. If str, will use the size of the passed image. sink (str, array or None): the location to write or return the resulting voxelization image, if None return array method (str or None): method for voxelization: 'Spherical', 'Rectangular' or 'Pixel' size (tuple): size parameter for the voxelization weights (array or None): weights for each point, None is uniform weights Returns: (array): volumetric data of smeared out points """ log.verbose('voxelizing points') points = io.readPoints(points) if dataSize is None: dataSize = tuple( int(math.ceil(points[:, i].max())) for i in range(points.shape[1])) elif isinstance(dataSize, str): dataSize = io.dataSize(dataSize) if method.lower() == 'spherical': if weights is None: data = vox.voxelizeSphere(points.astype('float'), dataSize[0], dataSize[1], dataSize[2], size[0], size[1], size[2]) else: data = vox.voxelizeSphereWithWeights(points.astype('float'), dataSize[0], dataSize[1], dataSize[2], size[0], size[1], size[2], weights) elif method.lower() == 'rectangular': if weights is None: data = vox.voxelizeRectangle(points.astype('float'), dataSize[0], dataSize[1], dataSize[2], size[0], size[1], size[2]) else: data = vox.voxelizeRectangleWithWeights(points.astype('float'), dataSize[0], dataSize[1], dataSize[2], size[0], size[1], size[2], weights) elif method.lower() == 'pixel': data = voxelizePixel(points, dataSize, weights) else: raise RuntimeError('voxelize: mode: %s not supported!' % method) if data.dtype == np.float64: log.warning( 'Converting dtype float64 to int32 for output. This may result in loss of info.' ) data = data.astype('int32') if sink: return io.writeData(sink, data, returnMemmap=True) else: return data
def overlay_points(pointSource, dataSource, output=None, overlay=True, x=all, y=all, z=all): """Overlay points on 3D data and return as color image Arguments: pointSource (str or array): point data to be overlayed on the image data dataSource (str or array): volumetric image data. if None just output points as an image overlay (bool): if False will not overlay and just output points as an image. x, y, z (all or tuple): sub-range specification Returns: (str or array): image overlayed with points See Also: :func:`overlayLabel` """ points = (io.readPoints(pointSource, x=x, y=y, z=z, shift=True)).astype(int) if overlay: X, Y, Z = io.dataSize(dataSource) datatype = io.readData(dataSource, x=x, y=y, z=0).dtype if io.isMappable(output): output = tif.tifffile.memmap(output, dtype=datatype, shape=(Z, 2, Y, X), imagej=True) # TZCYXS FIJI elif io.isFileExpression(output): output = [ tif.tifffile.memmap(output, dtype=datatype, shape=(2, Y, X), imagej=True) for z in range(Z) ] # sequence of memmap files elif output is None: output = numpy.zeros((Z, 2, Y, X), dtype=datatype) else: RuntimeError('output format not compatable with overlayPoints: ' + output) for z in range(Z): print(('Overlaying {}...'.format(z))) output[z][0][:] = io.readData(dataSource, x=x, y=y, z=z).squeeze().T z_points = points[points[:, -1] == z][:, :2] output[z][1][[*z_points[:, ::-1].T]] = 1 return output else: shape = io.dataSize(dataSource) cimage = vox.voxelize( points, shape, output=output, method='Pixel', weights=65535 ) # TODO: weight should depend on bit depth of dataSource return cimage
def processSubStack(flow, output_properties, source, overlap_indices, unique_indices, temp_dir_root): """ Helper to process stack in parallel Args: flow (tuple): images filters to run in sequential order. Entries should be a dict and will be passed to *bq3d.image_filters.filter_image*. The input image to each filter will the be output of the pevious filter. output_properties: (list): properties to include in output. See label_properties.region_props for more info source (str): path to image file to analyse. overlap_indices (tuple or list): list of indices as [start,stop] along each axis to analyse. unique_indices (tuple or list): list of indices as [start,stop] along each axis corresponding to the non-overlapping portion of the image being analyzed. temp_dir (str): temp dir to be used for processing. Returns: """ timer = Timer() zRng, yRng, xRng = overlap_indices log.info(f'chunk ranges: z= {zRng}, y= {yRng}, x = {xRng}') #memMap routine temp_dir = unique_temp_dir('run', path = temp_dir_root) if not os.path.exists(temp_dir): os.mkdir(temp_dir) mmapFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif' log.info('Creating memory mapped substack at: {}'.format(mmapFile)) img = io.copyData(source, mmapFile, x=xRng, y=yRng, z=zRng, returnMemmap=True) rawFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif' log.info('Creating raw substack at: {}'.format(rawFile)) raw = io.copyData(img.filename, rawFile, returnMemmap=True) # if a flow filtered_im = img for p in flow: params = dict(p) filter = params.pop('filter') if 'save' in params: save = params.pop('save') else: save = False filtered_im = filter_image(filter, filtered_im, temp_dir_root = temp_dir, **params) # save intermediate output if save: log.info(f'Saving output to {save}') h, ext, dfmt = splitFileExpression(save) for z in range(*zRng): fname = h + (dfmt % z) + ext if not os.path.isfile(fname): io.empty(fname, io.dataSize(source)[1:], filtered_im.dtype) unique = filtered_im[unique_slice(overlap_indices, unique_indices)] io.writeData(save, unique, substack=unique_indices) # get label properties and return if output_properties: props = label_props(raw, filtered_im, output_properties) else: props = [] shutil.rmtree(temp_dir, ignore_errors=True) timer.log_elapsed(prefix='Processed chunk') return props
def resamplePoints(source, sink=None, dataSizeSource=None, dataSizeSink=None, orientation=None, resolutionSource=(4.0625, 4.0625, 3), resolutionSink=(25, 25, 25), **args): """Resample Points to map from original data to the coordinates of the resampled image The resampling of points here corresponds to he resampling of an image in :func:`resampleData` Arguments: pointSource (str or array): image to be resampled pointSink (str or None): destination of resampled image orientation (tuple): orientation specified by permuation and change in sign of (1,2,3) dataSizeSource (str, tuple or None): size of the data source dataSizeSink (tuple or None): target size of the resampled image resolutionSource (tuple): resolution of the source image (in length per pixel) resolutionSink (tuple): resolution of the resampled image (in length per pixel) Returns: (array or str): data or file name of resampled points Notes: * resolutions are assumed to be given for the axes of the intrinsic orientation of the data and reference as when viewed by matplotlib or ImageJ * orientation: permuation of 1,2,3 with potential sign, indicating which axes map onto the reference axes, a negative sign indicates reversal of that particular axes * only a minimal set of information to detremine the resampling parameter has to be given, e.g. dataSizeSource and dataSizeSink """ log.info('resampling points') # size of data source if isinstance(dataSizeSource, str): dataSizeSource = io.dataSize(dataSizeSource) if isinstance(dataSizeSink, str): dataSizeSink = io.dataSize(dataSizeSink) dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize( dataSizeSource=dataSizeSource, dataSizeSink=dataSizeSink, resolutionSource=resolutionSource, resolutionSink=resolutionSink, orientation=orientation) points = io.readPoints(source) dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation) # scaling factors scale = [ float(dataSizeSource[i]) / float(dataSizeSinkI[i]) for i in range(3) ] repoints = points.copy() for i in range(3): repoints[:, i] = repoints[:, i] / scale[i] # permute for non trivial orientation if not orientation is None: per = orientationToPermuation(orientation) repoints = repoints[:, per] for i in range(3): if orientation[i] < 0: repoints[:, i] = dataSizeSink[i] - repoints[:, i] if sink: return io.writePoints(sink, repoints) else: return repoints
def resampleDataInverse(sink, source=None, dataSizeSource=None, orientation=None, resolutionSource=(4.0625, 4.0625, 3), resolutionSink=(25, 25, 25), processingDirectory=None, processes=bq3d.config.processes, cleanup=True, interpolation='linear', **args): """Resample data inversely to :func:`resampleData` routine Arguments: sink (str or None): image to be inversly resampled (=sink in :func:`resampleData`) source (str or array): destination for inversly resmapled image (=source in :func:`resampleData`) dataSizeSource (tuple or None): target size of the resampled image orientation (tuple): orientation specified by permuation and change in sign of (1,2,3) resolutionSource (tuple): resolution of the source image (in length per pixel) resolutionSink (tuple): resolution of the resampled image (in length per pixel) processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created processes (int): number of processes to use for parallel resampling cleanup (bool): remove temporary files interpolation (str): method to use for interpolating to the resmapled image Returns: (array or str): data or file name of resampled image Notes: * resolutions are assumed to be given for the axes of the intrinsic orientation of the data and reference as when viewed by matplotlib or ImageJ * orientation: permuation of 1,2,3 with potential sign, indicating which axes map onto the reference axes, a negative sign indicates reversal of that particular axes * only a minimal set of information to detremine the resampling parameter has to be given, e.g. dataSizeSource and dataSizeSink """ # assume we can read data fully into memory resampledData = io.readData(sink) dataSizeSink = resampledData.shape if isinstance(dataSizeSource, str): dataSizeSource = io.dataSize(dataSizeSource) dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize( dataSizeSource=dataSizeSource, dataSizeSink=dataSizeSink, resolutionSource=resolutionSource, resolutionSink=resolutionSink, orientation=orientation) dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation) # flip axes back and permute inversely if not orientation is None: if orientation[0] < 0: resampledData = resampledData[::-1, :, :] if orientation[1] < 0: resampledData = resampledData[:, ::-1, :] if orientation[2] < 0: resampledData = resampledData[:, :, ::-1] # reorient peri = invert_orientation(orientation) peri = orientationToPermuation(peri) resampledData = resampledData.transpose(peri) # upscale in z interpolation = parse_interpolation(interpolation) resampledDataXY = numpy.zeros( (dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSource[2]), dtype=resampledData.dtype) for i in range(dataSizeSinkI[0]): if i % 25 == 0: log.vebose("resampleDataInverse: processing %d/%d" % (i, dataSizeSinkI[0])) # cv2.resize takes reverse order of sizes ! resampledDataXY[i] = cv2.resize(resampledData[i], (dataSizeSource[2], dataSizeSinkI[1]), interpolation=interpolation) # upscale x, y in parallel if io.isFileExpression(source): files = source else: if processingDirectory is None: processingDirectory = tempfile.mkdtemp() files = os.path.join(sink[0], 'resample_\d{4}.tif') io.writeData(files, resampledDataXY) nZ = dataSizeSource[0] pool = multiprocessing.Pool(processes=processes) argdata = [] for i in range(nZ): argdata.append((source, fl.fileExpressionToFileName(files, i), dataSizeSource, interpolation, i, nZ)) pool.map(_resampleXYParallel, argdata) if io.isFileExpression(source): return source else: data = io.convertData(files, source) if cleanup: shutil.rmtree(processingDirectory) return data
def resampleData(source, sink=None, orientation=None, dataSizeSink=None, resolutionSource=(.91, .91, 8.3), resolutionSink=(25, 25, 25), processingDirectory=bq3d.config.temp_dir, processes=bq3d.config.processes, cleanup=True, interpolation='linear', **kwargs): """Resample data of source in resolution and orientation Arguments: source (str or array): image to be resampled sink (str or None): destination of resampled image orientation (tuple): orientation specified by permuation and change in sign of (1,2,3) dataSizeSink (tuple or None): target size of the resampled image resolutionSource (tuple): resolution of the source image (in length per pixel) resolutionSink (tuple): resolution of the resampled image (in length per pixel) processingDirectory (str or None): directory in which to perform resmapling in parallel, None a temporary directry will be created processes (int): number of processes to use for parallel resampling cleanup (bool): remove temporary files interpolation (str): method to use for interpolating to the resmapled image Returns: (array or str): data or file name of resampled image Notes: * resolutions are assumed to be given for the axes of the intrinsic orientation of the data and reference as when viewed by matplotlib or ImageJ * orientation: permuation of 1,2,3 with potential sign, indicating which axes map onto the reference axes, a negative sign indicates reversal of that particular axes * only a minimal set of information to detremine the resampling parameter has to be given, e.g. dataSizeSource and dataSizeSink """ log.info(f'interpolation method: {interpolation}') log.info(f'Number of processes: {processes}') interpolation = parse_interpolation(interpolation) dataSizeSource = io.dataSize(source) if isinstance(dataSizeSink, str): dataSizeSink = io.dataSize(dataSizeSink) # orient actual resolutions onto reference resolution dataSizeSource, dataSizeSink, resolutionSource, resolutionSink = resampleDataSize( dataSizeSource=dataSizeSource, dataSizeSink=dataSizeSink, resolutionSource=resolutionSource, resolutionSink=resolutionSink, orientation=orientation) dataSizeSinkI = orientDataSizeInverse(dataSizeSink, orientation) # setup intermediate output if processingDirectory is None: processingDirectory = tempfile.mkdtemp() else: io.createDirectory(processingDirectory) resampledXYFile = os.path.join(processingDirectory, 'resampleXY.tif') data_type = io.getDataType(source) resampledXY = io.empty(resampledXYFile, dtype=data_type, shape=(dataSizeSource[0], dataSizeSinkI[1], dataSizeSinkI[2]), imagej=True) nZ = dataSizeSource[0] # resample in XY # chunk for each process Zlist = list(range(nZ)) chunks = [Zlist[i::processes] for i in range(processes)] argdata = [(source, resampledXYFile, dataSizeSinkI, interpolation, chunk) for chunk in chunks] if processes == 1: _resampleXYParallel(argdata[0]) else: pool = multiprocessing.Pool(processes=processes) pool.map(_resampleXYParallel, argdata) pool.close() # rescale in z resampledXY = io.readData(resampledXYFile) resampledData = numpy.zeros( (dataSizeSinkI[0], dataSizeSinkI[1], dataSizeSinkI[2]), dtype=data_type) for i in range(dataSizeSinkI[1]): # faster if iterate over y if i % 50 == 0: log.verbose( ("resampleData: Z: Resampling %d/%d" % (i, dataSizeSinkI[0]))) resampledData[:, i] = cv2.resize(resampledXY[:, i], (dataSizeSinkI[2], dataSizeSinkI[0]), interpolation=interpolation) if cleanup: shutil.rmtree(processingDirectory) if not orientation is None: # reorient per = orientationToPermuation(orientation) resampledData = resampledData.transpose(per) # reverse orientation after permuting e.g. (-2,1) brings axis 2 to first axis and we can reorder there if orientation[0] < 0: resampledData = resampledData[::-1, :, :] if orientation[1] < 0: resampledData = resampledData[:, ::-1, :] if orientation[2] < 0: resampledData = resampledData[:, :, ::-1] log.verbose("resampleData: resampled data size: " + str(resampledData.shape)) return io.writeData(sink, resampledData)
def copyData(source, sink, processes=1, x=None, y=None, z=None, **kwargs): """Copy a data file from source to sink when for entire list of files Arguments: source (str): file name pattern of source sink (str): file name pattern of sink processes (int): number of processes to be used when writing files in parallel Returns: str: file name pattern of the copy Notes: TODO: replace cropData with this. currently still using because cropData is more flexible TODO: could simplify by not splitting up the regex files """ if isinstance(source, Path): source = source.as_posix() if isinstance(sink, Path): sink = sink.as_posix() fp, fl = readFileList(source, z=z) # crops is z by only reading files in range out_type = io.dataFileNameToType(sink) if out_type == 'FileList': # setup inputs for pool files = [] z_idx = [] for i, fn in enumerate(fl): files.append(os.path.join(fp, fn)) z_idx.append(i) f_chunks = [files[i::processes] for i in range(processes)] z_chunks = [z_idx[i::processes] for i in range(processes)] # setup pool args = [(sources, z_chunks[i], sink, x, y) for i, sources in enumerate(f_chunks)] if processes == 1: _parallelCopyToFileList(*args) else: pool = multiprocessing.Pool(processes) pool.map(_parallelCopyToFileList, args) pool.close() elif out_type == 'TIF': # get datasize Zsize, Ysize, Xsize = io.dataSize(source) # cropped size Xsize = io.toDataSize(Xsize, r=x) Ysize = io.toDataSize(Ysize, r=y) Zsize = io.toDataSize(Zsize, r=z) # setup inputs for pool data_type = getDataType(os.path.join(fp, fl[0])) files = [os.path.join(fp, i) for i in fl] idxs = list(range(len(files))) z_f_chunks = [files[i::processes] for i in range(processes)] z_i_chunks = [idxs[i::processes] for i in range(processes)] im = io.empty(sink, dtype=data_type, shape=(Zsize, Ysize, Xsize)) args = [(z_f_chunks[i], idxs, sink, x, y) for i, idxs in enumerate(z_i_chunks)] # setup pool if processes == 1: _parallelCopyToTif(*args) else: pool = multiprocessing.Pool(processes) pool.map(_parallelCopyToTif, args) pool.close() return sink