def detect_cells(source,
flow,
processes=config.processes,
log_level='info',
**parameter):
"""Detect cells in data
This is a main script to start running the cell detection.
Arguments:
source (str or array): Image source
flow (tuple): sequence of filters to call
processes (int): parallel processes to spawn
log_level (str): log level to print to console. can use 'verbose',
a custom level higher than info but lower than debug.
**parameter (dict): parameter for the image procesing sub-routines
Returns:
"""
timer = Timer()
set_console_level(log_level)
result = process_flow(source, flow=flow, processes=processes, **parameter)
timer.log_elapsed("Total Cell Detection")
return result
def label_props(img, labels, props):
"""Joins a list of coordiantes from distributed processing based on their IDs into a single list.
Also. Handles duplicate ID entries bbased on
Arguments:
img (np.array): raw image
labels (np.array): labeled image
props (list): list of strings where each string is the attibute from region_props to export
Returns:
array: label coordinates (list of tuples), intensities (list), sizes (list)
"""
img = io.readData(img)
labels = io.readData(labels)
timer = Timer()
log_parameters(props=props)
# get label properties
regions = region_props(labels, img)
# get relavant properties
res = []
for prop in props:
prop_res = []
for region in regions:
method = getattr(region, prop)
prop_res.append(method())
res.append(prop_res)
timer.log_elapsed()
return res
def add_voxelized_points_from_csv(self, voxels_file):
""" add voxelized point data. Rather that entering points lists, this is for data where the number of points have already been counted for each voxel
Arguments:
voxels_file (str): csv file where first column is the voxel coordinate as '(x,y,z)' with header 'voxel'.
each subsequent column contains point counts within that voxel with each column as a seperate data group.
"""
timer = Timer()
df = pd.read_csv(voxels_file, index_col='voxel')
df = df.loc[(df != 0).any(axis=1)] # ignore voxels with only 0 values
total_vox = df.shape[0]
first_group_index = len(self.tree_root.nPoints)
s = slice(first_group_index, None)
for region in PostOrderIter(self.tree_root):
region.add_empty_voxel_groups(ngroups = df.shape[1], index = s)
i = 0
for idx,counts in df.iterrows():
if i % 10000 == 0:
log.verbose(f'adding voxel {i} of {total_vox}')
idx = literal_eval(idx)
counts = list(counts)
self.get_region_by_voxel(idx).add_voxel_groups(idx, counts, index = s)
i += 1
if self.COLLAPSE:
for region in PostOrderIter(self.tree_root):
region.collapse_nPoints() #TODO: will not properly collapse because no index passed
self.calc_point_densities()
timer.log_elapsed(prefix='Added voxelized points')
def get_region_info_dataframe(self, index, columns, sink = None, iterator = PreOrderIter):
""" format region attributes into a dataframe
If an attribute in column is type list or tuple each member will get its own column in the dataframe
Arguments:
index (str): attribute used for row labels.
columns (str or list): attributes to be included in columns.
sink (str): file to save dataframe too.
iterator (Object): `anynode` iterator to use when populating dataframe. Will determine order of entries.
Returns:
pandas.DataFrame: if sink, will return filename of sink.
"""
timer = Timer()
if not isinstance(columns, list):
columns = [columns]
row_labels = []
data = []
# get column and row attributes by region and merge into list or lists
for region in iterator(self.tree_root):
row_labels.append(getattr(region, index, None))
region_attrs = []
for att in columns:
value = getattr(region, att, None)
if isinstance(value, (list, tuple)):
region_attrs.extend(value)
else:
region_attrs.append(value)
data.append(region_attrs)
# create column labels. If attribue has a lenght, duplicate column headers will be added.
col_labels = []
for col in columns:
att = getattr(self.tree_root, col, None)
if isinstance(att, (list, tuple)):
col_labels.extend([f'{col}.{i}' for i in range(len(att))])
else:
col_labels.append(col)
data_df = pd.DataFrame(data=data, index=row_labels, columns=col_labels)
data_df.index.name = index
timer.log_elapsed(prefix='Generated dataframe for regions')
if sink:
return io.writeData(sink, data_df)
else:
return data_df
def add_point_groups(self, points_sources):
""" add point counts to each region with voxel info
Points will be added to self.voxels as a list at each coordinate making up the region.
Point density by group will added to self.points_density
Arguments:
points_sources (array, str, list): array, file, or list of files with point coordinates in [(x,y,z),...]
"""
timer = Timer()
if not isinstance(points_sources, list):
points_sources = [points_sources]
start_group = len(self.tree_root.nPoints)
n = len(points_sources)
for region in PostOrderIter(self.tree_root):
region.add_empty_voxel_groups(ngroups = n)
self.backround.add_empty_voxel_groups(ngroups = n)
for group, points in enumerate(points_sources):
group_index = start_group + group
coords = io.readPoints(points).astype(int)
for i in coords:
i = tuple(i)
try:
self.get_region_by_voxel(i).add_point(i, group_index)
except: # if coord is out of the image it will be included in backgroud
self.get_region_by_id(0).nPoints[group_index] += 1
if self.COLLAPSE:
for region in PostOrderIter(self.tree_root):
region.collapse_nPoints(index = slice(start_group, None))
# calculate densities
self.calc_point_densities()
timer.log_elapsed(prefix='Added points group')
def run(self):
"""Calculates output and saves it to self.output.
Returns:
(array): Filtered image.
"""
timer = Timer()
if self.temp_dir != False:
self.set_temp_dir()
self.log_parameters()
try:
self.output = self._generate_output()
except Exception as err:
if self.temp_dir:
self.del_temp_dir()
raise err
if self.temp_dir and self.cleanup:
self.del_temp_dir()
timer.log_elapsed()
return self.output
def __init__(self, label_image = bq3d.config.labeled_image,
annotation_file = bq3d.config.annotations_default_file,
collapse = True):
# all attributes not listed here will be generated by the populate regions routine below
self.image = label_image
self.annotation_file = annotation_file
self.regions_by_id = {} # dict of int(region_id): region
self.regions_by_coord = {}
self.tree_root = None
self.backround = None
self.COLLAPSE = collapse
# populate regions
timer = Timer()
atlas_ext = io.fileExtension(self.annotation_file)
if atlas_ext == 'csv':
self.populate_regions_from_csv(file = annotation_file)
else:
ValueError(f'cannot generate regions from from type {atlas_ext}')
self._populate_region_coordiantes(self.image)
timer.log_elapsed(prefix = 'Atlas initialization')
def processSubStack(flow, output_properties, source, overlap_indices, unique_indices,
temp_dir_root):
""" Helper to process stack in parallel
Args:
flow (tuple): images filters to run in sequential order.
Entries should be a dict and will be passed to *bq3d.image_filters.filter_image*.
The input image to each filter will the be output of the pevious filter.
output_properties: (list): properties to include in output. See
label_properties.region_props for more info
source (str): path to image file to analyse.
overlap_indices (tuple or list): list of indices as [start,stop] along each axis to analyse.
unique_indices (tuple or list): list of indices as [start,stop] along each axis
corresponding
to the non-overlapping portion of the image being analyzed.
temp_dir (str): temp dir to be used for processing.
Returns:
"""
timer = Timer()
zRng, yRng, xRng = overlap_indices
log.info(f'chunk ranges: z= {zRng}, y= {yRng}, x = {xRng}')
#memMap routine
temp_dir = unique_temp_dir('run', path = temp_dir_root)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
mmapFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif'
log.info('Creating memory mapped substack at: {}'.format(mmapFile))
img = io.copyData(source, mmapFile, x=xRng, y=yRng, z=zRng, returnMemmap=True)
rawFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif'
log.info('Creating raw substack at: {}'.format(rawFile))
raw = io.copyData(img.filename, rawFile, returnMemmap=True)
# if a flow
filtered_im = img
for p in flow:
params = dict(p)
filter = params.pop('filter')
if 'save' in params:
save = params.pop('save')
else:
save = False
filtered_im = filter_image(filter, filtered_im, temp_dir_root = temp_dir, **params)
# save intermediate output
if save:
log.info(f'Saving output to {save}')
h, ext, dfmt = splitFileExpression(save)
for z in range(*zRng):
fname = h + (dfmt % z) + ext
if not os.path.isfile(fname):
io.empty(fname, io.dataSize(source)[1:], filtered_im.dtype)
unique = filtered_im[unique_slice(overlap_indices, unique_indices)]
io.writeData(save, unique, substack=unique_indices)
# get label properties and return
if output_properties:
props = label_props(raw, filtered_im, output_properties)
else:
props = []
shutil.rmtree(temp_dir, ignore_errors=True)
timer.log_elapsed(prefix='Processed chunk')
return props
def get_voxel_info_dataframe(self, columns, ignore_empty = False, sink=None, iterator=PreOrderIter):
""" format voxel info with corresponding region attributes into a dataframe
If an attribute in column is type list or tuple each member will get its own column in the dataframe
Rows will always be by voxel.
Arguments:
columns (str or list): attributes to be included in columns. 'nPoints' will return point counts by voxel.
ignore_empty (bool): only return voxels containing points
sink (str): file to save dataframe too.
iterator (Object): `anynode` iterator to use when populating dataframe. Will determine order of entries.
Returns:
pandasDataFrame: if sink, will return filename of sink.
"""
timer = Timer()
if not isinstance(columns, list):
columns = [columns]
row_labels = []
data = []
# get column and row attributes by region and merge into list or list s
for region in iterator(self.tree_root):
print(f'Adding region: {region.id}')
for vox, points in list(region.voxels.items()):
if ignore_empty:
if sum(points) == 0:
continue
row_labels.append(vox)
region_attrs = [vox[0],vox[1],vox[2]]
for att in columns:
if att == 'nPoints':
value = points
else:
value = getattr(region, att, None)
if isinstance(value, (list, tuple)):
region_attrs.extend(value)
else:
region_attrs.append(value)
data.append(region_attrs)
# create column labels. If attribue has a lenght, duplicate column headers will be added.
col_labels = ['x','y','z']
for col in columns:
att = getattr(self.tree_root, col, None) # voxel points list will be same lenght as nPoints
if isinstance(att, (list, tuple)):
col = [f'{col}{i}' for i in range(len(att))]
col_labels.extend(col)
else:
col_labels.extend([col])
data_df = pd.DataFrame(data=data, index=row_labels, columns=col_labels)
data_df.index.name = 'voxel'
timer.log_elapsed(prefix='Generated dataframe for voxels')
if sink:
return io.writeData(sink, data_df)
else:
return data_df