def detect_cells(source, flow, processes=config.processes, log_level='info', **parameter): """Detect cells in data This is a main script to start running the cell detection. Arguments: source (str or array): Image source flow (tuple): sequence of filters to call processes (int): parallel processes to spawn log_level (str): log level to print to console. can use 'verbose', a custom level higher than info but lower than debug. **parameter (dict): parameter for the image procesing sub-routines Returns: """ timer = Timer() set_console_level(log_level) result = process_flow(source, flow=flow, processes=processes, **parameter) timer.log_elapsed("Total Cell Detection") return result
def label_props(img, labels, props): """Joins a list of coordiantes from distributed processing based on their IDs into a single list. Also. Handles duplicate ID entries bbased on Arguments: img (np.array): raw image labels (np.array): labeled image props (list): list of strings where each string is the attibute from region_props to export Returns: array: label coordinates (list of tuples), intensities (list), sizes (list) """ img = io.readData(img) labels = io.readData(labels) timer = Timer() log_parameters(props=props) # get label properties regions = region_props(labels, img) # get relavant properties res = [] for prop in props: prop_res = [] for region in regions: method = getattr(region, prop) prop_res.append(method()) res.append(prop_res) timer.log_elapsed() return res
def add_voxelized_points_from_csv(self, voxels_file): """ add voxelized point data. Rather that entering points lists, this is for data where the number of points have already been counted for each voxel Arguments: voxels_file (str): csv file where first column is the voxel coordinate as '(x,y,z)' with header 'voxel'. each subsequent column contains point counts within that voxel with each column as a seperate data group. """ timer = Timer() df = pd.read_csv(voxels_file, index_col='voxel') df = df.loc[(df != 0).any(axis=1)] # ignore voxels with only 0 values total_vox = df.shape[0] first_group_index = len(self.tree_root.nPoints) s = slice(first_group_index, None) for region in PostOrderIter(self.tree_root): region.add_empty_voxel_groups(ngroups = df.shape[1], index = s) i = 0 for idx,counts in df.iterrows(): if i % 10000 == 0: log.verbose(f'adding voxel {i} of {total_vox}') idx = literal_eval(idx) counts = list(counts) self.get_region_by_voxel(idx).add_voxel_groups(idx, counts, index = s) i += 1 if self.COLLAPSE: for region in PostOrderIter(self.tree_root): region.collapse_nPoints() #TODO: will not properly collapse because no index passed self.calc_point_densities() timer.log_elapsed(prefix='Added voxelized points')
def get_region_info_dataframe(self, index, columns, sink = None, iterator = PreOrderIter): """ format region attributes into a dataframe If an attribute in column is type list or tuple each member will get its own column in the dataframe Arguments: index (str): attribute used for row labels. columns (str or list): attributes to be included in columns. sink (str): file to save dataframe too. iterator (Object): `anynode` iterator to use when populating dataframe. Will determine order of entries. Returns: pandas.DataFrame: if sink, will return filename of sink. """ timer = Timer() if not isinstance(columns, list): columns = [columns] row_labels = [] data = [] # get column and row attributes by region and merge into list or lists for region in iterator(self.tree_root): row_labels.append(getattr(region, index, None)) region_attrs = [] for att in columns: value = getattr(region, att, None) if isinstance(value, (list, tuple)): region_attrs.extend(value) else: region_attrs.append(value) data.append(region_attrs) # create column labels. If attribue has a lenght, duplicate column headers will be added. col_labels = [] for col in columns: att = getattr(self.tree_root, col, None) if isinstance(att, (list, tuple)): col_labels.extend([f'{col}.{i}' for i in range(len(att))]) else: col_labels.append(col) data_df = pd.DataFrame(data=data, index=row_labels, columns=col_labels) data_df.index.name = index timer.log_elapsed(prefix='Generated dataframe for regions') if sink: return io.writeData(sink, data_df) else: return data_df
def add_point_groups(self, points_sources): """ add point counts to each region with voxel info Points will be added to self.voxels as a list at each coordinate making up the region. Point density by group will added to self.points_density Arguments: points_sources (array, str, list): array, file, or list of files with point coordinates in [(x,y,z),...] """ timer = Timer() if not isinstance(points_sources, list): points_sources = [points_sources] start_group = len(self.tree_root.nPoints) n = len(points_sources) for region in PostOrderIter(self.tree_root): region.add_empty_voxel_groups(ngroups = n) self.backround.add_empty_voxel_groups(ngroups = n) for group, points in enumerate(points_sources): group_index = start_group + group coords = io.readPoints(points).astype(int) for i in coords: i = tuple(i) try: self.get_region_by_voxel(i).add_point(i, group_index) except: # if coord is out of the image it will be included in backgroud self.get_region_by_id(0).nPoints[group_index] += 1 if self.COLLAPSE: for region in PostOrderIter(self.tree_root): region.collapse_nPoints(index = slice(start_group, None)) # calculate densities self.calc_point_densities() timer.log_elapsed(prefix='Added points group')
def run(self): """Calculates output and saves it to self.output. Returns: (array): Filtered image. """ timer = Timer() if self.temp_dir != False: self.set_temp_dir() self.log_parameters() try: self.output = self._generate_output() except Exception as err: if self.temp_dir: self.del_temp_dir() raise err if self.temp_dir and self.cleanup: self.del_temp_dir() timer.log_elapsed() return self.output
def __init__(self, label_image = bq3d.config.labeled_image, annotation_file = bq3d.config.annotations_default_file, collapse = True): # all attributes not listed here will be generated by the populate regions routine below self.image = label_image self.annotation_file = annotation_file self.regions_by_id = {} # dict of int(region_id): region self.regions_by_coord = {} self.tree_root = None self.backround = None self.COLLAPSE = collapse # populate regions timer = Timer() atlas_ext = io.fileExtension(self.annotation_file) if atlas_ext == 'csv': self.populate_regions_from_csv(file = annotation_file) else: ValueError(f'cannot generate regions from from type {atlas_ext}') self._populate_region_coordiantes(self.image) timer.log_elapsed(prefix = 'Atlas initialization')
def processSubStack(flow, output_properties, source, overlap_indices, unique_indices, temp_dir_root): """ Helper to process stack in parallel Args: flow (tuple): images filters to run in sequential order. Entries should be a dict and will be passed to *bq3d.image_filters.filter_image*. The input image to each filter will the be output of the pevious filter. output_properties: (list): properties to include in output. See label_properties.region_props for more info source (str): path to image file to analyse. overlap_indices (tuple or list): list of indices as [start,stop] along each axis to analyse. unique_indices (tuple or list): list of indices as [start,stop] along each axis corresponding to the non-overlapping portion of the image being analyzed. temp_dir (str): temp dir to be used for processing. Returns: """ timer = Timer() zRng, yRng, xRng = overlap_indices log.info(f'chunk ranges: z= {zRng}, y= {yRng}, x = {xRng}') #memMap routine temp_dir = unique_temp_dir('run', path = temp_dir_root) if not os.path.exists(temp_dir): os.mkdir(temp_dir) mmapFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif' log.info('Creating memory mapped substack at: {}'.format(mmapFile)) img = io.copyData(source, mmapFile, x=xRng, y=yRng, z=zRng, returnMemmap=True) rawFile = os.path.join(temp_dir, str(uuid.uuid4())) + '.tif' log.info('Creating raw substack at: {}'.format(rawFile)) raw = io.copyData(img.filename, rawFile, returnMemmap=True) # if a flow filtered_im = img for p in flow: params = dict(p) filter = params.pop('filter') if 'save' in params: save = params.pop('save') else: save = False filtered_im = filter_image(filter, filtered_im, temp_dir_root = temp_dir, **params) # save intermediate output if save: log.info(f'Saving output to {save}') h, ext, dfmt = splitFileExpression(save) for z in range(*zRng): fname = h + (dfmt % z) + ext if not os.path.isfile(fname): io.empty(fname, io.dataSize(source)[1:], filtered_im.dtype) unique = filtered_im[unique_slice(overlap_indices, unique_indices)] io.writeData(save, unique, substack=unique_indices) # get label properties and return if output_properties: props = label_props(raw, filtered_im, output_properties) else: props = [] shutil.rmtree(temp_dir, ignore_errors=True) timer.log_elapsed(prefix='Processed chunk') return props
def get_voxel_info_dataframe(self, columns, ignore_empty = False, sink=None, iterator=PreOrderIter): """ format voxel info with corresponding region attributes into a dataframe If an attribute in column is type list or tuple each member will get its own column in the dataframe Rows will always be by voxel. Arguments: columns (str or list): attributes to be included in columns. 'nPoints' will return point counts by voxel. ignore_empty (bool): only return voxels containing points sink (str): file to save dataframe too. iterator (Object): `anynode` iterator to use when populating dataframe. Will determine order of entries. Returns: pandasDataFrame: if sink, will return filename of sink. """ timer = Timer() if not isinstance(columns, list): columns = [columns] row_labels = [] data = [] # get column and row attributes by region and merge into list or list s for region in iterator(self.tree_root): print(f'Adding region: {region.id}') for vox, points in list(region.voxels.items()): if ignore_empty: if sum(points) == 0: continue row_labels.append(vox) region_attrs = [vox[0],vox[1],vox[2]] for att in columns: if att == 'nPoints': value = points else: value = getattr(region, att, None) if isinstance(value, (list, tuple)): region_attrs.extend(value) else: region_attrs.append(value) data.append(region_attrs) # create column labels. If attribue has a lenght, duplicate column headers will be added. col_labels = ['x','y','z'] for col in columns: att = getattr(self.tree_root, col, None) # voxel points list will be same lenght as nPoints if isinstance(att, (list, tuple)): col = [f'{col}{i}' for i in range(len(att))] col_labels.extend(col) else: col_labels.extend([col]) data_df = pd.DataFrame(data=data, index=row_labels, columns=col_labels) data_df.index.name = 'voxel' timer.log_elapsed(prefix='Generated dataframe for voxels') if sink: return io.writeData(sink, data_df) else: return data_df