def process_image(input_img_path, output_img_path, projection, method): # Grab a free process with ProcessManager.process(): # initalize biowriter and bioreader with BioReader(input_img_path, max_workers=ProcessManager._active_threads) as br, \ BioWriter(output_img_path, metadata=br.metadata, max_workers=ProcessManager._active_threads) as bw: # output image is 2d bw.Z = 1 # iterate along the x,y direction for x in range(0, br.X, tile_size): x_max = min([br.X, x + tile_size]) for y in range(0, br.Y, tile_size): y_max = min([br.Y, y + tile_size]) ProcessManager.submit_thread(projection, br, bw, (x, x_max), (y, y_max), method=method) ProcessManager.join_threads()
def write_slide(self): with ProcessManager.process(f'{self.base_path} - {self.output_depth}'): ProcessManager.submit_thread(self._write_slide) ProcessManager.join_threads()
def image_to_zarr(inp_image: Path, out_dir: Path) -> None: with ProcessManager.process(): with BioReader(inp_image) as br: # Loop through timepoints for t in range(br.T): # Loop through channels for c in range(br.C): extension = "".join([ suffix for suffix in inp_image.suffixes[-2:] if len(suffix) < 5 ]) out_path = out_dir.joinpath( inp_image.name.replace(extension, FILE_EXT)) if br.C > 1: out_path = out_dir.joinpath( out_path.name.replace(FILE_EXT, f"_c{c}" + FILE_EXT)) if br.T > 1: out_path = out_dir.joinpath( out_path.name.replace(FILE_EXT, f"_t{t}" + FILE_EXT)) with BioWriter( out_path, max_workers=ProcessManager._active_threads, metadata=br.metadata, ) as bw: bw.C = 1 bw.T = 1 bw.channel_names = [br.channel_names[c]] # Loop through z-slices for z in range(br.Z): # Loop across the length of the image for y in range(0, br.Y, TILE_SIZE): y_max = min([br.Y, y + TILE_SIZE]) bw.max_workers = ProcessManager._active_threads br.max_workers = ProcessManager._active_threads # Loop across the depth of the image for x in range(0, br.X, TILE_SIZE): x_max = min([br.X, x + TILE_SIZE]) bw[y:y_max, x:x_max, z:z + 1, 0, 0] = br[y:y_max, x:x_max, z:z + 1, c, t]
def assemble_image(vector_path: pathlib.Path, out_path: pathlib.Path, depth: int) -> None: """Assemble a 2d or 3d image This method assembles one image from one stitching vector. It can assemble both 2d and z-stacked 3d images It is intended to run as a process to parallelize stitching of multiple images. The basic approach to stitching is: 1. Parse the stitching vector and abstract the image dimensions 2. Generate a thread for each subsection (supertile) of an image. Args: vector_path: Path to the stitching vector out_path: Path to the output directory depth: depth of the input images """ # Grab a free process with ProcessManager.process(): # Parse the stitching vector parsed_vector = _parse_stitch(vector_path, timesliceNaming) # Initialize the output image with BioReader(parsed_vector['filePos'][0]['file']) as br: bw = BioWriter(out_path.joinpath(parsed_vector['name']), metadata=br.metadata, max_workers=ProcessManager._active_threads) bw.x = parsed_vector['width'] bw.y = parsed_vector['height'] bw.z = depth # Assemble the images ProcessManager.log(f'Begin assembly') for z in range(depth): ProcessManager.log(f'Assembling Z position : {z}') for x in range(0, parsed_vector['width'], chunk_size): X_range = min(x + chunk_size, parsed_vector['width'] ) # max x-pixel index in the assembled image for y in range(0, parsed_vector['height'], chunk_size): Y_range = min(y + chunk_size, parsed_vector['height'] ) # max y-pixel index in the assembled image ProcessManager.submit_thread(make_tile, x, X_range, y, Y_range, z, parsed_vector, bw) ProcessManager.join_threads() bw.close()
def unshade_batch(files: typing.List[Path], out_dir: Path, brightfield: Path, darkfield: Path, photobleach: typing.Optional[Path] = None): if photobleach != None: with open(photobleach, 'r') as f: reader = csv.reader(f) photo_offset = { line[0]: float(line[1]) for line in reader if line[0] != 'file' } offset = np.mean([o for o in photo_offset.values()]) else: offset = None with ProcessManager.process(): with BioReader(brightfield, max_workers=2) as bf: brightfield_image = bf[:, :, :, 0, 0].squeeze() with BioReader(darkfield, max_workers=2) as df: darkfield_image = df[:, :, :, 0, 0].squeeze() threads = [] for file in files: if photobleach != None: pb = photo_offset[file['file']] else: pb = None ProcessManager.submit_thread(unshade_image, file['file'], out_dir, brightfield_image, darkfield_image, pb, offset) ProcessManager.join_threads()
def basic(files: typing.List[Path], out_dir: Path, metadata_dir: typing.Optional[Path] = None, darkfield: bool = False, photobleach: bool = False): # Try to infer a filename try: pattern = infer_pattern([f['file'].name for f in files]) fp = FilePattern(files[0]['file'].parent,pattern) base_output = fp.output_name() # Fallback to the first filename except: base_output = files[0]['file'].name extension = ''.join(files[0]['file'].suffixes) with ProcessManager.process(base_output): # Load files and sort ProcessManager.log('Loading and sorting images...') img_stk,X,Y = _get_resized_image_stack(files) img_stk_sort = np.sort(img_stk) # Initialize options new_options = _initialize_options(img_stk_sort,darkfield,OPTIONS) # Initialize flatfield/darkfield matrices ProcessManager.log('Beginning flatfield estimation') flatfield_old = np.ones((new_options['size'],new_options['size']),dtype=np.float64) darkfield_old = np.random.normal(size=(new_options['size'],new_options['size'])).astype(np.float64) # Optimize until the change in values is below tolerance or a maximum number of iterations is reached for w in range(new_options['max_reweight_iterations']): # Optimize using inexact augmented Legrangian multiplier method using L1 loss A, E1, A_offset = _inexact_alm_l1(copy.deepcopy(img_stk_sort),new_options) # Calculate the flatfield/darkfield images and update training weights flatfield, darkfield, new_options = _get_flatfield_and_reweight(A,E1,A_offset,new_options) # Calculate the change in flatfield and darkfield images between iterations mad_flat = np.sum(np.abs(flatfield-flatfield_old))/np.sum(np.abs(flatfield_old)) temp_diff = np.sum(np.abs(darkfield - darkfield_old)) if temp_diff < 10**-7: mad_dark =0 else: mad_dark = temp_diff/np.max(np.sum(np.abs(darkfield_old)),initial=10**-6) flatfield_old = flatfield darkfield_old = darkfield # Stop optimizing if the change in flatfield/darkfield is below threshold ProcessManager.log('Iteration {} loss: {}'.format(w+1,mad_flat)) if np.max(mad_flat,initial=mad_dark) < new_options['reweight_tol']: break # Calculate photobleaching effects if specified if photobleach: pb = _get_photobleach(copy.deepcopy(img_stk),flatfield,darkfield) # Resize images back to original image size ProcessManager.log('Saving outputs...') flatfield = cv2.resize(flatfield,(Y,X),interpolation=cv2.INTER_CUBIC).astype(np.float32) if new_options['darkfield']: darkfield = cv2.resize(darkfield,(Y,X),interpolation=cv2.INTER_CUBIC).astype(np.float32) # Export the flatfield image as a tiled tiff flatfield_out = base_output.replace(extension,'_flatfield' + extension) with BioReader(files[0]['file'],max_workers=2) as br: metadata = br.metadata with BioWriter(out_dir.joinpath(flatfield_out),metadata=metadata,max_workers=2) as bw: bw.dtype = np.float32 bw.x = X bw.y = Y bw[:] = np.reshape(flatfield,(Y,X,1,1,1)) # Export the darkfield image as a tiled tiff if new_options['darkfield']: darkfield_out = base_output.replace(extension,'_darkfield' + extension) with BioWriter(out_dir.joinpath(darkfield_out),metadata=metadata,max_workers=2) as bw: bw.dtype = np.float32 bw.x = X bw.y = Y bw[:] = np.reshape(darkfield,(Y,X,1,1,1)) # Export the photobleaching components as csv if photobleach: offsets_out = base_output.replace(extension,'_offsets.csv') with open(metadata_dir.joinpath(offsets_out),'w') as fw: fw.write('file,offset\n') for f,o in zip(files,pb[0,:].tolist()): fw.write("{},{}\n".format(f,o))
def extract_fovs(file_path: Path, out_path: Path): """ Extract individual FOVs from a czi file When CZI files are loaded by BioFormats, it will generally try to mosaic images together by stage position if the image was captured with the intention of mosaicing images together. At the time this function was written, there was no clear way of extracting individual FOVs so this algorithm was created. Every field of view in each z-slice, channel, and timepoint contained in a CZI file is saved as an individual image. Args: file_path (Path): Path to CZI file out_path (Path): Path to output directory """ with ProcessManager.process(file_path.name): logger.info('Starting extraction from ' + str(file_path) + '...') if Path(file_path).suffix != '.czi': TypeError("Path must be to a czi file.") base_name = Path(file_path.name).stem # Load files without mosaicing czi = czifile.CziFile(file_path,detectmosaic=False) subblocks = [s for s in czi.filtered_subblock_directory if s.mosaic_index is not None] ind = {'X': [], 'Y': [], 'Z': [], 'C': [], 'T': [], 'Row': [], 'Col': []} # Get the indices of each FOV for s in subblocks: scene = [dim.start for dim in s.dimension_entries if dim.dimension=='S'] if scene is not None and scene[0] != 0: continue for dim in s.dimension_entries: if dim.dimension=='X': ind['X'].append(dim.start) elif dim.dimension=='Y': ind['Y'].append(dim.start) elif dim.dimension=='Z': ind['Z'].append(dim.start) elif dim.dimension=='C': ind['C'].append(dim.start) elif dim.dimension=='T': ind['T'].append(dim.start) row_conv = {y:row for (y,row) in zip(np.unique(np.sort(ind['Y'])),range(0,len(np.unique(ind['Y']))))} col_conv = {x:col for (x,col) in zip(np.unique(np.sort(ind['X'])),range(0,len(np.unique(ind['X']))))} ind['Row'] = [row_conv[y] for y in ind['Y']] ind['Col'] = [col_conv[x] for x in ind['X']] with BioReader(file_path) as br: metadata = br.metadata chan_names = br.cnames for s,i in zip(subblocks,range(0,len(subblocks))): Z = None if len(ind['Z'])==0 else ind['Z'][i] C = None if len(ind['C'])==0 else ind['C'][i] T = None if len(ind['T'])==0 else ind['T'][i] out_file_path = out_path.joinpath(_get_image_name(base_name, row=ind['Row'][i], col=ind['Col'][i], Z=Z, C=C, T=T)) dims = [_get_image_dim(s,'Y'), _get_image_dim(s,'X'), _get_image_dim(s,'Z'), _get_image_dim(s,'C'), _get_image_dim(s,'T')] data = s.data_segment().data().reshape(dims) write_thread(out_file_path, data, metadata, chan_names[C])
def _merge_layers(input_files,output_path): with ProcessManager.process(output_path.name): # Get the number of layers to stack z_size = 0 for f in input_files: with BioReader(f['file']) as br: z_size += br.z # Get some basic info about the files to stack with BioReader(input_files[0]['file']) as br: # Get the physical z-distance if available, set to physical x if not ps_z = br.ps_z # If the z-distances are undefined, average the x and y together if None in ps_z: # Get the size and units for x and y x_val,x_units = br.ps_x y_val,y_units = br.ps_y # Convert x and y values to the same units and average z_val = (x_val*UNITS[x_units] + y_val*UNITS[y_units])/2 # Set z units to the smaller of the units between x and y z_units = x_units if UNITS[x_units] < UNITS[y_units] else y_units # Convert z to the proper unit scale z_val /= UNITS[z_units] ps_z = (z_val,z_units) ProcessManager.log('Could not find physical z-size. Using the average of x & y {}.'.format(ps_z)) # Hold a reference to the metadata once the file gets closed metadata = br.metadata # Create the output file within a context manager with BioWriter(output_path,metadata=metadata,max_workers=ProcessManager._active_threads) as bw: # Adjust the dimensions before writing bw.z = z_size bw.ps_z = ps_z # ZIndex tracking for the output file zi = 0 # Start stacking for file in input_files: # Open an image with BioReader(file['file'],max_workers=ProcessManager._active_threads) as br: # Open z-layers one at a time for z in range(br.z): # Use tiled reading in x&y to conserve memory # At most, [chunk_size, chunk_size] pixels are loaded for xs in range(0,br.x,chunk_size): xe = min([br.x,xs + chunk_size]) for ys in range(0,br.y,chunk_size): ye = min([br.y,ys + chunk_size]) bw[ys:ye,xs:xe,zi:zi+1,...] = br[ys:ye,xs:xe,z:z+1,...] zi += 1 # update the BioWriter in case the ProcessManager found more threads bw.max_workers = ProcessManager._active_threads