def main(imgPath: pathlib.Path, stitchPath: pathlib.Path, outDir: pathlib.Path, timesliceNaming: typing.Optional[bool] ) -> None: '''Setup stitching variables/objects''' # Get a list of stitching vectors vectors = list(stitchPath.iterdir()) vectors.sort() # Try to infer a filepattern from the files on disk for faster matching later global fp # make the filepattern global to share between processes try: pattern = filepattern.infer_pattern([f.name for f in imgPath.iterdir()]) logger.info(f'Inferred file pattern: {pattern}') fp = filepattern.FilePattern(imgPath,pattern) # Pattern inference didn't work, so just get a list of files except: logger.info(f'Unable to infer pattern, defaulting to: .*') fp = filepattern.FilePattern(imgPath,'.*') '''Run stitching jobs in separate processes''' ProcessManager.init_processes('main','asmbl') for v in vectors: # Check to see if the file is a valid stitching vector if 'img-global-positions' not in v.name: continue ProcessManager.submit_process(assemble_image,v,outDir) ProcessManager.join_processes()
def main( inpDir: Path, outDir: Path, ) -> None: pattern = ".*.ome.tif" fp = filepattern.FilePattern(inpDir, pattern) files = [f[0] for f in fp] threads = [] with ThreadPoolExecutor(cpu_count()) as executor: for ind, file in enumerate(files): threads.append(executor.submit(validate_and_copy, file, outDir)) done, not_done = wait(threads, timeout=0) logger.info('Copy progress: {:6.2f}%'.format(100 * len(done) / len(threads))) while len(not_done) > 0: done, not_done = wait(threads, timeout=5) logger.info('Copy progress: {:6.2f}%'.format(100 * len(done) / len(threads)))
def main( inpDir: Path, filePattern: str, outDir: Path, ) -> None: """Main execution function""" if filePattern is None: filePattern = ".*" fp = filepattern.FilePattern(inpDir, filePattern) processes = [] with ProcessPoolExecutor(NUM_CPUS) as executor: for files in fp: file = files[0] if filePattern == ".*.fcs": processes.append(executor.submit(fcs_to_feather, file, outDir)) else: processes.append( executor.submit(df_to_feather, file, filePattern, outDir)) remove_files(outDir) logger.info("Finished all processes!")
def main( input_dir: Path, file_pattern: str, output_dir: Path, ): fp = filepattern.FilePattern(input_dir, file_pattern) files = [Path(file[0]['file']).resolve() for file in fp] files = list(filter( lambda file_path: file_path.name.endswith('.ome.tif') or file_path.name.endswith('.ome.zarr'), files )) executor = (ThreadPoolExecutor if utils.USE_GPU else ProcessPoolExecutor)(utils.NUM_THREADS) processes: List[Future[bool]] = list() for in_file in files: with BioReader(in_file) as reader: x_shape, y_shape, z_shape = reader.X, reader.Y, reader.Z metadata = reader.metadata ndims = 2 if z_shape == 1 else 3 out_file = output_dir.joinpath(utils.replace_extension(in_file, extension='_flow.ome.zarr')) init_zarr_file(out_file, ndims, metadata) tile_count = 0 for z in range(0, z_shape, utils.TILE_SIZE): z = None if ndims == 2 else z for y in range(0, y_shape, utils.TILE_SIZE): for x in range(0, x_shape, utils.TILE_SIZE): coordinates = x, y, z device = (tile_count % utils.NUM_THREADS) if utils.USE_GPU else None tile_count += 1 # flow_thread(in_file, out_file, coordinates, device) processes.append(executor.submit( flow_thread, in_file, out_file, coordinates, device, )) done, not_done = wait(processes, 0) while len(not_done) > 0: logger.info(f'Percent complete: {100 * len(done) / len(processes):6.3f}%') for r in done: r.result() done, not_done = wait(processes, 5) executor.shutdown() return
def main(input_dir: pathlib.Path, file_pattern: str, output_dir: pathlib.Path ) -> None: # create the filepattern object fp = filepattern.FilePattern(input_dir,file_pattern) for files in fp(group_by='z'): output_name = fp.output_name(files) output_file = output_dir.joinpath(output_name) ProcessManager.submit_process(_merge_layers,files,output_file) ProcessManager.join_processes()
def main( *, input_dir: Path, file_pattern: str, flow_magnitude_threshold: float, output_dir: Path, ): fp = filepattern.FilePattern(input_dir, file_pattern) files = [Path(files.pop()['file']).resolve() for files in fp] files = list( filter(lambda file_path: str(file_path).endswith('.ome.zarr'), files)) if len(files) == 0: logger.critical('No flow files detected.') return for i, in_path in enumerate(files, start=1): logger.info(f'Processing image ({i}/{len(files)}): {in_path}') vector_to_label( in_path=in_path, flow_magnitude_threshold=flow_magnitude_threshold, output_dir=output_dir, ) return
class LabelToVectorTest(unittest.TestCase): data_path = Path('/data/axle/tests') input_path = data_path.joinpath('input') if input_path.joinpath('images').is_dir(): input_path = input_path.joinpath('images') fp = filepattern.FilePattern(input_path, '.+') infile = next(Path(files.pop()['file']).resolve() for files in fp) with BioReader(infile) as reader: labels = numpy.squeeze(reader[:, :, :, 0, 0]) labels = numpy.reshape( numpy.unique(labels, return_inverse=True)[1], labels.shape) if labels.ndim == 3: labels = numpy.transpose(labels, (2, 0, 1)) toy_labels = numpy.zeros((17, 17), dtype=numpy.uint8) toy_labels[2:15, 2:7] = 1 toy_labels[2:15, 10:15] = 2 def run_benches(self, device): # This is just a hack for running benchmarks num_samples = 10 torch_device = None if device is None else torch.device( f'cuda:{device}') if self.labels.ndim == 2: start = time.time() for _ in range(num_samples): cellpose.dynamics.masks_to_flows(self.labels, use_gpu=(device is not None), device=torch_device) cellpose_time = (time.time() - start) / num_samples else: # cellpose often bugs out on 3d images cellpose_time = float('inf') start = time.time() for _ in range(num_samples): dynamics.masks_to_flows(self.labels, device=device) polus_time = (time.time() - start) / num_samples self.assertLess(polus_time, cellpose_time, f'Polus slower than Cellpose :(') self.assertLess(cellpose_time, polus_time, f'Cellpose slower than Polus :)') return @unittest.skip def test_bench(self): self.run_benches(0) self.run_benches(None) return def test_cellpose_errors(self): cellpose_flows, _ = cellpose.dynamics.masks_to_flows(self.labels, use_gpu=False) self.assertEqual((self.labels.ndim, *self.labels.shape), cellpose_flows.shape, f'cpu shapes were different') if self.labels.ndim == 2: # cellpose often fails on 3d images... cellpose_flows, _ = cellpose.dynamics.masks_to_flows( self.labels, use_gpu=True, device=torch.device(f'cuda:{0}')) self.assertEqual((self.labels.ndim, *self.labels.shape), cellpose_flows.shape, f'cpu shapes were different') return def test_polus_errors(self): polus_flows = dynamics.masks_to_flows(self.labels, device=None) self.assertEqual((self.labels.ndim, *self.labels.shape), polus_flows.shape, f'cpu shapes were different') polus_flows = dynamics.masks_to_flows(self.labels, device=0) self.assertEqual((self.labels.ndim, *self.labels.shape), polus_flows.shape, f'gpu shapes were different') return def image_test(self, image, device): torch_device = None if device is None else torch.device( f'cuda:{device}') cellpose_flows, _ = cellpose.dynamics.masks_to_flows( image, use_gpu=(device is not None), device=torch_device) if image.ndim == 3: # 3d cellpose flows need to be normalized to unit-norm. cellpose_flows = (cellpose_flows / (numpy.linalg.norm(cellpose_flows, axis=0) + 1e-20)) * (image != 0) polus_flows = dynamics.masks_to_flows(image, device=device) self.assertEqual(cellpose_flows.shape, polus_flows.shape, f'flows had different shapes') error = numpy.mean(numpy.square(cellpose_flows - polus_flows)) self.assertLess(error, 0.05, f'error was too large {error:.3f}') return def test_converter(self): self.image_test(self.toy_labels, None) self.image_test(self.toy_labels, 0) self.image_test(self.labels, None) self.image_test(self.labels, 0) return
def main(): # Initialize the logger logging.basicConfig(format='%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s', datefmt='%d-%b-%y %H:%M:%S') logger = logging.getLogger("main") logger.setLevel(logging.INFO) # Setup the Argument parsing logger.info("Parsing arguments...") parser = argparse.ArgumentParser(prog='main', description='Extract individual fields of view from a czi file.') parser.add_argument('--stitchDir', dest='stitch_dir', type=str, help='Stitching vector to recycle', required=True) parser.add_argument('--collectionDir', dest='collection_dir', type=str, help='Image collection to place in new stitching vector', required=True) parser.add_argument('--stitchRegex', dest='stitch_regex', type=str, help='Stitching vector regular expression', required=True) parser.add_argument('--collectionRegex', dest='collection_regex', type=str, help='Image collection regular expression', required=True) parser.add_argument('--groupBy', dest='group_by', type=str, help='Variables to group within a single stitching vector', required=False) parser.add_argument('--outDir', dest='output_dir', type=str, help='The directory in which to save stitching vectors.', required=True) # Get the arguments args = parser.parse_args() stitch_dir = args.stitch_dir collection_dir = args.collection_dir stitch_regex = args.stitch_regex collection_regex = args.collection_regex group_by = args.group_by output_dir = args.output_dir logger.info('stitch_dir = {}'.format(stitch_dir)) logger.info('collection_dir = {}'.format(collection_dir)) logger.info('stitch_regex = {}'.format(stitch_regex)) logger.info('collection_regex = {}'.format(collection_regex)) logger.info('output_dir = {}'.format(output_dir)) # Parse files in the image collection fp = filepattern.FilePattern(collection_dir,collection_regex) # Process group_by variable if group_by==None: group_by = '' for v in 'xyp': if v not in group_by: group_by += v group_by = group_by.lower() group_by = ''.join([g for g in group_by if g in fp.variables]) logger.info('group_by = {}'.format(group_by)) # Loop through the stitching vectors vectors = [v for v in Path(stitch_dir).iterdir() if Path(v).name.startswith('img-global-positions')] vector_count = 1 for vector in vectors: logger.info("Processing vector: {}".format(str(vector.absolute()))) # Parse the stitching vector sp = filepattern.VectorPattern(str(vector.absolute()),stitch_regex) if sp.variables == None: ValueError('Stitching vector pattern must contain variables.') # Grouping variables for files in the image collection file_groups = [v for v in sp.var_order if v not in group_by] vector_groups = [v for v in file_groups if v not in 'xyp'] # Vector output dictionary vector_dict = {} # Loop through lines in the stitching vector, generate new vectors for v in sp(): variables = {key.upper():value for key,value in v[0].items() if key in group_by} file_matches = fp.get_matching(**variables) for f in file_matches: # Get the file writer, create it if it doesn't exist temp_dict = vector_dict for key in vector_groups: if f[key] not in temp_dict.keys(): if vector_groups[-1] != key: temp_dict[f[key]] = {} else: fname = "img-global-positions-{}.txt".format(vector_count) vector_count += 1 out_vars = {key:f[key] for key in vector_groups} logger.info("Creating vector ({}) for variables: {}".format(fname,out_vars)) temp_dict[f[key]] = open(str(Path(output_dir).joinpath(fname).absolute()),'w') temp_dict = temp_dict[f[key]] # If the only grouping variables are positional (xyp), then create an output file fw = temp_dict fw.write("file: {}; corr: {}; position: ({}, {}); grid: ({}, {});\n".format(Path(f['file']).name, v[0]['correlation'], v[0]['posX'], v[0]['posY'], v[0]['gridX'], v[0]['gridY'])) # Close all open stitching vectors close_vectors(vector_dict) logger.info("Plugin completed all operations!")
class VectorToLabelTest(unittest.TestCase): data_path = Path('/data/axle/tests') input_path = data_path.joinpath('input') if input_path.joinpath('images').is_dir(): input_path = input_path.joinpath('images') fp = filepattern.FilePattern(input_path, '.+') infile = next(Path(files.pop()['file']).resolve() for files in fp) with BioReader(infile) as reader: labels = numpy.squeeze(reader[:, :, :, 0, 0]) labels = numpy.reshape( numpy.unique(labels, return_inverse=True)[1], labels.shape) if labels.ndim == 3: labels = numpy.transpose(labels, (2, 0, 1)) device = 0 if torch.cuda.is_available() else None flows = dynamics.masks_to_flows(labels, device=device) toy_labels = numpy.zeros((17, 17), dtype=numpy.uint8) toy_labels[2:15, 2:7] = 1 toy_labels[2:15, 10:15] = 2 toy_flows = dynamics.masks_to_flows(toy_labels, device=device) @unittest.skip def test_benches(self): masks = self.labels > 0 flows = -self.flows * masks # Let's warm up... cellpose_locations = cellpose.dynamics.follow_flows(flows, niter=200, interp=False, use_gpu=True) cellpose.dynamics.get_masks(cellpose_locations, iscell=masks, flows=self.flows, use_gpu=True) polus_locations = dynamics.follow_flows(flows, num_iterations=200, interpolate=False, device=self.device) dynamics.get_masks(polus_locations, is_cell=masks, flows=self.flows, device=self.device) num_samples = 10 start = time.time() for _ in range(num_samples): cellpose_locations = cellpose.dynamics.follow_flows(flows, niter=200, interp=False, use_gpu=True) cellpose.dynamics.get_masks(cellpose_locations, iscell=masks, flows=self.flows, use_gpu=True) cellpose_time = round((time.time() - start) / num_samples, 12) start = time.time() for _ in range(num_samples): polus_locations = dynamics.follow_flows(flows, num_iterations=200, interpolate=False, device=self.device) dynamics.get_masks(polus_locations, is_cell=masks, flows=self.flows, device=self.device) polus_time = round((time.time() - start) / num_samples, 12) self.assertLess(polus_time, cellpose_time, f'Polus slower than Cellpose :(') self.assertLess(cellpose_time, polus_time, f'Cellpose slower than Polus :)') return def recover_masks_test(self, flows, labels): cellpose_locations = cellpose.dynamics.follow_flows( -flows * (labels != 0), niter=200, interp=False, use_gpu=True, ) polus_locations = dynamics.follow_flows( -flows * (labels != 0), num_iterations=200, interpolate=False, device=None, ) polus_masks = dynamics.get_masks( polus_locations, is_cell=(labels != 0), flows=flows, device=self.device, ) polus_masks = dynamics.fill_holes_and_remove_small_masks(polus_masks) self.assertEqual( cellpose_locations.shape, polus_locations.shape, f'locations had different shapes', ) # Some cellpose-locations contain horizontal artifacts but polus-locations do not. # Thus we clamp the error here. If there were a programmatic way to detect those artifacts, # we could deal with the error in a different way and present a fairer test. # As things stand, I believe my implementation to be correct. locations_diff = numpy.clip( numpy.abs(cellpose_locations - polus_locations), 0, 1) self.assertLess(numpy.mean(locations_diff**2), 0.1, f'error in convergent locations was too large...') masks_diff = (polus_masks == 0) != (labels == 0) self.assertLess(numpy.mean(masks_diff), 0.05, f'error in polus masks was too large...') return def test_masks(self): self.recover_masks_test(self.toy_flows, self.toy_labels) self.recover_masks_test(self.flows, self.labels) return
def main(input_dir: pathlib.Path, pyramid_type: str, image_type: str, file_pattern: str, output_dir: pathlib.Path): # Set ProcessManager config and initialize ProcessManager.num_processes(multiprocessing.cpu_count()) ProcessManager.num_threads(2 * ProcessManager.num_processes()) ProcessManager.threads_per_request(1) ProcessManager.init_processes('pyr') logger.info('max concurrent processes = %s', ProcessManager.num_processes()) # Parse the input file directory fp = filepattern.FilePattern(input_dir, file_pattern) group_by = '' if 'z' in fp.variables and pyramid_type == 'Neuroglancer': group_by += 'z' logger.info( 'Stacking images by z-dimension for Neuroglancer precomputed format.' ) elif 'c' in fp.variables and pyramid_type == 'Zarr': group_by += 'c' logger.info('Stacking channels by c-dimension for Zarr format') elif 't' in fp.variables and pyramid_type == 'DeepZoom': group_by += 't' logger.info('Creating time slices by t-dimension for DeepZoom format.') else: logger.info( f'Creating one pyramid for each image in {pyramid_type} format.') depth = 0 depth_max = 0 image_dir = '' processes = [] for files in fp(group_by=group_by): # Create the output name for Neuroglancer format if pyramid_type in ['Neuroglancer', 'Zarr']: try: image_dir = fp.output_name([file for file in files]) except: pass if image_dir in ['', '.*']: image_dir = files[0]['file'].name # Reset the depth depth = 0 depth_max = 0 pyramid_writer = None for file in files: with bfio.BioReader(file['file'], max_workers=1) as br: if pyramid_type == 'Zarr': d_z = br.c else: d_z = br.z depth_max += d_z for z in range(d_z): pyramid_args = { 'base_dir': output_dir.joinpath(image_dir), 'image_path': file['file'], 'image_depth': z, 'output_depth': depth, 'max_output_depth': depth_max, 'image_type': image_type } pw = PyramidWriter[pyramid_type](**pyramid_args) ProcessManager.submit_process(pw.write_slide) depth += 1 if pyramid_type == 'DeepZoom': pw.write_info() if pyramid_type in ['Neuroglancer', 'Zarr']: if image_type == 'segmentation': ProcessManager.join_processes() pw.write_info() ProcessManager.join_processes()
'See the README for more details on what these parameters should be.' ) message = f'Oh no! Something went wrong:\n' + '\n'.join(error_messages) logger.error(message) raise ValueError(message) else: logger.info(f'inputDir = {input_dir}') logger.info(f'filePattern = {pattern}') logger.info(f'groupBy = {group_by}') logger.info(f'cropX = {crop_x}') logger.info(f'cropY = {crop_y}') logger.info(f'cropZ = {crop_z}') logger.info(f'smoothing = {smoothing}') logger.info(f'outputDir = {output_dir}') fp = filepattern.FilePattern(input_dir, pattern) groups = list(fp(group_by=group_by)) for i, group in enumerate(groups): if len(group) == 0: continue file_paths = [files['file'] for files in group] logger.info( f'Working on group {i + 1}/{len(groups)} containing {len(file_paths)} images...' ) crop_image_group( file_paths=file_paths, crop_axes=(crop_x, crop_y, crop_z), smoothing=smoothing, output_dir=output_dir, )
import bfio, filepattern, pathlib, pprint filepath = pathlib.Path(__file__).parent filepath = filepath.joinpath( 'data/Small_Fluorescent_Test_Dataset/image-tiles/') fp = filepattern.FilePattern(filepath, 'img_r00{y}_c00{x}.tif') pprint.pprint(fp.get_matching(X=1)) # for file in fp(group_by='y'): # pprint.pprint(file)
def main(stitch_dir: Path, collection_dir: Path, output_dir: Path, pattern: str): if pattern in [None, ".+", ".*"]: pattern = filepattern.infer_pattern( [f.name for f in collection_dir.iterdir()]) logger.info(f"Inferred filepattern: {pattern}") # Parse files in the image collection fp = filepattern.FilePattern(collection_dir, pattern) # Get valid stitching vectors vectors = [ v for v in Path(stitch_dir).iterdir() if Path(v).name.startswith("img-global-positions") ] """Get filepatterns for each stitching vector This section of code creates a filepattern for each stitching vector, and while traversing the stitching vectors analyzes the patterns to see which values in the filepattern are static or variable within a single stitching vector and across stitching vectors. The `singulars` variable determines which case each variable is: `singulars[v]==-1` when the variable, v, changes within a stitching vector. `singulars[v]==None` when the variable, v, changes across stitching vectors. `singulars[v]==int` when the variable, v, doesn't change. The variables that change across stitching vectors are grouping variables for the filepattern iterator. """ singulars = {} vps = {} for vector in vectors: vps[vector.name] = filepattern.VectorPattern(vector, pattern) for variable in vps[vector.name].variables: if variable not in singulars.keys(): if len(vps[vector.name].uniques[variable]) == 1: singulars[variable] = vps[vector.name].uniques[variable] else: singulars[variable] = -1 elif (variable in singulars.keys() and vps[vector.name].uniques[variable] != singulars[variable]): singulars[variable] = None if singulars[variable] != -1 else -1 group_by = "".join([k for k, v in singulars.items() if v == -1]) vector_count = 1 for vector in vectors: logger.info("Processing vector: {}".format(str(vector.absolute()))) sp = vps[vector.name] # Define the variables used in the current vector pattern so that corresponding # files can be located from files in the image collection with filepattern. matching = { k.upper(): sp.uniques[k][0] for k, v in singulars.items() if v is None } vector_groups = [ k for k, v in singulars.items() if v not in [None, -1] ] # Vector output dictionary vector_dict = {} # Loop through lines in the stitching vector, generate new vectors for v in sp(): variables = { key.upper(): value for key, value in v[0].items() if key in group_by } variables.update(matching) for files in fp(**variables): for f in files: # Get the file writer, create it if it doesn't exist temp_dict = vector_dict for key in vector_groups: if f[key] not in temp_dict.keys(): if vector_groups[-1] != key: temp_dict[f[key]] = {} else: fname = "img-global-positions-{}.txt".format( vector_count) vector_count += 1 logger.info( "Creating vector: {}".format(fname)) temp_dict[f[key]] = open( str( Path(output_dir).joinpath( fname).absolute()), "w", ) temp_dict = temp_dict[f[key]] # If the only grouping variables are positional (xyp), then create an output file fw = temp_dict fw.write( "file: {}; corr: {}; position: ({}, {}); grid: ({}, {});\n" .format( Path(f["file"]).name, v[0]["correlation"], v[0]["posX"], v[0]["posY"], v[0]["gridX"], v[0]["gridY"], )) # Close all open stitching vectors close_vectors(vector_dict) logger.info("Plugin completed all operations!")
# Parse the layout layout = [None if l == '' else int(l) for l in layout.split(',')] if len(layout) > 7: layout = layout[:7] # Parse the bounds if bounds != None: bounds = [[None] if l == '' else get_number(l) for l in bounds.split(',')] bounds = bounds[:len(layout)] else: bounds = [[None] for _ in layout] # Parse files fp = filepattern.FilePattern(inpDir, filePattern) # A channel variable is expected, throw an error if it doesn't exist if 'c' not in fp.variables: raise ValueError('A channel variable is expected in the filepattern.') count = 0 for files in fp.iterate(group_by=fp.variables): count += 1 outDirFrame = outDir.joinpath('{}_files'.format(count)) outDirFrame.mkdir() bioreaders = [] threads = [] with ThreadPoolExecutor(max([multiprocessing.cpu_count() // 2,
def main(inpDir: Path, outDir: Path, filePattern: str = None) -> None: """ Turn labels into flow fields. Args: inpDir: Path to the input directory outDir: Path to the output directory """ # Use a gpu if it's available use_gpu = torch.cuda.is_available() if use_gpu: dev = torch.device("cuda") else: dev = torch.device("cpu") logger.info(f'Running on: {dev}') # Determine the number of threads to run on num_threads = max([cpu_count() // 2, 1]) logger.info(f'Number of threads: {num_threads}') # Get all file names in inpDir image collection based on input pattern if filePattern: fp = filepattern.FilePattern(inpDir, filePattern) inpDir_files = [file[0]['file'].name for file in fp()] logger.info('Processing %d labels based on filepattern ' % (len(inpDir_files))) else: inpDir_files = [f.name for f in Path(inpDir).iterdir() if f.is_file()] # Loop through files in inpDir image collection and process processes = [] if use_gpu: executor = ThreadPoolExecutor(num_threads) else: executor = ProcessPoolExecutor(num_threads) for f in inpDir_files: br = BioReader(Path(inpDir).joinpath(f).absolute()) out_file = Path(outDir).joinpath( f.replace('.ome', '_flow.ome').replace('.tif', '.zarr')).absolute() bw = BioWriter(out_file, metadata=br.metadata) bw.C = 4 bw.dtype = np.float32 bw.channel_names = ['cell_probability', 'x', 'y', 'labels'] bw._backend._init_writer() for z in range(br.Z): for x in range(0, br.X, TILE_SIZE): for y in range(0, br.Y, TILE_SIZE): processes.append( executor.submit(flow_thread, Path(inpDir).joinpath(f).absolute(), out_file, use_gpu, dev, x, y, z)) bw.close() br.close() done, not_done = wait(processes, 0) logger.info(f'Percent complete: {100 * len(done) / len(processes):6.3f}%') while len(not_done) > 0: for r in done: r.result() done, not_done = wait(processes, 5) logger.info( f'Percent complete: {100 * len(done) / len(processes):6.3f}%') executor.shutdown()