def masker(animal, channel, downsample, scale): """ Main method that starts the cleaning/rotating process. :param animal: prep_id of the animal we are working on. :param channel: channel {1,2,3} :param flip: flip or flop or nothing :param rotation: usually 1 for rotating 90 degrees :param full: resolution, either full or thumbnail :return: nothing, writes to disk the cleaned image """ sqlController = SqlController(animal) fileLocationManager = FileLocationManager(animal) channel_dir = 'CH{}'.format(channel) CLEANED = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_cleaned') INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail') MASKS = os.path.join(fileLocationManager.prep, 'thumbnail_masked') os.makedirs(CLEANED, exist_ok=True) width = sqlController.scan_run.width height = sqlController.scan_run.height rotation = sqlController.scan_run.rotation flip = sqlController.scan_run.flip max_width = int(width * SCALING_FACTOR) max_height = int(height * SCALING_FACTOR) stain = sqlController.histology.counterstain if channel == 1: sqlController.set_task(animal, CLEAN_CHANNEL_1_THUMBNAIL_WITH_MASK) if not downsample: CLEANED = os.path.join(fileLocationManager.prep, channel_dir, 'full_cleaned') os.makedirs(CLEANED, exist_ok=True) INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full') MASKS = os.path.join(fileLocationManager.prep, 'full_masked') max_width = width max_height = height error = test_dir(animal, INPUT, downsample, same_size=False) if len(error) > 0: print(error) sys.exit() files = sorted(os.listdir(INPUT)) progress_id = sqlController.get_progress_id(downsample, channel, 'CLEAN') sqlController.set_task(animal, progress_id) file_keys = [] for file in files: infile = os.path.join(INPUT, file) outpath = os.path.join(CLEANED, file) if os.path.exists(outpath): continue maskfile = os.path.join(MASKS, file) if 'thion' in stain.lower(): print('Not implemented.') #fixed = fix_thion(infile, mask, maskfile, logger, rotation, flip, max_width, max_height) else: file_keys.append([ infile, outpath, maskfile, rotation, flip, max_width, max_height, scale ]) start = timer() workers = 4 # this is the upper limit. More than this and it crashes. print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(fix_ntb, sorted(file_keys)) end = timer() print(f'Create cleaned files took {end - start} seconds total', end="\t") print(f' { (end - start)/len(files)} per file')
def create_neuroglancer(animal, channel, downsample, debug=False): fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) channel_dir = f'CH{channel}' channel_outdir = f'C{channel}T_rechunkme' INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned') db_resolution = sqlController.scan_run.resolution resolution = int(db_resolution * 1000 / SCALING_FACTOR) workers, _ = get_cpus() chunks = calculate_chunks(downsample, -1) progress_id = sqlController.get_progress_id(downsample, channel, 'NEUROGLANCER') sqlController.session.close() if not downsample: INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned') channel_outdir = f'C{channel}_rechunkme' sqlController.set_task(animal, progress_id) if 'thion' in sqlController.histology.counterstain: sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_2_FULL_RES) sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_3_FULL_RES) resolution = int(db_resolution * 1000) OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') error = test_dir(animal, INPUT, downsample, same_size=True) if len(error) > 0 and not debug: print(error) sys.exit() os.makedirs(OUTPUT_DIR, exist_ok=True) files = sorted(os.listdir(INPUT)) midpoint = len(files) // 2 midfilepath = os.path.join(INPUT, files[midpoint]) midfile = io.imread(midfilepath, img_num=0) height = midfile.shape[0] width = midfile.shape[1] num_channels = midfile.shape[2] if len(midfile.shape) > 2 else 1 file_keys = [] scales = (resolution, resolution, 20000) volume_size = (width, height, len(files)) print('Volume shape:', volume_size) ng = NumpyToNeuroglancer(animal, None, scales, 'image', midfile.dtype, num_channels=num_channels, chunk_size=chunks) ng.init_precomputed(OUTPUT_DIR, volume_size, progress_id=progress_id) for i, f in enumerate(files): filepath = os.path.join(INPUT, f) file_keys.append([i,filepath]) #ng.process_3channel([i, filepath]) #sys.exit() start = timer() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: if num_channels == 1: executor.map(ng.process_image, sorted(file_keys)) else: executor.map(ng.process_3channel, sorted(file_keys)) end = timer() print(f'Create volume method took {end - start} seconds') ng.precomputed_vol.cache.flush()
def make_histogram(animal, channel): """ This method creates an individual histogram for each tif file by channel. Args: animal: the prep id of the animal channel: the channel of the stack to process {1,2,3} Returns: nothing """ logger = get_logger(animal) fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'thumbnail') MASK_INPUT = fileLocationManager.thumbnail_masked tifs = sqlController.get_sections(animal, channel) error = test_dir(animal, INPUT, downsample=True, same_size=False) if len(tifs) == 0: error += " No sections in the database" if len(error) > 0: print(error) sys.exit() ch_dir = f'CH{channel}' OUTPUT = os.path.join(fileLocationManager.histogram, ch_dir) os.makedirs(OUTPUT, exist_ok=True) progress_id = sqlController.get_progress_id(True, channel, 'HISTOGRAM') sqlController.set_task(animal, progress_id) for i, tif in enumerate(tqdm(tifs)): filename = str(i).zfill(3) + '.tif' input_path = os.path.join(INPUT, filename) mask_path = os.path.join(MASK_INPUT, filename) output_path = os.path.join(OUTPUT, os.path.splitext(tif.file_name)[0] + '.png') if not os.path.exists(input_path): print('Input tif does not exist', input_path) continue if os.path.exists(output_path): continue try: img = io.imread(input_path) except: logger.warning(f'Could not open {input_path}') continue try: mask = io.imread(mask_path) except: logger.warning(f'Could not open {mask_path}') continue img = cv2.bitwise_and(img, img, mask=mask) if img.shape[0] * img.shape[1] > 1000000000: scale = 1 / float(2) img = img[::int(1. / scale), ::int(1. / scale)] try: flat = img.flatten() except: logger.warning(f'Could not flat {input_path}') continue fig = plt.figure() plt.rcParams['figure.figsize'] = [10, 6] plt.hist(flat, flat.max(), [0, 10000], color=COLORS[channel]) plt.style.use('ggplot') plt.yscale('log') plt.grid(axis='y', alpha=0.75) plt.xlabel('Value') plt.ylabel('Frequency') plt.title(f'{tif.file_name} @16bit') plt.close() fig.savefig(output_path, bbox_inches='tight')
def run_offsets(animal, transforms, channel, downsample, masks, create_csv, allen): """ This gets the dictionary from the above method, and uses the coordinates to feed into the Imagemagick convert program. This method also uses a Pool to spawn multiple processes. Args: animal: the animal transforms: the dictionary of file, coordinates limit: number of jobs Returns: nothing """ fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) channel_dir = 'CH{}'.format(channel) INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_cleaned') OUTPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned') if not downsample: INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_cleaned') OUTPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned') error = test_dir(animal, INPUT, downsample=downsample, same_size=True) if len(error) > 0 and not create_csv: print(error) sys.exit() if masks: INPUT = os.path.join(fileLocationManager.prep, 'rotated_masked') error = test_dir(animal, INPUT, full=False, same_size=True) if len(error) > 0: print(error) sys.exit() OUTPUT = os.path.join(fileLocationManager.prep, 'rotated_aligned_masked') os.makedirs(OUTPUT, exist_ok=True) progress_id = sqlController.get_progress_id(downsample, channel, 'ALIGN') sqlController.set_task(animal, progress_id) warp_transforms = create_warp_transforms(animal, transforms, 'thumbnail', downsample) ordered_transforms = OrderedDict(sorted(warp_transforms.items())) file_keys = [] r90 = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]) for i, (file, T) in enumerate(ordered_transforms.items()): if allen: ROT_DIR = os.path.join(fileLocationManager.root, animal, 'rotations') rotfile = file.replace('tif', 'txt') rotfile = os.path.join(ROT_DIR, rotfile) R_cshl = np.loadtxt(rotfile) R_cshl[0, 2] = R_cshl[0, 2] / 32 R_cshl[1, 2] = R_cshl[1, 2] / 32 R_cshl = R_cshl @ r90 R_cshl = np.linalg.inv(R_cshl) R = T @ R_cshl infile = os.path.join(INPUT, file) outfile = os.path.join(OUTPUT, file) if os.path.exists(outfile) and not create_csv: continue file_keys.append([i, infile, outfile, T]) if create_csv: create_csv_data(animal, file_keys) else: start = timer() workers, _ = get_cpus() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(process_image, sorted(file_keys)) end = timer() print(f'Create cleaned files took {end - start} seconds total', end="\t") print(f' { (end - start)/len(file_keys)} per file') print('Finished')
cmd = [ 'convert', input_path, '-resize', '3.125%', '-compress', 'lzw', output_path ] commands.append(cmd) with Pool(4) as p: p.map(workernoshell, commands) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Work on Animal') parser.add_argument('--animal', help='Enter the animal animal', required=True) parser.add_argument('--channel', help='Enter channel', required=True) args = parser.parse_args() animal = args.animal channel = int(args.channel) make_full_resolution(animal, channel) make_low_resolution(animal, channel) sqlController = SqlController(animal) if channel == 1: sqlController.update_scanrun(sqlController.scan_run.id) progress_id = sqlController.get_progress_id(True, channel, 'TIF') sqlController.set_task(animal, progress_id) progress_id = sqlController.get_progress_id(False, channel, 'TIF') sqlController.set_task(animal, progress_id)