def make_tifs(animal, channel, njobs): """ This method will: 1. Fetch the sections from the database 2. Yank the tif out of the czi file according to the index and channel with the bioformats tool. 3. Then updates the database with updated meta information Args: animal: the prep id of the animal channel: the channel of the stack to process njobs: number of jobs for parallel computing compression: default is no compression so we can create jp2 files for CSHL. The files get compressed using LZW when running create_preps.py Returns: nothing """ logger = get_logger(animal) fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) INPUT = fileLocationManager.czi OUTPUT = fileLocationManager.tif os.makedirs(OUTPUT, exist_ok=True) sections = sqlController.get_distinct_section_filenames(animal, channel) sqlController.set_task(animal, QC_IS_DONE_ON_SLIDES_IN_WEB_ADMIN) sqlController.set_task(animal, CZI_FILES_ARE_CONVERTED_INTO_NUMBERED_TIFS_FOR_CHANNEL_1) commands = [] for section in sections: input_path = os.path.join(INPUT, section.czi_file) output_path = os.path.join(OUTPUT, section.file_name) cmd = ['/usr/local/share/bftools/bfconvert', '-bigtiff', '-separate', '-series', str(section.scene_index), '-channel', str(section.channel_index), '-nooverwrite', input_path, output_path] if not os.path.exists(input_path): continue if os.path.exists(output_path): continue commands.append(cmd) with Pool(njobs) as p: p.map(workernoshell, commands)
def make_low_resolution(animal, channel): """ Args: takes the full resolution tifs and downsamples them. animal: the prep id of the animal channel: the channel of the stack to process Returns: list of commands """ sqlController = SqlController(animal) if 'thion' in sqlController.histology.counterstain: sqlController.set_task(animal, CREATE_CHANNEL_2_THUMBNAILS) sqlController.set_task(animal, CREATE_CHANNEL_3_THUMBNAILS) fileLocationManager = FileLocationManager(animal) commands = [] INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'full') ##### Check if files in dir are valid error = test_dir(animal, INPUT, downsample=False, same_size=False) if len(error) > 0: print(error) sys.exit() OUTPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'thumbnail') os.makedirs(OUTPUT, exist_ok=True) tifs = sorted(os.listdir(INPUT)) for tif in tifs: input_path = os.path.join(INPUT, tif) output_path = os.path.join(OUTPUT, tif) if os.path.exists(output_path): continue cmd = [ 'convert', input_path, '-resize', '3.125%', '-compress', 'lzw', output_path ] commands.append(cmd) with Pool(4) as p: p.map(workernoshell, commands)
def make_full_resolution(animal, channel): """ Args: animal: the prep id of the animal channel: the channel of the stack to process compress: Use the default LZW compression, otherwise just copy the file with the correct name Returns: list of commands """ fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) if 'thion' in sqlController.histology.counterstain: sqlController.set_task(animal, CREATE_CHANNEL_2_FULL_RES) sqlController.set_task(animal, CREATE_CHANNEL_3_FULL_RES) INPUT = os.path.join(fileLocationManager.tif) ##### Check if files in dir are valid OUTPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'full') os.makedirs(OUTPUT, exist_ok=True) sections = sqlController.get_sections(animal, channel) for section_number, section in enumerate(tqdm(sections)): input_path = os.path.join(INPUT, section.file_name) output_path = os.path.join(OUTPUT, str(section_number).zfill(3) + '.tif') if not os.path.exists(input_path): #print('Input tif does not exist', input_path) continue if os.path.exists(output_path): continue copyfile(input_path, output_path) width, height = get_image_size(input_path) sqlController.update_tif(section.id, width, height)
def masker(animal, channel, downsample, scale): """ Main method that starts the cleaning/rotating process. :param animal: prep_id of the animal we are working on. :param channel: channel {1,2,3} :param flip: flip or flop or nothing :param rotation: usually 1 for rotating 90 degrees :param full: resolution, either full or thumbnail :return: nothing, writes to disk the cleaned image """ sqlController = SqlController(animal) fileLocationManager = FileLocationManager(animal) channel_dir = 'CH{}'.format(channel) CLEANED = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_cleaned') INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail') MASKS = os.path.join(fileLocationManager.prep, 'thumbnail_masked') os.makedirs(CLEANED, exist_ok=True) width = sqlController.scan_run.width height = sqlController.scan_run.height rotation = sqlController.scan_run.rotation flip = sqlController.scan_run.flip max_width = int(width * SCALING_FACTOR) max_height = int(height * SCALING_FACTOR) stain = sqlController.histology.counterstain if channel == 1: sqlController.set_task(animal, CLEAN_CHANNEL_1_THUMBNAIL_WITH_MASK) if not downsample: CLEANED = os.path.join(fileLocationManager.prep, channel_dir, 'full_cleaned') os.makedirs(CLEANED, exist_ok=True) INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full') MASKS = os.path.join(fileLocationManager.prep, 'full_masked') max_width = width max_height = height error = test_dir(animal, INPUT, downsample, same_size=False) if len(error) > 0: print(error) sys.exit() files = sorted(os.listdir(INPUT)) progress_id = sqlController.get_progress_id(downsample, channel, 'CLEAN') sqlController.set_task(animal, progress_id) file_keys = [] for file in files: infile = os.path.join(INPUT, file) outpath = os.path.join(CLEANED, file) if os.path.exists(outpath): continue maskfile = os.path.join(MASKS, file) if 'thion' in stain.lower(): print('Not implemented.') #fixed = fix_thion(infile, mask, maskfile, logger, rotation, flip, max_width, max_height) else: file_keys.append([ infile, outpath, maskfile, rotation, flip, max_width, max_height, scale ]) start = timer() workers = 4 # this is the upper limit. More than this and it crashes. print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(fix_ntb, sorted(file_keys)) end = timer() print(f'Create cleaned files took {end - start} seconds total', end="\t") print(f' { (end - start)/len(files)} per file')
def create_neuroglancer(animal, channel, downsample, debug=False): fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) channel_dir = f'CH{channel}' channel_outdir = f'C{channel}T_rechunkme' INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned') db_resolution = sqlController.scan_run.resolution resolution = int(db_resolution * 1000 / SCALING_FACTOR) workers, _ = get_cpus() chunks = calculate_chunks(downsample, -1) progress_id = sqlController.get_progress_id(downsample, channel, 'NEUROGLANCER') sqlController.session.close() if not downsample: INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned') channel_outdir = f'C{channel}_rechunkme' sqlController.set_task(animal, progress_id) if 'thion' in sqlController.histology.counterstain: sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_2_FULL_RES) sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_3_FULL_RES) resolution = int(db_resolution * 1000) OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') error = test_dir(animal, INPUT, downsample, same_size=True) if len(error) > 0 and not debug: print(error) sys.exit() os.makedirs(OUTPUT_DIR, exist_ok=True) files = sorted(os.listdir(INPUT)) midpoint = len(files) // 2 midfilepath = os.path.join(INPUT, files[midpoint]) midfile = io.imread(midfilepath, img_num=0) height = midfile.shape[0] width = midfile.shape[1] num_channels = midfile.shape[2] if len(midfile.shape) > 2 else 1 file_keys = [] scales = (resolution, resolution, 20000) volume_size = (width, height, len(files)) print('Volume shape:', volume_size) ng = NumpyToNeuroglancer(animal, None, scales, 'image', midfile.dtype, num_channels=num_channels, chunk_size=chunks) ng.init_precomputed(OUTPUT_DIR, volume_size, progress_id=progress_id) for i, f in enumerate(files): filepath = os.path.join(INPUT, f) file_keys.append([i,filepath]) #ng.process_3channel([i, filepath]) #sys.exit() start = timer() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: if num_channels == 1: executor.map(ng.process_image, sorted(file_keys)) else: executor.map(ng.process_3channel, sorted(file_keys)) end = timer() print(f'Create volume method took {end - start} seconds') ng.precomputed_vol.cache.flush()
tif.scene_index = series_index channel_counter += 1 newtif = '{}_S{}_C{}.tif'.format(czi_file, scene_number, channel_counter) newtif = newtif.replace('.czi', '').replace('__', '_') tif.file_name = newtif tif.channel = channel_counter tif.processing_duration = 0 tif.created = time.strftime('%Y-%m-%d %H:%M:%S') session.add(tif) section_number += 1 session.commit() if __name__ == '__main__': # Parsing argument parser = argparse.ArgumentParser(description='Work on Animal') parser.add_argument('--animal', help='Enter the animal', required=True) parser.add_argument('--remove', help='Enter true or false', required=False, default='false') args = parser.parse_args() animal = args.animal remove = bool({'true': True, 'false': False}[str(args.remove).lower()]) make_meta(animal, remove) sqlController = SqlController(animal) sqlController.set_task(animal, SLIDES_ARE_SCANNED) sqlController.set_task(animal, CZI_FILES_ARE_PLACED_ON_BIRDSTORE) sqlController.set_task(animal, CZI_FILES_ARE_SCANNED_TO_GET_METADATA)
def run_offsets(animal, transforms, channel, downsample, masks, create_csv, allen): """ This gets the dictionary from the above method, and uses the coordinates to feed into the Imagemagick convert program. This method also uses a Pool to spawn multiple processes. Args: animal: the animal transforms: the dictionary of file, coordinates limit: number of jobs Returns: nothing """ fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) channel_dir = 'CH{}'.format(channel) INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_cleaned') OUTPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned') if not downsample: INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_cleaned') OUTPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned') error = test_dir(animal, INPUT, downsample=downsample, same_size=True) if len(error) > 0 and not create_csv: print(error) sys.exit() if masks: INPUT = os.path.join(fileLocationManager.prep, 'rotated_masked') error = test_dir(animal, INPUT, full=False, same_size=True) if len(error) > 0: print(error) sys.exit() OUTPUT = os.path.join(fileLocationManager.prep, 'rotated_aligned_masked') os.makedirs(OUTPUT, exist_ok=True) progress_id = sqlController.get_progress_id(downsample, channel, 'ALIGN') sqlController.set_task(animal, progress_id) warp_transforms = create_warp_transforms(animal, transforms, 'thumbnail', downsample) ordered_transforms = OrderedDict(sorted(warp_transforms.items())) file_keys = [] r90 = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]) for i, (file, T) in enumerate(ordered_transforms.items()): if allen: ROT_DIR = os.path.join(fileLocationManager.root, animal, 'rotations') rotfile = file.replace('tif', 'txt') rotfile = os.path.join(ROT_DIR, rotfile) R_cshl = np.loadtxt(rotfile) R_cshl[0, 2] = R_cshl[0, 2] / 32 R_cshl[1, 2] = R_cshl[1, 2] / 32 R_cshl = R_cshl @ r90 R_cshl = np.linalg.inv(R_cshl) R = T @ R_cshl infile = os.path.join(INPUT, file) outfile = os.path.join(OUTPUT, file) if os.path.exists(outfile) and not create_csv: continue file_keys.append([i, infile, outfile, T]) if create_csv: create_csv_data(animal, file_keys) else: start = timer() workers, _ = get_cpus() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(process_image, sorted(file_keys)) end = timer() print(f'Create cleaned files took {end - start} seconds total', end="\t") print(f' { (end - start)/len(file_keys)} per file') print('Finished')
def make_histogram(animal, channel): """ This method creates an individual histogram for each tif file by channel. Args: animal: the prep id of the animal channel: the channel of the stack to process {1,2,3} Returns: nothing """ logger = get_logger(animal) fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) INPUT = os.path.join(fileLocationManager.prep, f'CH{channel}', 'thumbnail') MASK_INPUT = fileLocationManager.thumbnail_masked tifs = sqlController.get_sections(animal, channel) error = test_dir(animal, INPUT, downsample=True, same_size=False) if len(tifs) == 0: error += " No sections in the database" if len(error) > 0: print(error) sys.exit() ch_dir = f'CH{channel}' OUTPUT = os.path.join(fileLocationManager.histogram, ch_dir) os.makedirs(OUTPUT, exist_ok=True) progress_id = sqlController.get_progress_id(True, channel, 'HISTOGRAM') sqlController.set_task(animal, progress_id) for i, tif in enumerate(tqdm(tifs)): filename = str(i).zfill(3) + '.tif' input_path = os.path.join(INPUT, filename) mask_path = os.path.join(MASK_INPUT, filename) output_path = os.path.join(OUTPUT, os.path.splitext(tif.file_name)[0] + '.png') if not os.path.exists(input_path): print('Input tif does not exist', input_path) continue if os.path.exists(output_path): continue try: img = io.imread(input_path) except: logger.warning(f'Could not open {input_path}') continue try: mask = io.imread(mask_path) except: logger.warning(f'Could not open {mask_path}') continue img = cv2.bitwise_and(img, img, mask=mask) if img.shape[0] * img.shape[1] > 1000000000: scale = 1 / float(2) img = img[::int(1. / scale), ::int(1. / scale)] try: flat = img.flatten() except: logger.warning(f'Could not flat {input_path}') continue fig = plt.figure() plt.rcParams['figure.figsize'] = [10, 6] plt.hist(flat, flat.max(), [0, 10000], color=COLORS[channel]) plt.style.use('ggplot') plt.yscale('log') plt.grid(axis='y', alpha=0.75) plt.xlabel('Value') plt.ylabel('Frequency') plt.title(f'{tif.file_name} @16bit') plt.close() fig.savefig(output_path, bbox_inches='tight')
cmd = [ 'convert', input_path, '-resize', '3.125%', '-compress', 'lzw', output_path ] commands.append(cmd) with Pool(4) as p: p.map(workernoshell, commands) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Work on Animal') parser.add_argument('--animal', help='Enter the animal animal', required=True) parser.add_argument('--channel', help='Enter channel', required=True) args = parser.parse_args() animal = args.animal channel = int(args.channel) make_full_resolution(animal, channel) make_low_resolution(animal, channel) sqlController = SqlController(animal) if channel == 1: sqlController.update_scanrun(sqlController.scan_run.id) progress_id = sqlController.get_progress_id(True, channel, 'TIF') sqlController.set_task(animal, progress_id) progress_id = sqlController.get_progress_id(False, channel, 'TIF') sqlController.set_task(animal, progress_id)