def create_downsamples(animal, channel, suffix, downsample): fileLocationManager = FileLocationManager(animal) channel_outdir = f'C{channel}' first_chunk = calculate_chunks(downsample, 0) mips = [0, 1, 2, 3, 4, 5, 6, 7] if downsample: channel_outdir += 'T' mips = [0, 1] outpath = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') outpath = f'file://{outpath}' if suffix is not None: outpath += suffix channel_outdir += "_rechunkme" INPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') if not os.path.exists(INPUT_DIR): print(f'DIR {INPUT_DIR} does not exist, exiting.') sys.exit() cloudpath = f"file://{INPUT_DIR}" _, workers = get_cpus() tq = LocalTaskQueue(parallel=workers) tasks = tc.create_transfer_tasks(cloudpath, dest_layer_path=outpath, chunk_size=first_chunk, mip=0, skip_downsamples=True) tq.insert(tasks) tq.execute() #mips = 7 shows good results in neuroglancer for mip in mips: cv = CloudVolume(outpath, mip) chunks = calculate_chunks(downsample, mip) factors = calculate_factors(downsample, mip) tasks = tc.create_downsampling_tasks(cv.layer_cloudpath, mip=mip, num_mips=1, factor=factors, preserve_chunk_size=False, compress=True, chunk_size=chunks) tq.insert(tasks) tq.execute() print("Done!")
def add_downsampled_volumes(self, chunk_size=[128, 128, 64], num_mips=4): if self.precomputed_vol is None: raise NotImplementedError( 'You have to call init_precomputed before calling this function.' ) _, cpus = get_cpus() tq = LocalTaskQueue(parallel=cpus) tasks = tc.create_downsampling_tasks( self.precomputed_vol.layer_cloudpath, preserve_chunk_size=False, num_mips=num_mips, chunk_size=chunk_size, compress=True) tq.insert(tasks) tq.execute()
def add_rechunking(self, outpath, downsample, chunks=None): if self.precomputed_vol is None: raise NotImplementedError( 'You have to call init_precomputed before calling this function.' ) cpus, _ = get_cpus() tq = LocalTaskQueue(parallel=cpus) outpath = f'file://{outpath}' if chunks is None: chunks = calculate_chunks(downsample, 0) tasks = tc.create_transfer_tasks(self.precomputed_vol.layer_cloudpath, dest_layer_path=outpath, chunk_size=chunks, skip_downsamples=True) tq.insert(tasks) tq.execute()
def add_segmentation_mesh(self, shape=[448, 448, 448], mip=0): if self.precomputed_vol is None: raise NotImplementedError( 'You have to call init_precomputed before calling this function.' ) _, cpus = get_cpus() tq = LocalTaskQueue(parallel=cpus) tasks = tc.create_meshing_tasks( self.precomputed_vol.layer_cloudpath, mip=mip, max_simplification_error=40, shape=shape, compress=True) # The first phase of creating mesh tq.insert(tasks) tq.execute() # It should be able to incoporated to above tasks, but it will give a weird bug. Don't know the reason tasks = tc.create_mesh_manifest_tasks( self.precomputed_vol.layer_cloudpath ) # The second phase of creating mesh tq.insert(tasks) tq.execute()
def create_neuroglancer(animal, channel, downsample, debug=False): fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) channel_dir = f'CH{channel}' channel_outdir = f'C{channel}T_rechunkme' INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned') db_resolution = sqlController.scan_run.resolution resolution = int(db_resolution * 1000 / SCALING_FACTOR) workers, _ = get_cpus() chunks = calculate_chunks(downsample, -1) progress_id = sqlController.get_progress_id(downsample, channel, 'NEUROGLANCER') sqlController.session.close() if not downsample: INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned') channel_outdir = f'C{channel}_rechunkme' sqlController.set_task(animal, progress_id) if 'thion' in sqlController.histology.counterstain: sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_2_FULL_RES) sqlController.set_task(animal, RUN_PRECOMPUTE_NEUROGLANCER_CHANNEL_3_FULL_RES) resolution = int(db_resolution * 1000) OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') error = test_dir(animal, INPUT, downsample, same_size=True) if len(error) > 0 and not debug: print(error) sys.exit() os.makedirs(OUTPUT_DIR, exist_ok=True) files = sorted(os.listdir(INPUT)) midpoint = len(files) // 2 midfilepath = os.path.join(INPUT, files[midpoint]) midfile = io.imread(midfilepath, img_num=0) height = midfile.shape[0] width = midfile.shape[1] num_channels = midfile.shape[2] if len(midfile.shape) > 2 else 1 file_keys = [] scales = (resolution, resolution, 20000) volume_size = (width, height, len(files)) print('Volume shape:', volume_size) ng = NumpyToNeuroglancer(animal, None, scales, 'image', midfile.dtype, num_channels=num_channels, chunk_size=chunks) ng.init_precomputed(OUTPUT_DIR, volume_size, progress_id=progress_id) for i, f in enumerate(files): filepath = os.path.join(INPUT, f) file_keys.append([i,filepath]) #ng.process_3channel([i, filepath]) #sys.exit() start = timer() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: if num_channels == 1: executor.map(ng.process_image, sorted(file_keys)) else: executor.map(ng.process_3channel, sorted(file_keys)) end = timer() print(f'Create volume method took {end - start} seconds') ng.precomputed_vol.cache.flush()
def create_mesh(animal, limit, mse): #chunks = calculate_chunks('full', -1) chunks = [64,64,1] scales = (10400, 10400, 20000) fileLocationManager = FileLocationManager(animal) INPUT = "/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/shapes" OUTPUT1_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh_input') OUTPUT2_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh') if 'ultraman' in get_hostname(): if os.path.exists(OUTPUT1_DIR): shutil.rmtree(OUTPUT1_DIR) if os.path.exists(OUTPUT2_DIR): shutil.rmtree(OUTPUT2_DIR) files = sorted(os.listdir(INPUT)) os.makedirs(OUTPUT1_DIR, exist_ok=True) os.makedirs(OUTPUT2_DIR, exist_ok=True) len_files = len(files) midpoint = len_files // 2 midfilepath = os.path.join(INPUT, files[midpoint]) midfile = io.imread(midfilepath) data_type = midfile.dtype #image = np.load('/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/structures/allen/allen.npy') #ids = np.unique(image) ids = {'infrahypoglossal': 200, 'perifacial': 210, 'suprahypoglossal': 220} height, width = midfile.shape volume_size = (width, height, len(files)) # neuroglancer is width, height print('volume size', volume_size) ng = NumpyToNeuroglancer(animal, None, scales, layer_type='segmentation', data_type=data_type, chunk_size=chunks) ng.init_precomputed(OUTPUT1_DIR, volume_size, progress_id=1) file_keys = [] for i,f in enumerate(tqdm(files)): infile = os.path.join(INPUT, f) file_keys.append([i, infile]) #ng.process_image([i, infile]) #sys.exit() start = timer() workers, cpus = get_cpus() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(ng.process_image, sorted(file_keys), chunksize=1) executor.shutdown(wait=True) ng.precomputed_vol.cache.flush() end = timer() print(f'Create volume method took {end - start} seconds') ##### rechunk cloudpath1 = f"file://{OUTPUT1_DIR}" cv1 = CloudVolume(cloudpath1, 0) _, workers = get_cpus() tq = LocalTaskQueue(parallel=workers) cloudpath2 = f'file://{OUTPUT2_DIR}' tasks = tc.create_transfer_tasks(cloudpath1, dest_layer_path=cloudpath2, chunk_size=[64,64,64], mip=0, skip_downsamples=True) tq.insert(tasks) tq.execute() ##### add segment properties cv2 = CloudVolume(cloudpath2, 0) cv2.info['segment_properties'] = 'names' cv2.commit_info() segment_properties_path = os.path.join(cloudpath2.replace('file://', ''), 'names') os.makedirs(segment_properties_path, exist_ok=True) info = { "@type": "neuroglancer_segment_properties", "inline": { "ids": [str(value) for key, value in ids.items()], "properties": [{ "id": "label", "type": "label", "values": [str(key) for key, value in ids.items()] }] } } with open(os.path.join(segment_properties_path, 'info'), 'w') as file: json.dump(info, file, indent=2) ##### first mesh task, create meshing tasks workers, _ = get_cpus() tq = LocalTaskQueue(parallel=workers) mesh_dir = f'mesh_mip_0_err_{mse}' cv2.info['mesh'] = mesh_dir cv2.commit_info() tasks = tc.create_meshing_tasks(cv2.layer_cloudpath, mip=0, mesh_dir=mesh_dir, max_simplification_error=mse) tq.insert(tasks) tq.execute() ##### 2nd mesh task, create manifest tasks = tc.create_mesh_manifest_tasks(cv2.layer_cloudpath, mesh_dir=mesh_dir) tq.insert(tasks) tq.execute() print("Done!")
def run_offsets(animal, transforms, channel, downsample, masks, create_csv, allen): """ This gets the dictionary from the above method, and uses the coordinates to feed into the Imagemagick convert program. This method also uses a Pool to spawn multiple processes. Args: animal: the animal transforms: the dictionary of file, coordinates limit: number of jobs Returns: nothing """ fileLocationManager = FileLocationManager(animal) sqlController = SqlController(animal) channel_dir = 'CH{}'.format(channel) INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_cleaned') OUTPUT = os.path.join(fileLocationManager.prep, channel_dir, 'thumbnail_aligned') if not downsample: INPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_cleaned') OUTPUT = os.path.join(fileLocationManager.prep, channel_dir, 'full_aligned') error = test_dir(animal, INPUT, downsample=downsample, same_size=True) if len(error) > 0 and not create_csv: print(error) sys.exit() if masks: INPUT = os.path.join(fileLocationManager.prep, 'rotated_masked') error = test_dir(animal, INPUT, full=False, same_size=True) if len(error) > 0: print(error) sys.exit() OUTPUT = os.path.join(fileLocationManager.prep, 'rotated_aligned_masked') os.makedirs(OUTPUT, exist_ok=True) progress_id = sqlController.get_progress_id(downsample, channel, 'ALIGN') sqlController.set_task(animal, progress_id) warp_transforms = create_warp_transforms(animal, transforms, 'thumbnail', downsample) ordered_transforms = OrderedDict(sorted(warp_transforms.items())) file_keys = [] r90 = np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]) for i, (file, T) in enumerate(ordered_transforms.items()): if allen: ROT_DIR = os.path.join(fileLocationManager.root, animal, 'rotations') rotfile = file.replace('tif', 'txt') rotfile = os.path.join(ROT_DIR, rotfile) R_cshl = np.loadtxt(rotfile) R_cshl[0, 2] = R_cshl[0, 2] / 32 R_cshl[1, 2] = R_cshl[1, 2] / 32 R_cshl = R_cshl @ r90 R_cshl = np.linalg.inv(R_cshl) R = T @ R_cshl infile = os.path.join(INPUT, file) outfile = os.path.join(OUTPUT, file) if os.path.exists(outfile) and not create_csv: continue file_keys.append([i, infile, outfile, T]) if create_csv: create_csv_data(animal, file_keys) else: start = timer() workers, _ = get_cpus() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(process_image, sorted(file_keys)) end = timer() print(f'Create cleaned files took {end - start} seconds total', end="\t") print(f' { (end - start)/len(file_keys)} per file') print('Finished')