def test_transfer_task_vanilla(tq, src_cv, transfer_data): tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath) tq.insert_all(tasks) dest_cv = CloudVolume(destpath) assert np.all(src_cv[:] == dest_cv[:]) rmsrc() rmdest()
def test_transfer_task_skip_downsample(tq, src_cv, chunk_size): tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath, chunk_size=chunk_size, skip_downsamples=True) tq.insert_all(tasks) dest_cv = CloudVolume(destpath) assert len(dest_cv.scales) == 1 assert np.all(src_cv[:] == dest_cv[:]) rmsrc() rmdest()
def create_downsamples(animal, channel, suffix, downsample): fileLocationManager = FileLocationManager(animal) channel_outdir = f'C{channel}' first_chunk = calculate_chunks(downsample, 0) mips = [0, 1, 2, 3, 4, 5, 6, 7] if downsample: channel_outdir += 'T' mips = [0, 1] outpath = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') outpath = f'file://{outpath}' if suffix is not None: outpath += suffix channel_outdir += "_rechunkme" INPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, f'{channel_outdir}') if not os.path.exists(INPUT_DIR): print(f'DIR {INPUT_DIR} does not exist, exiting.') sys.exit() cloudpath = f"file://{INPUT_DIR}" _, workers = get_cpus() tq = LocalTaskQueue(parallel=workers) tasks = tc.create_transfer_tasks(cloudpath, dest_layer_path=outpath, chunk_size=first_chunk, mip=0, skip_downsamples=True) tq.insert(tasks) tq.execute() #mips = 7 shows good results in neuroglancer for mip in mips: cv = CloudVolume(outpath, mip) chunks = calculate_chunks(downsample, mip) factors = calculate_factors(downsample, mip) tasks = tc.create_downsampling_tasks(cv.layer_cloudpath, mip=mip, num_mips=1, factor=factors, preserve_chunk_size=False, compress=True, chunk_size=chunks) tq.insert(tasks) tq.execute() print("Done!")
def test_transfer_task_rechunk(tq, src_cv, transfer_data): tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath, chunk_size=(50, 50, 50)) tq.insert_all(tasks) dest_cv = CloudVolume(destpath) assert len(dest_cv.scales) == 4 assert np.all(src_cv[:] == dest_cv[:]) for mip in range(1, 4): dest_cv.mip = mip assert np.all(dest_cv[:] == transfer_data[mip]) rmsrc() rmdest()
def add_rechunking(self, outpath, downsample, chunks=None): if self.precomputed_vol is None: raise NotImplementedError( 'You have to call init_precomputed before calling this function.' ) cpus, _ = get_cpus() tq = LocalTaskQueue(parallel=cpus) outpath = f'file://{outpath}' if chunks is None: chunks = calculate_chunks(downsample, 0) tasks = tc.create_transfer_tasks(self.precomputed_vol.layer_cloudpath, dest_layer_path=outpath, chunk_size=chunks, skip_downsamples=True) tq.insert(tasks) tq.execute()
def test_transfer_task_dest_offset(tq, src_cv, transfer_data): tasks = tc.create_transfer_tasks(src_cv.cloudpath, destpath, chunk_size=(50, 50, 50), dest_voxel_offset=(100, 100, 100)) tq.insert_all(tasks) dest_cv = CloudVolume(destpath) assert len(dest_cv.scales) == 4 assert tuple(dest_cv.voxel_offset) == (100, 100, 100) assert tuple(src_cv.voxel_offset) == (0, 0, 0) assert np.all(src_cv[:] == dest_cv[:]) for mip in range(1, 4): dest_cv.mip = mip assert np.all(dest_cv[:] == transfer_data[mip]) rmsrc() rmdest()
def test_transfer_task_subset(tq, src_cv, transfer_data): dest_cv = CloudVolume(destpath, info=copy.deepcopy(src_cv.info)) dest_cv.scale["size"] = [256, 256, 64] dest_cv.commit_info() tasks = tc.create_transfer_tasks( src_cv.cloudpath, destpath, chunk_size=(64, 64, 64), translate=(-128, -128, -64), ) tq.insert_all(tasks) dest_cv.refresh_info() assert len(dest_cv.scales) == 3 assert np.all(src_cv[128:128 + 256, 128:128 + 256, 64:64 + 64] == dest_cv[:]) rmsrc() rmdest()
def xfer( ctx, src, dest, queue, translate, downsample, mip, fill_missing, num_mips, cseg, shape, sparse, chunk_size, compress, volumetric, delete_bg, bg_color ): """ Transfer an image layer to another location. It is crucial to choose a good task shape. The task shape must be a multiple of two of the destination image layer chunk size. Too small, and you'll have an inefficient transfer. Too big, and you'll run out of memory and also have an inefficient transfer. Downsamples will by default be automatically calculated from whatever material is available. For the default 2x2x1 downsampling, larger XY dimension is desirable compared to Z as more downsamples can be computed for each 2x2 increase in the task size. """ encoding = ("compressed_segmentation" if cseg else None) factor = (2,2,1) if volumetric: factor = (2,2,2) shape = [ int(axis) for axis in shape.split(",") ] translate = [ int(amt) for amt in translate.split(",") ] tasks = tc.create_transfer_tasks( src, dest, chunk_size=chunk_size, fill_missing=fill_missing, translate=translate, mip=mip, shape=shape, encoding=encoding, skip_downsamples=(not downsample), delete_black_uploads=delete_bg, background_color=bg_color, compress=compress, factor=factor, sparse=sparse, ) parallel = int(ctx.obj.get("parallel", 1)) tq = TaskQueue(normalize_path(queue)) tq.insert(tasks, parallel=parallel)
to_upload.sort() print(f"Have {len(to_upload)} planes to upload") with ProcessPoolExecutor(max_workers=cpus) as executor: for job in executor.map(process_slice, to_upload): try: print(job) except Exception as exc: print(f'generated an exception: {exc}') elif step == 'step2': # transfer tasks orig_vol = CloudVolume(f'file://{orig_layer_dir}') first_chunk = calculate_chunks(downsample='full', mip=0) tq = LocalTaskQueue(parallel=cpus) tasks = tc.create_transfer_tasks(orig_vol.cloudpath, dest_layer_path=rechunked_cloudpath, chunk_size=first_chunk, mip=0, skip_downsamples=True) print(len(tasks)) tq.insert(tasks) tq.execute() elif step == 'step3': # downsampling print("step 3, downsampling") tq = LocalTaskQueue(parallel=cpus) downsample = "full" mips = [0, 1, 2, 3, 4] for mip in mips: print(f"Mip: {mip}") cv = CloudVolume(rechunked_cloudpath, mip) chunks = calculate_chunks(downsample, mip)
def create_mesh(animal, limit, mse): #chunks = calculate_chunks('full', -1) chunks = [64,64,1] scales = (10400, 10400, 20000) fileLocationManager = FileLocationManager(animal) INPUT = "/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/DK52/preps/CH3/shapes" OUTPUT1_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh_input') OUTPUT2_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'mesh') if 'ultraman' in get_hostname(): if os.path.exists(OUTPUT1_DIR): shutil.rmtree(OUTPUT1_DIR) if os.path.exists(OUTPUT2_DIR): shutil.rmtree(OUTPUT2_DIR) files = sorted(os.listdir(INPUT)) os.makedirs(OUTPUT1_DIR, exist_ok=True) os.makedirs(OUTPUT2_DIR, exist_ok=True) len_files = len(files) midpoint = len_files // 2 midfilepath = os.path.join(INPUT, files[midpoint]) midfile = io.imread(midfilepath) data_type = midfile.dtype #image = np.load('/net/birdstore/Active_Atlas_Data/data_root/pipeline_data/structures/allen/allen.npy') #ids = np.unique(image) ids = {'infrahypoglossal': 200, 'perifacial': 210, 'suprahypoglossal': 220} height, width = midfile.shape volume_size = (width, height, len(files)) # neuroglancer is width, height print('volume size', volume_size) ng = NumpyToNeuroglancer(animal, None, scales, layer_type='segmentation', data_type=data_type, chunk_size=chunks) ng.init_precomputed(OUTPUT1_DIR, volume_size, progress_id=1) file_keys = [] for i,f in enumerate(tqdm(files)): infile = os.path.join(INPUT, f) file_keys.append([i, infile]) #ng.process_image([i, infile]) #sys.exit() start = timer() workers, cpus = get_cpus() print(f'Working on {len(file_keys)} files with {workers} cpus') with ProcessPoolExecutor(max_workers=workers) as executor: executor.map(ng.process_image, sorted(file_keys), chunksize=1) executor.shutdown(wait=True) ng.precomputed_vol.cache.flush() end = timer() print(f'Create volume method took {end - start} seconds') ##### rechunk cloudpath1 = f"file://{OUTPUT1_DIR}" cv1 = CloudVolume(cloudpath1, 0) _, workers = get_cpus() tq = LocalTaskQueue(parallel=workers) cloudpath2 = f'file://{OUTPUT2_DIR}' tasks = tc.create_transfer_tasks(cloudpath1, dest_layer_path=cloudpath2, chunk_size=[64,64,64], mip=0, skip_downsamples=True) tq.insert(tasks) tq.execute() ##### add segment properties cv2 = CloudVolume(cloudpath2, 0) cv2.info['segment_properties'] = 'names' cv2.commit_info() segment_properties_path = os.path.join(cloudpath2.replace('file://', ''), 'names') os.makedirs(segment_properties_path, exist_ok=True) info = { "@type": "neuroglancer_segment_properties", "inline": { "ids": [str(value) for key, value in ids.items()], "properties": [{ "id": "label", "type": "label", "values": [str(key) for key, value in ids.items()] }] } } with open(os.path.join(segment_properties_path, 'info'), 'w') as file: json.dump(info, file, indent=2) ##### first mesh task, create meshing tasks workers, _ = get_cpus() tq = LocalTaskQueue(parallel=workers) mesh_dir = f'mesh_mip_0_err_{mse}' cv2.info['mesh'] = mesh_dir cv2.commit_info() tasks = tc.create_meshing_tasks(cv2.layer_cloudpath, mip=0, mesh_dir=mesh_dir, max_simplification_error=mse) tq.insert(tasks) tq.execute() ##### 2nd mesh task, create manifest tasks = tc.create_mesh_manifest_tasks(cv2.layer_cloudpath, mesh_dir=mesh_dir) tq.insert(tasks) tq.execute() print("Done!")
def xfer(ctx, src, dest, queue, translate, downsample, mip, fill_missing, memory, max_mips, shape, sparse, cseg, compresso, chunk_size, compress, volumetric, delete_bg, bg_color, sharded, dest_voxel_offset, clean_info, no_src_update): """ Transfer an image layer to another location. It is crucial to choose a good task shape. The task shape must be a multiple of two of the destination image layer chunk size. Too small, and you'll have an inefficient transfer. Too big, and you'll run out of memory and also have an inefficient transfer. Downsamples will by default be automatically calculated from whatever material is available. For the default 2x2x1 downsampling, larger XY dimension is desirable compared to Z as more downsamples can be computed for each 2x2 increase in the task size. Use the --memory flag to automatically compute the a reasonable task shape based on your memory limits. """ src = cloudfiles.paths.normalize(src) dest = cloudfiles.paths.normalize(dest) if cseg and compresso: print("igneous: must choose one of --cseg or --compresso") sys.exit() encoding = None if cseg: encoding = "compressed_segmentation" elif compresso: encoding = "compresso" factor = (2, 2, 1) if volumetric: factor = (2, 2, 2) if compress and compress.lower() == "false": compress = False if sharded: tasks = tc.create_image_shard_transfer_tasks( src, dest, chunk_size=chunk_size, fill_missing=fill_missing, mip=mip, dest_voxel_offset=dest_voxel_offset, translate=translate, encoding=encoding, memory_target=memory, clean_info=clean_info) else: tasks = tc.create_transfer_tasks(src, dest, chunk_size=chunk_size, fill_missing=fill_missing, dest_voxel_offset=dest_voxel_offset, translate=translate, mip=mip, shape=shape, encoding=encoding, skip_downsamples=(not downsample), delete_black_uploads=delete_bg, background_color=bg_color, compress=compress, factor=factor, sparse=sparse, memory_target=memory, max_mips=max_mips, clean_info=clean_info, no_src_update=no_src_update) parallel = int(ctx.obj.get("parallel", 1)) tq = TaskQueue(normalize_path(queue)) tq.insert(tasks, parallel=parallel)