def create_quantized_affinity_tasks(taskqueue, src_layer, dest_layer, shape, fill_missing=False): shape = Vec(*shape) info = create_quantized_affinity_info(src_layer, dest_layer, shape) destvol = CloudVolume(dest_layer, info=info) destvol.commit_info() create_downsample_scales(dest_layer, mip=0, ds_shape=shape) for startpt in tqdm(xyzrange(destvol.bounds.minpt, destvol.bounds.maxpt, shape), desc="Inserting QuantizeAffinities Tasks"): task = QuantizeAffinitiesTask( source_layer_path=src_layer, dest_layer_path=dest_layer, shape=list(shape.clone()), offset=list(startpt.clone()), fill_missing=fill_missing, ) task_queue.insert(task) task_queue.wait('Uploading')
def create_meshing_tasks(task_queue, layer_path, mip, shape=Vec(512, 512, 512)): shape = Vec(*shape) max_simplification_error = 40 vol = CloudVolume(layer_path, mip) if not 'mesh' in vol.info: vol.info['mesh'] = 'mesh_mip_{}_err_{}'.format( mip, max_simplification_error) vol.commit_info() for startpt in tqdm(xyzrange(vol.bounds.minpt, vol.bounds.maxpt, shape), desc="Inserting Mesh Tasks"): task = MeshTask( layer_path=layer_path, mip=vol.mip, shape=shape.clone(), offset=startpt.clone(), max_simplification_error=max_simplification_error, ) task_queue.insert(task) task_queue.wait('Uploading MeshTasks') vol.provenance.processing.append({ 'method': { 'task': 'MeshTask', 'layer_path': layer_path, 'mip': vol.mip, 'shape': shape.tolist(), }, 'by': USER_EMAIL, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance()
def create_watershed_remap_tasks(task_queue, map_path, src_layer_path, dest_layer_path, shape=Vec(2048, 2048, 64)): shape = Vec(*shape) vol = CloudVolume(src_layer_path) create_downsample_scales(dest_layer_path, mip=0, ds_shape=shape) for startpt in tqdm(xyzrange(vol.bounds.minpt, vol.bounds.maxpt, shape), desc="Inserting Remap Tasks"): task = WatershedRemapTask( map_path=map_path, src_path=src_layer_path, dest_path=dest_layer_path, shape=shape.clone(), offset=startpt.clone(), ) task_queue.insert(task) task_queue.wait('Uploading Remap Tasks') dvol = CloudVolume(dest_layer_path) dvol.provenance.processing.append({ 'method': { 'task': 'WatershedRemapTask', 'src': src_layer_path, 'dest': dest_layer_path, 'remap_file': map_path, 'shape': list(shape), }, 'by': '*****@*****.**', 'date': strftime('%Y-%m-%d %H:%M %Z'), }) dvol.commit_provenance()
def create_transfer_tasks(task_queue, src_layer_path, dest_layer_path, chunk_size=None, shape=Vec(2048, 2048, 64), fill_missing=False, translate=(0, 0, 0)): shape = Vec(*shape) translate = Vec(*translate) vol = CloudVolume(src_layer_path) if not chunk_size: chunk_size = vol.info['scales'][0]['chunk_sizes'][0] chunk_size = Vec(*chunk_size) try: dvol = CloudVolume(dest_layer_path) except Exception: # no info file info = copy.deepcopy(vol.info) dvol = CloudVolume(dest_layer_path, info=info) dvol.info['scales'] = dvol.info['scales'][:1] dvol.info['scales'][0]['chunk_sizes'] = [chunk_size.tolist()] dvol.commit_info() create_downsample_scales(dest_layer_path, mip=0, ds_shape=shape, preserve_chunk_size=True) bounds = vol.bounds.clone() for startpt in tqdm(xyzrange(bounds.minpt, bounds.maxpt, shape), desc="Inserting Transfer Tasks"): task = TransferTask( src_path=src_layer_path, dest_path=dest_layer_path, shape=shape.clone(), offset=startpt.clone(), fill_missing=fill_missing, translate=translate, ) task_queue.insert(task) task_queue.wait('Uploading Transfer Tasks') dvol = CloudVolume(dest_layer_path) dvol.provenance.processing.append({ 'method': { 'task': 'TransferTask', 'src': src_layer_path, 'dest': dest_layer_path, 'shape': list(map(int, shape)), 'fill_missing': fill_missing, 'translate': list(map(int, translate)), }, 'by': USER_EMAIL, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) dvol.commit_provenance()
def create_fixup_quantize_tasks(task_queue, src_layer, dest_layer, shape, points): shape = Vec(*shape) vol = CloudVolume(src_layer, 0) offsets = compute_fixup_offsets(vol, points, shape) for offset in tqdm(offsets, desc="Inserting Corrective Quantization Tasks"): task = QuantizeAffinitiesTask( source_layer_path=src_layer, dest_layer_path=dest_layer, shape=list(shape.clone()), offset=list(offset.clone()), ) # task.execute() task_queue.insert(task) task_queue.wait('Uploading')
class FinelyDividedTaskIterator(): """ Parallelizes tasks that do not have overlap. Evenly splits tasks between processes without regards to whether the dividing line lands in the middle of a slice. """ def __init__(self, bounds, shape): self.bounds = bounds self.shape = Vec(*shape) self.start = 0 self.end = num_tasks(bounds, shape) def __len__(self): return self.end - self.start def __getitem__(self, slc): itr = copy.deepcopy(self) itr.start = max(self.start + slc.start, self.start) itr.end = min(self.start + slc.stop, self.end) return itr def __iter__(self): for i in range(self.start, self.end): pt = self.to_coord(i) offset = pt * self.shape + self.bounds.minpt yield self.task(self.shape.clone(), offset.clone()) self.on_finish() def to_coord(self, index): """Convert an index into a grid coordinate defined by the task shape.""" sx, sy, sz = np.ceil(self.bounds.size3() / self.shape).astype(int) sxy = sx * sy z = index // sxy y = (index - (z * sxy)) // sx x = index - sx * (y + z * sy) return Vec(x, y, z) def task(self, shape, offset): raise NotImplementedError() def on_finish(self): pass
def create_transfer_tasks(task_queue, src_layer_path, dest_layer_path, shape=Vec(2048, 2048, 64), fill_missing=False, translate=(0, 0, 0)): shape = Vec(*shape) translate = Vec(*translate) vol = CloudVolume(src_layer_path) create_downsample_scales(dest_layer_path, mip=0, ds_shape=shape, preserve_chunk_size=True) bounds = vol.bounds.clone() for startpt in tqdm(xyzrange(bounds.minpt, bounds.maxpt, shape), desc="Inserting Transfer Tasks"): task = TransferTask( src_path=src_layer_path, dest_path=dest_layer_path, shape=shape.clone(), offset=startpt.clone(), fill_missing=fill_missing, translate=translate, ) task_queue.insert(task) task_queue.wait('Uploading Transfer Tasks') dvol = CloudVolume(dest_layer_path) dvol.provenance.processing.append({ 'method': { 'task': 'TransferTask', 'src': src_layer_path, 'dest': dest_layer_path, 'shape': list(map(int, shape)), }, 'by': '*****@*****.**', 'date': strftime('%Y-%m-%d %H:%M %Z'), }) dvol.commit_provenance()
def create_boss_transfer_tasks(task_queue, src_layer_path, dest_layer_path, shape=Vec(1024, 1024, 64)): # Note: Weird errors with datatype changing to float64 when requesting 2048,2048,64 # 1024,1024,64 worked nicely though. shape = Vec(*shape) vol = CloudVolume(dest_layer_path) create_downsample_scales(dest_layer_path, mip=0, ds_shape=shape) for startpt in tqdm(xyzrange(vol.bounds.minpt, vol.bounds.maxpt, shape), desc="Inserting Boss Transfer Tasks"): task = BossTransferTask( src_path=src_layer_path, dest_path=dest_layer_path, shape=shape.clone(), offset=startpt.clone(), ) task_queue.insert(task) task_queue.wait('Uploading Boss Transfer Tasks')
def create_meshing_tasks(task_queue, layer_path, mip, shape=Vec(512, 512, 512)): shape = Vec(*shape) max_simplification_error = 40 vol = CloudVolume(layer_path, mip) if not 'mesh' in vol.info: vol.info['mesh'] = 'mesh_mip_{}_err_{}'.format( mip, max_simplification_error) vol.commit_info() for startpt in tqdm(xyzrange(vol.bounds.minpt, vol.bounds.maxpt, shape), desc="Inserting Mesh Tasks"): task = MeshTask( layer_path=layer_path, mip=vol.mip, shape=shape.clone(), offset=startpt.clone(), max_simplification_error=max_simplification_error, ) task_queue.insert(task) task_queue.wait('Uploading MeshTasks')
def create_luminance_levels_tasks(layer_path, levels_path=None, coverage_factor=0.01, shape=None, offset=(0, 0, 0), mip=0, bounds=None): """ Compute per slice luminance level histogram and write them as $layer_path/levels/$z. Each z file looks like: { "levels": [ 0, 35122, 12, ... ], # 256 indices, index = luminance i.e. 0 is black, 255 is white "patch_size": [ sx, sy, sz ], # metadata on how large the patches were "num_patches": 20, # metadata on "coverage_ratio": 0.011, # actual sampled area on this slice normalized by ROI size. } layer_path: source image to sample from levels_path: which path to write ./levels/ to (default: $layer_path) coverage_factor: what fraction of the image to sample offset & shape: Allows you to specify an ROI if much of the edges are black. Defaults to entire image. mip: int, which mip to work with, default maximum resolution """ vol = CloudVolume(layer_path, mip=mip) if shape == None: shape = vol.shape.clone() shape.z = 1 offset = Vec(*offset) zoffset = offset.clone() bounds = get_bounds(vol, bounds, shape, mip) protocol = vol.path.protocol class LuminanceLevelsTaskIterator(object): def __len__(self): return bounds.maxpt.z - bounds.minpt.z def __iter__(self): for z in range(bounds.minpt.z, bounds.maxpt.z + 1): zoffset.z = z yield LuminanceLevelsTask( src_path=layer_path, levels_path=levels_path, shape=shape, offset=zoffset, coverage_factor=coverage_factor, mip=mip, ) if protocol == 'boss': raise StopIteration() if levels_path: try: vol = CloudVolume(levels_path) except cloudvolume.exceptions.InfoUnavailableError: vol = CloudVolume(levels_path, info=vol.info) else: vol = CloudVolume(layer_path, mip=mip) vol.provenance.processing.append({ 'method': { 'task': 'LuminanceLevelsTask', 'src': layer_path, 'levels_path': levels_path, 'shape': Vec(*shape).tolist(), 'offset': Vec(*offset).tolist(), 'bounds': [bounds.minpt.tolist(), bounds.maxpt.tolist()], 'coverage_factor': coverage_factor, 'mip': mip, }, 'by': OPERATOR_CONTACT, 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return LuminanceLevelsTaskIterator()