def _upload_spatial_index(self, bbox, mesh_bboxes): cf = CloudFiles(self.layer_path, progress=self.options['progress']) cf.put_json( f"{self._mesh_dir}/{bbox.to_filename()}.spatial", mesh_bboxes, compress=self.options['compress'], cache_control=False, )
def MeshSpatialIndex( cloudpath:str, shape:Tuple[int,int,int], offset:Tuple[int,int,int], mip:int = 0, fill_missing:bool=False, compress:Optional[Union[str,bool]] = 'gzip', mesh_dir:Optional[str] = None ) -> None: """ The main way to add a spatial index is to use the MeshTask, but old datasets or broken datasets may need it to be reconstituted. An alternative use is create the spatial index over a different area size than the mesh task. """ cv = CloudVolume( cloudpath, mip=mip, bounded=False, fill_missing=fill_missing ) cf = CloudFiles(cloudpath) bounds = Bbox(Vec(*offset), Vec(*shape) + Vec(*offset)) bounds = Bbox.clamp(bounds, cv.bounds) data_bounds = bounds.clone() data_bounds.maxpt += 1 # match typical Marching Cubes overlap precision = cv.mesh.spatial_index.precision resolution = cv.resolution if not mesh_dir: mesh_dir = cv.info["mesh"] # remap: old img -> img img, remap = cv.download(data_bounds, renumber=True) img = img[...,0] slcs = find_objects(img) del img reverse_map = { v:k for k,v in remap.items() } # img -> old img bboxes = {} for label, slc in enumerate(slcs): if slc is None: continue mesh_bounds = Bbox.from_slices(slc) mesh_bounds += Vec(*offset) mesh_bounds *= Vec(*resolution, dtype=np.float32) bboxes[str(reverse_map[label+1])] = \ mesh_bounds.astype(resolution.dtype).to_list() bounds = bounds.astype(resolution.dtype) * resolution cf.put_json( f"{mesh_dir}/{bounds.to_filename(precision)}.spatial", bboxes, compress=compress, cache_control=False, )
class MeshManifestOperator(OperatorBase): """Create mesh manifest files for Neuroglancer visualization.""" def __init__(self, volume_path: str, lod: int = 0, name: str = 'mesh-manifest'): """ Parameters ------------ volume_path: path to store mesh manifest files lod: level of detail. we always use 0! """ super().__init__(name=name) self.lod = lod vol = CloudVolume(volume_path) info = vol.info assert 'mesh' in info self.mesh_path = os.path.join(volume_path, info['mesh']) self.storage = CloudFiles(self.mesh_path) def __call__(self, prefix: Union[int, str], digits: int) -> None: assert int(prefix) < 10**digits prefix = str(prefix).zfill(digits) id2filenames = defaultdict(list) for filename in tqdm(self.storage.list(prefix=prefix), desc='list mesh files'): filename = os.path.basename(filename) # `match` implies the beginning (^). `search` matches whole string matches = re.search(r'(\d+):(\d+):', filename) if not matches: continue seg_id, lod = matches.groups() seg_id, lod = int(seg_id), int(lod) # currently we are not using `level of detail`, it is always 0 # will need to adjust code if we start using variants assert lod == self.lod id2filenames[seg_id].append(filename) for seg_id, frags in tqdm(id2filenames.items(), desc='upload aggregated manifest file'): logging.info(f'segment id: {seg_id}') logging.info(f'fragments: {frags}') self.storage.put_json( path=f'{seg_id}:{self.lod}', content={"fragments": frags}, ) # the last few hundred files will not be uploaded without sleeping! sleep(0.01)
def _upload_spatial_index(self, bbox, mesh_bboxes): cf = CloudFiles(self.layer_path, progress=self.options['progress']) precision = self._volume.mesh.spatial_index.precision resolution = self._volume.resolution bbox = bbox.astype(resolution.dtype) * resolution cf.put_json( f"{self._mesh_dir}/{bbox.to_filename(precision)}.spatial", mesh_bboxes, compress=self.options['compress'], cache_control=False, )
def upload_spatial_index(self, vol, path, bbox, skeletons): spatial_index = {} for segid, skel in tqdm(skeletons.items(), disable=(not vol.progress), desc="Extracting Bounding Boxes"): segid_bbx = Bbox.from_points(skel.vertices) spatial_index[segid] = segid_bbx.to_list() bbox = bbox * vol.resolution cf = CloudFiles(path, progress=vol.progress) cf.put_json( path=f"{bbox.to_filename()}.spatial", content=spatial_index, compress='gzip', cache_control=False, )
def test_read_write(s3, protocol, num_threads, green): from cloudfiles import CloudFiles, exceptions url = compute_url(protocol, "rw") cf = CloudFiles(url, num_threads=num_threads, green=green) content = b'some_string' cf.put('info', content, compress=None, cache_control='no-cache') cf['info2'] = content assert cf.get('info') == content assert cf['info2'] == content assert cf['info2', 0:3] == content[0:3] assert cf['info2', :] == content[:] assert cf.get('nonexistentfile') is None assert cf.get('info', return_dict=True) == {"info": content} assert cf.get(['info', 'info2'], return_dict=True) == { "info": content, "info2": content } del cf['info2'] assert cf.exists('info2') == False num_infos = max(num_threads, 1) results = cf.get(['info' for i in range(num_infos)]) assert len(results) == num_infos assert results[0]['path'] == 'info' assert results[0]['content'] == content assert all(map(lambda x: x['error'] is None, results)) assert cf.get(['nonexistentfile'])[0]['content'] is None cf.delete('info') cf.put_json('info', {'omg': 'wow'}, cache_control='no-cache') results = cf.get_json('info') assert results == {'omg': 'wow'} cf.delete('info') if protocol == 'file': rmtree(url)
def setUp(self): print('test volume cutout...') # compute parameters self.mip = 0 self.size = (36, 448, 448) # create image dataset using cloud-volume img = np.random.randint(0, 256, size=self.size) self.img = img.astype(np.uint8) # save the input to disk self.volume_path = 'file:///tmp/test/cutout/' + generate_random_string( ) CloudVolume.from_numpy(np.transpose(self.img), vol_path=self.volume_path) # prepare blackout section ids self.blackout_section_ids = [17, 20] ids = {'section_ids': self.blackout_section_ids} stor = CloudFiles(self.volume_path) stor.put_json('blackout_section_ids.json', ids)
def create_xfer_meshes_tasks( src:str, dest:str, mesh_dir:Optional[str] = None, magnitude=2, ): cv_src = CloudVolume(src) cf_dest = CloudFiles(dest) if not mesh_dir: info = cf_dest.get_json("info") if info.get("mesh", None): mesh_dir = info.get("mesh") cf_dest.put_json(f"{mesh_dir}/info", cv_src.mesh.meta.info) alphabet = [ str(i) for i in range(10) ] if cv_src.mesh.meta.is_sharded(): alphabet += [ 'a', 'b', 'c', 'd', 'e', 'f' ] prefixes = itertools.product(*([ alphabet ] * magnitude)) prefixes = [ "".join(x) for x in prefixes ] # explicitly enumerate all prefixes smaller than the magnitude. for i in range(1, magnitude): explicit_prefix = itertools.product(*([ alphabet ] * i)) explicit_prefix = [ "".join(x) for x in explicit_prefix ] if cv_src.mesh.meta.is_sharded(): prefixes += [ f"{x}." for x in explicit_prefix ] else: prefixes += [ f"{x}:0" for x in explicit_prefix ] return [ partial(TransferMeshFilesTask, src=src, dest=dest, prefix=prefix, mesh_dir=mesh_dir, ) for prefix in prefixes ]
def configure_multires_info( cloudpath:str, vertex_quantization_bits:int, mesh_dir:str ): """ Computes properties and uploads a multires mesh info file """ assert vertex_quantization_bits in (10, 16) vol = CloudVolume(cloudpath) mesh_dir = mesh_dir or vol.info.get("mesh", None) if not "mesh" in vol.info: vol.info['mesh'] = mesh_dir vol.commit_info() res = vol.meta.resolution(vol.mesh.meta.mip) cf = CloudFiles(cloudpath) info_filename = f'{mesh_dir}/info' mesh_info = cf.get_json(info_filename) or {} new_mesh_info = copy.deepcopy(mesh_info) new_mesh_info['@type'] = "neuroglancer_multilod_draco" new_mesh_info['vertex_quantization_bits'] = vertex_quantization_bits new_mesh_info['transform'] = [ res[0], 0, 0, 0, 0, res[1], 0, 0, 0, 0, res[2], 0, ] new_mesh_info['lod_scale_multiplier'] = 1.0 if new_mesh_info != mesh_info: cf.put_json( info_filename, new_mesh_info, cache_control="no-cache" )
def execute(self): srccv = CloudVolume(self.src_path, mip=self.mip, fill_missing=True) # Accumulate a histogram of the luminance levels nbits = np.dtype(srccv.dtype).itemsize * 8 levels = np.zeros(shape=(2 ** nbits,), dtype=np.uint64) bounds = Bbox(self.offset, self.shape[:3] + self.offset) bounds = Bbox.clamp(bounds, srccv.bounds) bboxes = self.select_bounding_boxes(bounds) for bbox in bboxes: img2d = srccv[bbox.to_slices()].reshape((bbox.volume())) cts = np.bincount(img2d) levels[0:len(cts)] += cts.astype(np.uint64) covered_area = sum([bbx.volume() for bbx in bboxes]) bboxes = [(bbox.volume(), bbox.size3()) for bbox in bboxes] bboxes.sort(key=lambda x: x[0]) biggest = bboxes[-1][1] output = { "levels": levels.tolist(), "patch_size": biggest.tolist(), "num_patches": len(bboxes), "coverage_ratio": covered_area / self.shape.rectVolume(), } path = self.levels_path if self.levels_path else self.src_path path = os.path.join(path, 'levels') cf = CloudFiles(path) cf.put_json( path="{}/{}".format(self.mip, self.offset.z), content=output, cache_control='no-cache' )
class MeshOperator(OperatorBase): """Create mesh files from segmentation.""" def __init__(self, output_path: str, output_format: str, mip: int = None, voxel_size: tuple = (1, 1, 1), simplification_factor: int = 100, max_simplification_error: int = 8, manifest: bool = False, shard: bool = False, name: str = 'mesh'): """ Parameters ------------ output_path: path to store mesh files output_format: format of output {'ply', 'obj', 'precomputed'} voxel_size: size of voxels simplification_factor: mesh simplification factor. max_simplification_error: maximum tolerance error of meshing. manifest: create manifest files or not. This should not be True if you are only doing meshing for a segmentation chunk. name: operator name. Note that some functions are adopted from igneous. """ super().__init__(name=name) self.simplification_factor = simplification_factor self.max_simplification_error = max_simplification_error # zmesh use fortran order, translate zyx to xyz self.output_path = output_path self.output_format = output_format self.manifest = manifest self.shard = shard if manifest: assert output_format == 'precomputed' if output_format == 'precomputed': # adjust the mesh path according to info vol = CloudVolume(self.output_path, mip) info = vol.info if 'mesh' not in info: # add mesh to info and update it info['mesh'] = 'mesh_err_{}'.format(max_simplification_error) vol.info = info vol.commit_info() self.mesh_path = os.path.join(output_path, info['mesh']) self.voxel_size = vol.resolution[::-1] self.mesher = Mesher( vol.resolution ) else: self.mesh_path = output_path self.mesher = Mesher(voxel_size[::-1]) self.storage = CloudFiles(self.mesh_path) def _get_mesh_data(self, obj_id, offset): mesh = self.mesher.get_mesh( obj_id, normals=False, simplification_factor=self.simplification_factor, max_simplification_error=self.max_simplification_error) # delete high resolution mesh self.mesher.erase(obj_id) if self.output_format == 'precomputed': mesh.vertices[:] += offset[::-1] * self.voxel_size[::-1] data = mesh.to_precomputed() elif self.output_format == 'ply': data = mesh.to_ply() elif self.output_format == 'obj': data = mesh.to_obj() else: raise NotImplementedError mesh_bounds = Bbox( np.amin(mesh.vertices, axis=0), np.amax(mesh.vertices, axis=0) ) return data, mesh_bounds def _get_file_name(self, bbox, obj_id): if self.output_format == 'precomputed': # bbox is in z,y,x order, should transform to x,y,z order bbox2 = Bbox.from_slices(bbox.to_slices()[::-1]) return '{}:0:{}'.format(obj_id, bbox2.to_filename()) elif self.output_format == 'ply': return '{}.ply'.format(obj_id) elif self.output_format == 'obj': return '{}.obj'.format(obj_id) else: raise ValueError('unsupported format!') def __call__(self, seg: Chunk): """Meshing the segmentation. Parameters ------------ seg: 3D segmentation chunk. """ if seg is None: return assert isinstance(seg, Chunk) assert seg.ndim == 3 assert np.issubdtype(seg.dtype, np.integer) bbox = seg.bbox # use ndarray after getting the bounding box seg = seg.array logging.info('computing meshes from segmentation...') self.mesher.mesh(seg) logging.info('write mesh to storage...') if self.shard: assert 'precomputed' in self.output_format meshes = [] mesh_bboxes = {} for obj_id in self.mesher.ids(): data, mesh_bbox = self._get_mesh_data(obj_id, bbox.minpt) meshes.append(data) mesh_bboxes[obj_id] = mesh_bbox.to_list() # use shared format in default! self.storage.put( f"{self.mesh_path}/{bbox.to_filename()}.frags", content=pickle.dumps(meshes), compress='gzip', content_type="application/python-pickle", cache_control=False, ) self.storage.put_json( f"{self.mesh_path}/{bbox.to_filename()}.spatial", mesh_bboxes, compress='gzip', cache_control=False, ) else: if 'precomputed' in self.output_format: compress = 'gzip' else: compress = None for obj_id in tqdm(self.mesher.ids(), desc='writing out meshes'): # print('object id: ', obj_id) data, _ = self._get_mesh_data(obj_id, bbox.minpt) file_name = self._get_file_name(bbox, obj_id) self.storage.put( file_name, data, cache_control=None, compress=compress ) # create manifest file if self.manifest: self.storage.put_json( f'{obj_id}:0', {'fragments': [file_name]} ) self.storage.put_json( 'info', {"@type": "neuroglancer_legacy_mesh"} ) # release memory self.mesher.clear()
class WritePrecomputedOperator(OperatorBase): def __init__(self, volume_path: str, mip: int, upload_log: bool = True, create_thumbnail: bool = False, intensity_threshold: int = None, name: str = 'write-precomputed'): super().__init__(name=name) self.upload_log = upload_log self.create_thumbnail = create_thumbnail self.mip = mip self.intensity_threshold = intensity_threshold # if not volume_path.startswith('precomputed://'): # volume_path += 'precomputed://' self.volume_path = volume_path # gevent.monkey.patch_all(thread=False) self.volume = CloudVolume(self.volume_path, fill_missing=True, bounded=False, autocrop=True, mip=self.mip, cache=False, green_threads=True, delete_black_uploads=True, progress=True) #parallel=True, if upload_log: log_path = os.path.join(volume_path, 'log') self.log_storage = CloudFiles(log_path) def create_chunk_with_zeros(self, bbox, num_channels, dtype): """Create a fake all zero chunk. this is used in skip some operation based on mask.""" shape = (num_channels, *bbox.size3()) arr = np.zeros(shape, dtype=dtype) chunk = Chunk(arr, voxel_offset=(0, *bbox.minpt)) return chunk def __call__(self, chunk, log=None): assert isinstance(chunk, Chunk) logging.info('save chunk.') start = time.time() if self.intensity_threshold is not None and np.all( chunk.array < self.intensity_threshold): print( 'the voxel intensity in this chunk are all below intensity threshold, return directly without saving anything.' ) return chunk = self._auto_convert_dtype(chunk, self.volume) # transpose czyx to xyzc order arr = np.transpose(chunk.array) self.volume[chunk.slices[::-1]] = arr if self.create_thumbnail: self._create_thumbnail(chunk) # add timer for save operation itself if log: log['timer'][self.name] = time.time() - start if self.upload_log: self._upload_log(log, chunk.bbox) def _auto_convert_dtype(self, chunk, volume): """convert the data type to fit volume datatype""" if np.issubdtype(volume.dtype, np.floating) and np.issubdtype( chunk.dtype, np.uint8): chunk = chunk.astype(volume.dtype) chunk /= 255. # chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max elif np.issubdtype(volume.dtype, np.uint8) and np.issubdtype( chunk.dtype, np.floating): chunk.max() <= 1. chunk *= 255 if volume.dtype != chunk.dtype: print( yellow(f'converting chunk data type {chunk.dtype} ' + f'to volume data type: {volume.dtype}')) # float_chunk = chunk.astype(np.float64) # chunk = float_chunk / np.iinfo(chunk.dtype).max * np.iinfo(self.volume.dtype).max # chunk = chunk / chunk.array.max() * np.iinfo(volume.dtype).max return chunk.astype(volume.dtype) else: return chunk def _create_thumbnail(self, chunk): logging.info('creating thumbnail...') thumbnail_layer_path = os.path.join(self.volume_path, 'thumbnail') thumbnail_volume = CloudVolume(thumbnail_layer_path, compress='gzip', fill_missing=True, bounded=False, autocrop=True, mip=self.mip, cache=False, green_threads=True, delete_black_uploads=True, progress=False) # only use the last channel, it is the Z affinity # if this is affinitymap image = chunk[-1, :, :, :] if np.issubdtype(image.dtype, np.floating): image = (image * 255).astype(np.uint8) #self.thumbnail_operator(image) # transpose to xyzc image = np.transpose(image) image_bbox = BoundingBox.from_slices(chunk.slices[::-1][:3]) downsample_and_upload(image, image_bbox, thumbnail_volume, Vec(*(image.shape)), mip=self.mip, max_mip=6, axis='z', skip_first=True, only_last_mip=True) def _upload_log(self, log, output_bbox): assert log assert isinstance(output_bbox, BoundingBox) logging.info(f'uploaded log: {log}') # write to google cloud storage self.log_storage.put_json(output_bbox.to_filename() + '.json', content=json.dumps(log))
def create_meshing_tasks( layer_path, mip, shape=(448, 448, 448), simplification=True, max_simplification_error=40, mesh_dir=None, cdn_cache=False, dust_threshold=None, object_ids=None, progress=False, fill_missing=False, encoding='precomputed', spatial_index=True, sharded=False, compress='gzip' ): shape = Vec(*shape) vol = CloudVolume(layer_path, mip) if mesh_dir is None: mesh_dir = 'mesh_mip_{}_err_{}'.format(mip, max_simplification_error) if not 'mesh' in vol.info: vol.info['mesh'] = mesh_dir vol.commit_info() cf = CloudFiles(layer_path) info_filename = '{}/info'.format(mesh_dir) mesh_info = cf.get_json(info_filename) or {} mesh_info['@type'] = 'neuroglancer_legacy_mesh' mesh_info['mip'] = int(vol.mip) mesh_info['chunk_size'] = shape.tolist() if spatial_index: mesh_info['spatial_index'] = { 'resolution': vol.resolution.tolist(), 'chunk_size': (shape*vol.resolution).tolist(), } cf.put_json(info_filename, mesh_info) class MeshTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): return MeshTask( shape=shape.clone(), offset=offset.clone(), layer_path=layer_path, mip=vol.mip, simplification_factor=(0 if not simplification else 100), max_simplification_error=max_simplification_error, mesh_dir=mesh_dir, cache_control=('' if cdn_cache else 'no-cache'), dust_threshold=dust_threshold, progress=progress, object_ids=object_ids, fill_missing=fill_missing, encoding=encoding, spatial_index=spatial_index, sharded=sharded, compress=compress, ) def on_finish(self): vol.provenance.processing.append({ 'method': { 'task': 'MeshTask', 'layer_path': layer_path, 'mip': vol.mip, 'shape': shape.tolist(), 'simplification': simplification, 'max_simplification_error': max_simplification_error, 'mesh_dir': mesh_dir, 'fill_missing': fill_missing, 'cdn_cache': cdn_cache, 'dust_threshold': dust_threshold, 'encoding': encoding, 'object_ids': object_ids, 'spatial_index': spatial_index, 'sharded': sharded, 'compress': compress, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return MeshTaskIterator(vol.mip_bounds(mip), shape)
def create_spatial_index_mesh_tasks( cloudpath:str, shape:Tuple[int,int,int] = (448,448,448), mip:int = 0, fill_missing:bool = False, compress:Optional[Union[str,bool]] = 'gzip', mesh_dir:Optional[str] = None ): """ The main way to add a spatial index is to use the MeshTask, but old datasets or broken datasets may need it to be reconstituted. An alternative use is create the spatial index over a different area size than the mesh task. """ shape = Vec(*shape) vol = CloudVolume(cloudpath, mip=mip) if mesh_dir is None: mesh_dir = f"mesh_mip_{mip}_err_40" if not "mesh" in vol.info: vol.info['mesh'] = mesh_dir vol.commit_info() cf = CloudFiles(cloudpath) info_filename = '{}/info'.format(mesh_dir) mesh_info = cf.get_json(info_filename) or {} new_mesh_info = copy.deepcopy(mesh_info) new_mesh_info['@type'] = new_mesh_info.get('@type', 'neuroglancer_legacy_mesh') new_mesh_info['mip'] = new_mesh_info.get("mip", int(vol.mip)) new_mesh_info['chunk_size'] = shape.tolist() new_mesh_info['spatial_index'] = { 'resolution': vol.resolution.tolist(), 'chunk_size': (shape * vol.resolution).tolist(), } if new_mesh_info != mesh_info: cf.put_json(info_filename, new_mesh_info) class SpatialIndexMeshTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): return partial(MeshSpatialIndex, cloudpath=cloudpath, shape=shape, offset=offset, mip=int(mip), fill_missing=bool(fill_missing), compress=compress, mesh_dir=mesh_dir, ) def on_finish(self): vol.provenance.processing.append({ 'method': { 'task': 'MeshSpatialIndex', 'cloudpath': vol.cloudpath, 'shape': shape.tolist(), 'mip': int(mip), 'mesh_dir': mesh_dir, 'fill_missing': fill_missing, 'compress': compress, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return SpatialIndexMeshTaskIterator(vol.bounds, shape)
def maybe_cache_info(self): if self.enabled: cf = CloudFiles('file://' + self.path) cf.put_json('info', self.meta.info)
def create_spatial_index_skeleton_tasks( cloudpath: str, shape: Tuple[int, int, int] = (448, 448, 448), mip: int = 0, fill_missing: bool = False, compress: Optional[Union[str, bool]] = 'gzip', skel_dir: Optional[str] = None): """ The main way to add a spatial index is to use the SkeletonTask, but old datasets or broken datasets may need it to be reconstituted. An alternative use is create the spatial index over a different area size than the skeleton task. """ shape = Vec(*shape) vol = CloudVolume(cloudpath, mip=mip) if skel_dir is None and not vol.info.get("skeletons", None): skel_dir = f"skeletons_mip_{mip}" elif skel_dir is None and vol.info.get("skeletons", None): skel_dir = vol.info["skeletons"] if not "skeletons" in vol.info: vol.info['skeletons'] = skel_dir vol.commit_info() cf = CloudFiles(cloudpath) info_filename = cf.join(skel_dir, 'info') skel_info = cf.get_json(info_filename) or {} new_skel_info = copy.deepcopy(skel_info) new_skel_info['@type'] = new_skel_info.get('@type', 'neuroglancer_skeletons') new_skel_info['mip'] = new_skel_info.get("mip", int(vol.mip)) new_skel_info['chunk_size'] = shape.tolist() new_skel_info['spatial_index'] = { 'resolution': vol.resolution.tolist(), 'chunk_size': (shape * vol.resolution).tolist(), } if new_skel_info != skel_info: cf.put_json(info_filename, new_skel_info) vol = CloudVolume(cloudpath, mip=mip) # reload spatial_index class SpatialIndexSkeletonTaskIterator(FinelyDividedTaskIterator): def task(self, shape, offset): return partial( SpatialIndexTask, cloudpath=cloudpath, shape=shape, offset=offset, subdir=skel_dir, precision=vol.skeleton.spatial_index.precision, mip=int(mip), fill_missing=bool(fill_missing), compress=compress, ) def on_finish(self): vol.provenance.processing.append({ 'method': { 'task': 'SpatialIndexTask', 'cloudpath': vol.cloudpath, 'shape': shape.tolist(), 'mip': int(mip), 'subdir': skel_dir, 'fill_missing': fill_missing, 'compress': compress, }, 'by': operator_contact(), 'date': strftime('%Y-%m-%d %H:%M %Z'), }) vol.commit_provenance() return SpatialIndexSkeletonTaskIterator(vol.bounds, shape)