def __init__( self, levels_path: str, lower_clip_fraction: float = 0.01, upper_clip_fraction: float = 0.01, minval: int = 1, maxval: int = np.iinfo(np.uint8).max, #maxval: int = np.iinfo(np.uint32).max, name: str = 'normalize-contrast', verbose: bool = True): """ levels_path: (str) path of section histogram files. clip_fraction: (float) the fraction of intensity to be clamped. minval: (float) maxval: (float) Note that the original algorithm use default maxval as float(np.iinfo(np.float32)).max. It is meaningless to use value larger than the max value of uint8. """ super().__init__(name=name, verbose=verbose) assert 0 <= lower_clip_fraction <= 1 assert 0 <= upper_clip_fraction <= 1 assert lower_clip_fraction + upper_clip_fraction <= 1 self.levels_path = levels_path self.lower_clip_fraction = float(lower_clip_fraction) self.upper_clip_fraction = float(upper_clip_fraction) self.minval = minval self.maxval = maxval # intensity value lookup table cache self.lookup_tables = dict() self.stor = SimpleStorage(self.levels_path) self.verbose = verbose
def skeldir(cloudpath): with SimpleStorage(cloudpath) as storage: info = storage.get_json('info') skel_dir = 'skeletons/' if 'skeletons' in info: skel_dir = info['skeletons'] return skel_dir
def download_chunk(meta, cache, cloudpath, mip, filename, fill_missing, enable_cache, compress_cache): with SimpleStorage(cloudpath) as stor: content = stor.get_file(filename) if enable_cache: with SimpleStorage('file://' + cache.path) as stor: stor.put_file( file_path=filename, content=(content or b''), content_type=content_type(meta.encoding(mip)), compress=compress_cache, ) bbox = Bbox.from_filename( filename) # possible off by one error w/ exclusive bounds img3d = decode(meta, filename, content, fill_missing, mip) return img3d, bbox
def get_chunk_components(components_dir, chunk_coord) -> Dict: # filename format - components_x_y_z.serliazation file_name = f"components_{'_'.join(str(coord) for coord in chunk_coord)}.proto" with SimpleStorage(components_dir) as storage: content = storage.get_file(file_name) if not content: return {} components_message = ChunkComponentsMsg() components_message.ParseFromString(content) return deserialize(components_message)
def upload_batch(self, vol, path, bbox, skeletons): with SimpleStorage(path, progress=vol.progress) as stor: # Create skeleton batch for postprocessing later stor.put_file( file_path="{}.frags".format(bbox.to_filename()), content=pickle.dumps(skeletons), compress='gzip', content_type="application/python-pickle", cache_control=False, )
def _upload_spatial_index(self, bbox, mesh_bboxes): with SimpleStorage(self.layer_path, progress=self.options['progress']) as stor: stor.put_file( file_path="{}/{}.spatial".format(self._mesh_dir, bbox.to_filename()), content=jsonify(mesh_bboxes).encode('utf8'), compress=self.options['compress'], content_type="application/json", cache_control=False, )
def put_chunk_components(components_dir, components, chunk_coord) -> None: # filename format - components_x_y_z.serliazation components_message = serialize(components) file_name = f"components_{'_'.join(str(coord) for coord in chunk_coord)}.proto" with SimpleStorage(components_dir) as storage: storage.put_file( file_path=file_name, content=components_message.SerializeToString(), compress="gzip", cache_control="no-cache", )
def _upload_batch(self, meshes, bbox): with SimpleStorage(self.layer_path, progress=self.options['progress']) as stor: # Create mesh batch for postprocessing later stor.put_file( file_path="{}/{}.frags".format(self._mesh_dir, bbox.to_filename()), content=pickle.dumps(meshes), compress=self.options['compress'], content_type="application/python-pickle", cache_control=False, )
def test_skeleton_fidelity(): segid = 1822975381 cv = CloudVolume('gs://seunglab-test/sharded') sharded_skel = cv.skeleton.get(segid) with SimpleStorage('gs://seunglab-test/sharded') as stor: binary = stor.get_file('skeletons/' + str(segid)) unsharded_skel = Skeleton.from_precomputed( binary, segid=1822975381, vertex_attributes=cv.skeleton.meta.info['vertex_attributes']) assert sharded_skel == unsharded_skel
def upload_spatial_index(self, vol, path, bbox, skeletons): spatial_index = {} for segid, skel in tqdm(skeletons.items(), disable=(not vol.progress), desc="Extracting Bounding Boxes"): segid_bbx = Bbox.from_points( skel.vertices ) spatial_index[segid] = segid_bbx.to_list() bbox = bbox * vol.resolution with SimpleStorage(path, progress=vol.progress) as stor: stor.put_file( file_path="{}.spatial".format(bbox.to_filename()), content=jsonify(spatial_index).encode('utf8'), compress='gzip', content_type="application/json", cache_control=False, )
def labels_for_shard(self, cv): """ Try to fetch precalculated labels from `$shardno.labels` (faster) otherwise, compute which labels are applicable to this shard from the shard index (much slower). """ labels = SimpleStorage( cv.skeleton.meta.layerpath).get_json(self.shard_no + '.labels') if labels is not None: return labels labels = cv.skeleton.spatial_index.query(cv.bounds * cv.resolution) spec = cv.skeleton.reader.spec return [ lbl for lbl in tqdm(labels, desc="Computing Shard Numbers", disable=(not self.progress)) \ if spec.compute_shard_location(lbl).shard_number == self.shard_no ]
def upload_batch(self, vol, path, bbox, skeletons): # neuroglancer doesn't support int attributes for shards for skel in skeletons.values(): skel.extra_attributes = [ attr for attr in skel.extra_attributes if attr['data_type'] in ('float32', 'float64') ] with SimpleStorage(path, progress=vol.progress) as stor: # Create skeleton batch for postprocessing later stor.put_file( file_path="{}.frags".format(bbox.to_filename()), content=pickle.dumps(skeletons), compress='gzip', content_type="application/python-pickle", cache_control=False, )
def test_no_vertices(): vertices = np.array([], np.float32).reshape(0,3) edges = None vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons', info=info) vol.skeleton.upload(3, vertices, edges) skel = vol.skeleton.get(3) assert skel.id == 3 assert np.all(skel.vertices == vertices) assert np.all(skel.edges.shape == (0, 2)) assert vol.skeleton.path == 'skeletons' with SimpleStorage('file:///tmp/cloudvolume/test-skeletons/') as stor: rawskel = stor.get_file('skeletons/3') assert len(rawskel) == 8 + 0 * (12 + 8 + 4 + 1) stor.delete_file('skeletons/3')
def create_key(self, email): """Creates a service account key.""" key = (self.resource.projects().serviceAccounts().keys().create( name=f"projects/-/serviceAccounts/{email}", body={}).execute()) bucket_name = os.environ["KEY_FILES_BUCKET"] bucket_gs = f"gs://{bucket_name}/keys" key_file = f"{key['name']}.json" with SimpleStorage(bucket_gs) as storage: storage.put_file( file_path=key_file, content=base64.b64decode(key["privateKeyData"]), compress=None, cache_control="no-cache", ) url = utils.generate_signed_url(bucket_name, f"keys/{key_file}") msg = f"Key created `{key['name'].split('/')[-1]}`." msg = f"{msg}\nAvailable <{url}|here> (link valid for" return f"{msg} {int(os.environ['KEY_LINK_EXPIRATION'])/60}m)."
def test_no_edges(): vertices = np.array([ [ 0, 1, 0 ], [ 1, 0, 0 ], ], np.float32) edges = None vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons', info=info) vol.skeleton.upload(2, vertices, edges) skel = vol.skeleton.get(2) assert skel.id == 2 assert np.all(skel.vertices == vertices) assert np.all(skel.edges.shape == (0, 2)) assert vol.skeleton.path == 'skeletons' with SimpleStorage('file:///tmp/cloudvolume/test-skeletons/') as stor: rawskel = stor.get_file('skeletons/2') assert len(rawskel) == 8 + 2 * (12 + 0 + 4 + 1) stor.delete_file('skeletons/2')
def upload_meshes(self, meshes): if len(meshes) == 0: return reader = self.cv.mesh.readers[self.layer_id] shard_binary = reader.spec.synthesize_shard(meshes) # the shard filename is derived from the chunk position, # so any label inside this L2 chunk will do shard_filename = reader.get_filename(list(meshes.keys())[0]) with SimpleStorage(self.cv.cloudpath) as stor: stor.put_file( file_path="{}/initial/{}/{}".format(self.get_mesh_dir(), self.layer_id, shard_filename), content=shard_binary, compress=None, content_type="application/octet-stream", cache_control="no-cache", )
def get_chunk_edges( edges_dir: str, chunks_coordinates: List[np.ndarray], cv_threads: int = 1 ) -> Dict: """ :param edges_dir: cloudvolume storage path :type str: :param chunks_coordinates: list of chunk coords for which to load edges :type List[np.ndarray]: :param cv_threads: cloudvolume storage client thread count :type int: :return: dictionary {"edge_type": Edges} """ fnames = [] for chunk_coords in chunks_coordinates: chunk_str = "_".join(str(coord) for coord in chunk_coords) # filename format - edges_x_y_z.serialization.compression fnames.append(f"edges_{chunk_str}.proto.zst") storage = ( Storage(edges_dir, n_threads=cv_threads) if cv_threads > 1 else SimpleStorage(edges_dir) ) chunk_edge_dicts = [] with storage: files = storage.get_files(fnames) for _file in files: # cv error if _file["error"]: raise ValueError(_file["error"]) # empty chunk if not _file["content"]: continue edges_dict = _decompress_edges(_file["content"]) chunk_edge_dicts.append(edges_dict) return concatenate_chunk_edges(chunk_edge_dicts)
def test_skeletons(): # Skeleton of my initials # z=0: W ; z=1 S vertices = np.array([ [0, 1, 0], [1, 0, 0], [1, 1, 0], [2, 0, 0], [2, 1, 0], [0, 0, 1], [1, 0, 1], [1, 1, 1], [0, 1, 1], [0, 2, 1], [1, 2, 1], ], np.float32) edges = np.array([ [0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 6], [6, 7], [7, 8], [8, 9], [9, 10], [10, 11], ], dtype=np.uint32) radii = np.array([ 1.0, 2.5, 3.0, 4.1, 1.2, 5.6, 2000.123123, 15.33332221, 8128.124, -1, 1824.03 ], dtype=np.float32) vertex_types = np.array([ 1, 2, 3, 5, 8, 2, 0, 5, 9, 11, 22, ], dtype=np.uint8) vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons', info=info) vol.skeleton.upload_raw(segid=1, vertices=vertices, edges=edges, radii=radii, vertex_types=vertex_types) skel = vol.skeleton.get(1) assert skel.id == 1 assert np.all(skel.vertices == vertices) assert np.all(skel.edges == edges) assert np.all(skel.radii == radii) assert np.all(skel.vertex_types == vertex_types) assert vol.skeleton.meta.skeleton_path == 'skeletons' assert not skel.empty() with SimpleStorage('file:///tmp/cloudvolume/test-skeletons/') as stor: rawskel = stor.get_file('skeletons/1') assert len(rawskel) == 8 + 11 * (12 + 8 + 4 + 1) stor.delete_file('skeletons/1') try: vol.skeleton.get(5) assert False except SkeletonDecodeError: pass
def setup_environment(dry_run, volume_start, volume_stop, volume_size, layer_path, max_ram_size, output_patch_size, input_patch_size, channel_num, dtype, output_patch_overlap, crop_chunk_margin, mip, thumbnail_mip, max_mip, thumbnail, encoding, voxel_size, overwrite_info): """Prepare storage info files and produce tasks.""" assert not (volume_stop is None and volume_size is None) if isinstance(volume_start, tuple): volume_start = Vec(*volume_start) if isinstance(volume_stop, tuple): volume_stop = Vec(*volume_stop) if isinstance(volume_size, tuple): volume_size = Vec(*volume_size) if input_patch_size is None: input_patch_size = output_patch_size if volume_size is not None: assert len(volume_size) == 3 assert volume_stop is None volume_stop = volume_start + volume_size else: volume_size = volume_stop - volume_start print('\noutput volume start: ' + tuple2string(volume_start)) print('output volume stop: ' + tuple2string(volume_stop)) print('output volume size: ' + tuple2string(volume_size)) if output_patch_overlap is None: # use 50% patch overlap in default output_patch_overlap = tuple(s//2 for s in output_patch_size) assert output_patch_overlap[1] == output_patch_overlap[2] if crop_chunk_margin is None: crop_chunk_margin = output_patch_overlap assert crop_chunk_margin[1] == crop_chunk_margin[2] print('margin size: ' + tuple2string(crop_chunk_margin)) if thumbnail: # thumnail requires maximum mip level of 5 thumbnail_mip = max(thumbnail_mip, 5) block_size, output_chunk_size, factor = get_optimized_block_size( output_patch_size, output_patch_overlap, max_ram_size, channel_num, max_mip, crop_chunk_margin, input_patch_size, mip, thumbnail_mip, volume_start ) if not dry_run: storage = SimpleStorage(layer_path) thumbnail_layer_path = os.path.join(layer_path, 'thumbnail') thumbnail_storage = SimpleStorage(thumbnail_layer_path) if not overwrite_info: print('\ncheck that we are not overwriting existing info file.') assert storage.exists('info') assert thumbnail_storage.exists('info') if overwrite_info: print('create and upload info file to ', layer_path) # Note that cloudvolume use fortran order rather than C order info = CloudVolume.create_new_info(channel_num, layer_type='image', data_type=dtype, encoding=encoding, resolution=voxel_size[::-1], voxel_offset=volume_start[::-1], volume_size=volume_size[::-1], chunk_size=block_size[::-1], max_mip=mip) vol = CloudVolume(layer_path, info=info) vol.commit_info() if overwrite_info: thumbnail_factor = 2**thumbnail_mip thumbnail_block_size = (output_chunk_size[0]//factor, output_chunk_size[1]//thumbnail_factor, output_chunk_size[2]//thumbnail_factor) print('thumbnail block size: ' + tuple2string(thumbnail_block_size)) thumbnail_info = CloudVolume.create_new_info( 1, layer_type='image', data_type='uint8', encoding='raw', resolution=voxel_size[::-1], voxel_offset=volume_start[::-1], volume_size=volume_size[::-1], chunk_size=thumbnail_block_size[::-1], max_mip=thumbnail_mip) thumbnail_vol = CloudVolume(thumbnail_layer_path, info=thumbnail_info) thumbnail_vol.commit_info() print('create a list of bounding boxes...') roi_start = (volume_start[0], volume_start[1]//factor, volume_start[2]//factor) roi_size = (volume_size[0], volume_size[1]//factor, volume_size[2]//factor) roi_stop = tuple(s+z for s, z in zip(roi_start, roi_size)) # create bounding boxes and ingest to queue bboxes = BoundingBoxes.from_manual_setup( output_chunk_size, roi_start=roi_start, roi_stop=roi_stop) logging.info(f'total number of tasks: {len(bboxes)}') logging.debug(f'bounding boxes: {bboxes}') print(yellow( 'Note that you should reuse the printed out parameters in the production run.' + ' These parameters are not ingested to AWS SQS queue.')) return bboxes
def setup_environment(dry_run, volume_start, volume_stop, volume_size, layer_path, max_ram_size, output_patch_size, input_patch_size, channel_num, dtype, output_patch_overlap, crop_chunk_margin, mip, thumbnail_mip, max_mip, queue_name, visibility_timeout, thumbnail, encoding, voxel_size, overwrite_info, verbose): """Prepare storage info files and produce tasks.""" assert not (volume_stop is None and volume_size is None) if isinstance(volume_start, tuple): volume_start = Vec(*volume_start) if isinstance(volume_stop, tuple): volume_stop = Vec(*volume_stop) if isinstance(volume_size, tuple): volume_size = Vec(*volume_size) if input_patch_size is None: input_patch_size = output_patch_size if volume_size: assert volume_stop is None volume_stop = volume_start + volume_size else: volume_size = volume_stop - volume_start print('\noutput volume start: ' + tuple2string(volume_start)) print('output volume stop: ' + tuple2string(volume_stop)) print('output volume size: ' + tuple2string(volume_size)) if output_patch_overlap is None: # use 50% patch overlap in default output_patch_overlap = tuple(s // 2 for s in output_patch_size) assert output_patch_overlap[1] == output_patch_overlap[2] if crop_chunk_margin is None: crop_chunk_margin = output_patch_overlap assert crop_chunk_margin[1] == crop_chunk_margin[2] print('margin size: ' + tuple2string(crop_chunk_margin)) if thumbnail: # thumnail requires maximum mip level of 5 thumbnail_mip = max(thumbnail_mip, 5) patch_stride = tuple( s - o for s, o in zip(output_patch_size, output_patch_overlap)) # total number of voxels per patch in one stride patch_voxel_num = np.product(patch_stride) # use half of the maximum ram size to store output buffer ideal_total_patch_num = int(max_ram_size * 1e9 / 2 / 4 / channel_num / patch_voxel_num) # the xy size should be the same assert output_patch_size[1] == output_patch_size[2] # compute the output chunk/block size in cloud storage # assume that the crop margin size is the same with the patch overlap patch_num_start = int(ideal_total_patch_num**(1. / 3.) / 2) patch_num_stop = patch_num_start * 3 # find the patch number solution with minimum cost by bruteforce search cost = sys.float_info.max patch_num = None # patch number in x and y max_factor = 2**max_mip factor = 2**mip for pnxy in range(patch_num_start, patch_num_stop): if (pnxy * patch_stride[2] + output_patch_overlap[2] - 2 * crop_chunk_margin[2]) % max_factor != 0: continue # patch number in z for pnz in range(patch_num_start, patch_num_stop): if (pnz * patch_stride[0] + output_patch_overlap[0] - 2 * crop_chunk_margin[0]) % factor != 0: continue current_cost = (pnxy * pnxy * pnz / ideal_total_patch_num - 1)**2 #+ (pnxy / pnz - 1) ** 2 if current_cost < cost: cost = current_cost patch_num = (pnz, pnxy, pnxy) print('\n--input-patch-size ', tuple2string(input_patch_size)) print('--output-patch-size ', tuple2string(output_patch_size)) print('--output-patch-overlap ', tuple2string(output_patch_overlap)) print('--output-patch-stride ', tuple2string(patch_stride)) print('--patch-num ', patch_num) assert mip >= 0 block_mip = (mip + thumbnail_mip) // 2 block_factor = 2**block_mip output_chunk_size = tuple(n * s + o - 2 * c for n, s, o, c in zip( patch_num, patch_stride, output_patch_overlap, crop_chunk_margin)) input_chunk_size = tuple( ocs + ccm * 2 + ips - ops for ocs, ccm, ips, ops in zip(output_chunk_size, crop_chunk_margin, input_patch_size, output_patch_size)) expand_margin_size = tuple( (ics - ocs) // 2 for ics, ocs in zip(input_chunk_size, output_chunk_size)) input_chunk_start = tuple( vs - ccm - (ips - ops) // 2 for vs, ccm, ips, ops in zip(volume_start, crop_chunk_margin, input_patch_size, output_patch_size)) block_size = (output_chunk_size[0] // factor, output_chunk_size[1] // block_factor, output_chunk_size[2] // block_factor) print('\n--input-chunk-size ' + tuple2string(input_chunk_size)) print('--input-volume-start ' + tuple2string(input_chunk_start)) print('--output-chunk-size ' + tuple2string(output_chunk_size)) print('cutout expand margin size ' + tuple2string(expand_margin_size)) print('output volume start: ' + tuple2string(volume_start)) print('block size ' + tuple2string(block_size)) print('RAM size of each block: ', np.prod(output_chunk_size) / 1024 / 1024 / 1024 * 4 * channel_num, ' GB') voxel_utilization = np.prod(output_chunk_size) / np.prod( patch_num) / np.prod(output_patch_size) print('voxel utilization: {:.2f}'.format(voxel_utilization)) if not dry_run: storage = SimpleStorage(layer_path) thumbnail_layer_path = os.path.join(layer_path, 'thumbnail') thumbnail_storage = SimpleStorage(thumbnail_layer_path) if not overwrite_info: print('\ncheck that we are not overwriting existing info file.') assert storage.exists('info') assert thumbnail_storage.exists('info') print('create and upload info file to ', layer_path) # Note that cloudvolume use fortran order rather than C order info = CloudVolume.create_new_info(channel_num, layer_type='image', data_type=dtype, encoding=encoding, resolution=voxel_size[::-1], voxel_offset=volume_start[::-1], volume_size=volume_size[::-1], chunk_size=block_size[::-1], max_mip=mip) vol = CloudVolume(layer_path, info=info) if overwrite_info: vol.commit_info() thumbnail_factor = 2**thumbnail_mip thumbnail_block_size = (output_chunk_size[0] // factor, output_chunk_size[1] // thumbnail_factor, output_chunk_size[2] // thumbnail_factor) print('thumbnail block size: ' + tuple2string(thumbnail_block_size)) thumbnail_info = CloudVolume.create_new_info( 1, layer_type='image', data_type='uint8', encoding='raw', resolution=voxel_size[::-1], voxel_offset=volume_start[::-1], volume_size=volume_size[::-1], chunk_size=thumbnail_block_size[::-1], max_mip=thumbnail_mip) thumbnail_vol = CloudVolume(thumbnail_layer_path, info=thumbnail_info) if overwrite_info: thumbnail_vol.commit_info() print('create a list of bounding boxes...') roi_start = (volume_start[0], volume_start[1] // factor, volume_start[2] // factor) roi_size = (volume_size[0], volume_size[1] // factor, volume_size[2] // factor) roi_stop = tuple(s + z for s, z in zip(roi_start, roi_size)) # create bounding boxes and ingest to queue bboxes = create_bounding_boxes(output_chunk_size, roi_start=roi_start, roi_stop=roi_stop, verbose=verbose) print('total number of tasks: ', len(bboxes)) if verbose > 1: print('bounding boxes: ', bboxes) return bboxes
class NormalizeSectionContrastOperator(OperatorBase): """Contrast Correction based on LuminanceLevelsTask output. Note that this operator was modified from Will's ContrastNormalizationTask in igneous: https://github.com/seung-lab/igneous/blob/master/igneous/tasks.py#L735 """ def __init__( self, levels_path: str, lower_clip_fraction: float = 0.01, upper_clip_fraction: float = 0.01, minval: int = 1, maxval: int = np.iinfo(np.uint8).max, #maxval: int = np.iinfo(np.uint32).max, name: str = 'normalize-contrast', verbose: bool = True): """ levels_path: (str) path of section histogram files. clip_fraction: (float) the fraction of intensity to be clamped. minval: (float) maxval: (float) Note that the original algorithm use default maxval as float(np.iinfo(np.float32)).max. It is meaningless to use value larger than the max value of uint8. """ super().__init__(name=name, verbose=verbose) assert 0 <= lower_clip_fraction <= 1 assert 0 <= upper_clip_fraction <= 1 assert lower_clip_fraction + upper_clip_fraction <= 1 self.levels_path = levels_path self.lower_clip_fraction = float(lower_clip_fraction) self.upper_clip_fraction = float(upper_clip_fraction) self.minval = minval self.maxval = maxval # intensity value lookup table cache self.lookup_tables = dict() self.stor = SimpleStorage(self.levels_path) self.verbose = verbose def __call__(self, chunk): # this is a image, not affinitymap assert chunk.ndim == 3 # we have to use np.dtype function to make it match # https://stackoverflow.com/questions/26921836/correct-way-to-test-for-numpy-dtype assert chunk.dtype is np.dtype(np.uint8) for z in range(chunk.bbox.minpt[-3], chunk.bbox.maxpt[-3]): lookup_table = self.fetch_lookup_table(z) slices = (slice(z, z + 1), *chunk.slices[-2:]) image = chunk.cutout(slices) image_global_offset = image.global_offset image = lookup_table[image] image = Chunk(image, global_offset=image_global_offset) chunk.save(image) return chunk def find_section_clamping_values(self, zlevel): """compute the clamping values for each section.""" # remove the np.copy from original code since we only need this once filtered = zlevel # remove pure black from frequency counts as # it has no information in our images filtered[0] = 0 cdf = np.zeros(shape=(len(filtered), ), dtype=np.uint64) cdf[0] = filtered[0] for i in range(1, len(filtered)): cdf[i] = cdf[i - 1] + filtered[i] total = cdf[-1] if total == 0: return (0, 0) lower = 0 for i, val in enumerate(cdf): if float(val) / float(total) > self.lower_clip_fraction: break lower = i upper = 0 for i, val in enumerate(cdf): if float(val) / float(total) > 1 - self.upper_clip_fraction: break upper = i return lower, upper def fetch_lookup_table(self, z): """ readout the histograms in each corresponding section. lookup tables are constructed and cached. """ if z not in self.lookup_tables: data = self.stor.get_file(f'{z}') assert data is not None, f'histogram of section {z} is missing!' data = json.loads(data.decode('utf-8')) levels = np.array(data['levels'], dtype=np.uint64) lower, upper = self.find_section_clamping_values(levels) if lower == upper: lookup_table = np.arange(0, 256, dtype=np.uint8) else: # compute the lookup table lookup_table = np.arange(0, 256, dtype=np.float32) lookup_table = (lookup_table - float(lower)) * (self.maxval / (float(upper) - float(lower))) np.clip(lookup_table, self.minval, self.maxval, out=lookup_table) lookup_table = np.round(lookup_table) lookup_table = lookup_table.astype(np.uint8) self.lookup_tables[z] = lookup_table return self.lookup_tables[z]
def get_simple_storage(): local_path = current_app.config['STORAGE_CV_PATH'] local_storage = SimpleStorage(local_path) return local_storage
def threaded_upload_chunks( meta, cache, img, mip, chunk_ranges, compress, cdn_cache, progress, n_threads=DEFAULT_THREADS, delete_black_uploads=False, background_color=0, green=False, compress_level=None, ): if cache.enabled: mkdir(cache.path) while img.ndim < 4: img = img[ ..., np.newaxis ] remotestor = lambda: SimpleStorage(meta.cloudpath, progress=progress) localstor = lambda: SimpleStorage('file://' + cache.path, progress=progress) def do_upload(imgchunk, cloudpath): encoded = chunks.encode(imgchunk, meta.encoding(mip), meta.compressed_segmentation_block_size(mip)) with remotestor() as cloudstorage: cloudstorage.put_file( file_path=cloudpath, content=encoded, content_type=content_type(meta.encoding(mip)), compress=should_compress(meta.encoding(mip), compress, cache), compress_level=compress_level, cache_control=cdn_cache_control(cdn_cache), ) if cache.enabled: with localstor() as cachestorage: cachestorage.put_file( file_path=cloudpath, content=encoded, content_type=content_type(meta.encoding(mip)), compress=should_compress(meta.encoding(mip), compress, cache, iscache=True) ) def do_delete(cloudpath): with remotestor() as cloudstorage: cloudstorage.delete_file(cloudpath) if cache.enabled: with localstor() as cachestorage: cachestorage.delete_file(cloudpath) def process(startpt, endpt, spt, ept): if np.array_equal(spt, ept): return imgchunk = img[ startpt.x:endpt.x, startpt.y:endpt.y, startpt.z:endpt.z, : ] # handle the edge of the dataset clamp_ept = min2(ept, meta.bounds(mip).maxpt) newept = clamp_ept - spt imgchunk = imgchunk[ :newept.x, :newept.y, :newept.z, : ] filename = "{}-{}_{}-{}_{}-{}".format( spt.x, clamp_ept.x, spt.y, clamp_ept.y, spt.z, clamp_ept.z ) cloudpath = meta.join(meta.key(mip), filename) if delete_black_uploads: if np.any(imgchunk != background_color): do_upload(imgchunk, cloudpath) else: do_delete(cloudpath) else: do_upload(imgchunk, cloudpath) schedule_jobs( fns=( partial(process, *vals) for vals in chunk_ranges ), concurrency=DEFAULT_THREADS, progress=('Uploading' if progress else None), total=len(chunk_ranges), green=green, )