def __init__(
self,
levels_path: str,
lower_clip_fraction: float = 0.01,
upper_clip_fraction: float = 0.01,
minval: int = 1,
maxval: int = np.iinfo(np.uint8).max,
#maxval: int = np.iinfo(np.uint32).max,
name: str = 'normalize-contrast',
verbose: bool = True):
"""
levels_path: (str) path of section histogram files.
clip_fraction: (float) the fraction of intensity to be clamped.
minval: (float)
maxval: (float) Note that the original algorithm use default maxval as
float(np.iinfo(np.float32)).max. It is meaningless to use value
larger than the max value of uint8.
"""
super().__init__(name=name, verbose=verbose)
assert 0 <= lower_clip_fraction <= 1
assert 0 <= upper_clip_fraction <= 1
assert lower_clip_fraction + upper_clip_fraction <= 1
self.levels_path = levels_path
self.lower_clip_fraction = float(lower_clip_fraction)
self.upper_clip_fraction = float(upper_clip_fraction)
self.minval = minval
self.maxval = maxval
# intensity value lookup table cache
self.lookup_tables = dict()
self.stor = SimpleStorage(self.levels_path)
self.verbose = verbose
def skeldir(cloudpath):
with SimpleStorage(cloudpath) as storage:
info = storage.get_json('info')
skel_dir = 'skeletons/'
if 'skeletons' in info:
skel_dir = info['skeletons']
return skel_dir
def download_chunk(meta, cache, cloudpath, mip, filename, fill_missing,
enable_cache, compress_cache):
with SimpleStorage(cloudpath) as stor:
content = stor.get_file(filename)
if enable_cache:
with SimpleStorage('file://' + cache.path) as stor:
stor.put_file(
file_path=filename,
content=(content or b''),
content_type=content_type(meta.encoding(mip)),
compress=compress_cache,
)
bbox = Bbox.from_filename(
filename) # possible off by one error w/ exclusive bounds
img3d = decode(meta, filename, content, fill_missing, mip)
return img3d, bbox
def get_chunk_components(components_dir, chunk_coord) -> Dict:
# filename format - components_x_y_z.serliazation
file_name = f"components_{'_'.join(str(coord) for coord in chunk_coord)}.proto"
with SimpleStorage(components_dir) as storage:
content = storage.get_file(file_name)
if not content:
return {}
components_message = ChunkComponentsMsg()
components_message.ParseFromString(content)
return deserialize(components_message)
def upload_batch(self, vol, path, bbox, skeletons):
with SimpleStorage(path, progress=vol.progress) as stor:
# Create skeleton batch for postprocessing later
stor.put_file(
file_path="{}.frags".format(bbox.to_filename()),
content=pickle.dumps(skeletons),
compress='gzip',
content_type="application/python-pickle",
cache_control=False,
)
def _upload_spatial_index(self, bbox, mesh_bboxes):
with SimpleStorage(self.layer_path,
progress=self.options['progress']) as stor:
stor.put_file(
file_path="{}/{}.spatial".format(self._mesh_dir,
bbox.to_filename()),
content=jsonify(mesh_bboxes).encode('utf8'),
compress=self.options['compress'],
content_type="application/json",
cache_control=False,
)
def put_chunk_components(components_dir, components, chunk_coord) -> None:
# filename format - components_x_y_z.serliazation
components_message = serialize(components)
file_name = f"components_{'_'.join(str(coord) for coord in chunk_coord)}.proto"
with SimpleStorage(components_dir) as storage:
storage.put_file(
file_path=file_name,
content=components_message.SerializeToString(),
compress="gzip",
cache_control="no-cache",
)
def _upload_batch(self, meshes, bbox):
with SimpleStorage(self.layer_path,
progress=self.options['progress']) as stor:
# Create mesh batch for postprocessing later
stor.put_file(
file_path="{}/{}.frags".format(self._mesh_dir,
bbox.to_filename()),
content=pickle.dumps(meshes),
compress=self.options['compress'],
content_type="application/python-pickle",
cache_control=False,
)
def test_skeleton_fidelity():
segid = 1822975381
cv = CloudVolume('gs://seunglab-test/sharded')
sharded_skel = cv.skeleton.get(segid)
with SimpleStorage('gs://seunglab-test/sharded') as stor:
binary = stor.get_file('skeletons/' + str(segid))
unsharded_skel = Skeleton.from_precomputed(
binary,
segid=1822975381,
vertex_attributes=cv.skeleton.meta.info['vertex_attributes'])
assert sharded_skel == unsharded_skel
def upload_spatial_index(self, vol, path, bbox, skeletons):
spatial_index = {}
for segid, skel in tqdm(skeletons.items(), disable=(not vol.progress), desc="Extracting Bounding Boxes"):
segid_bbx = Bbox.from_points( skel.vertices )
spatial_index[segid] = segid_bbx.to_list()
bbox = bbox * vol.resolution
with SimpleStorage(path, progress=vol.progress) as stor:
stor.put_file(
file_path="{}.spatial".format(bbox.to_filename()),
content=jsonify(spatial_index).encode('utf8'),
compress='gzip',
content_type="application/json",
cache_control=False,
)
def labels_for_shard(self, cv):
"""
Try to fetch precalculated labels from `$shardno.labels` (faster) otherwise,
compute which labels are applicable to this shard from the shard index (much slower).
"""
labels = SimpleStorage(
cv.skeleton.meta.layerpath).get_json(self.shard_no + '.labels')
if labels is not None:
return labels
labels = cv.skeleton.spatial_index.query(cv.bounds * cv.resolution)
spec = cv.skeleton.reader.spec
return [
lbl for lbl in tqdm(labels, desc="Computing Shard Numbers", disable=(not self.progress)) \
if spec.compute_shard_location(lbl).shard_number == self.shard_no
]
def upload_batch(self, vol, path, bbox, skeletons):
# neuroglancer doesn't support int attributes for shards
for skel in skeletons.values():
skel.extra_attributes = [
attr for attr in skel.extra_attributes
if attr['data_type'] in ('float32', 'float64')
]
with SimpleStorage(path, progress=vol.progress) as stor:
# Create skeleton batch for postprocessing later
stor.put_file(
file_path="{}.frags".format(bbox.to_filename()),
content=pickle.dumps(skeletons),
compress='gzip',
content_type="application/python-pickle",
cache_control=False,
)
def test_no_vertices():
vertices = np.array([], np.float32).reshape(0,3)
edges = None
vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons', info=info)
vol.skeleton.upload(3, vertices, edges)
skel = vol.skeleton.get(3)
assert skel.id == 3
assert np.all(skel.vertices == vertices)
assert np.all(skel.edges.shape == (0, 2))
assert vol.skeleton.path == 'skeletons'
with SimpleStorage('file:///tmp/cloudvolume/test-skeletons/') as stor:
rawskel = stor.get_file('skeletons/3')
assert len(rawskel) == 8 + 0 * (12 + 8 + 4 + 1)
stor.delete_file('skeletons/3')
def create_key(self, email):
"""Creates a service account key."""
key = (self.resource.projects().serviceAccounts().keys().create(
name=f"projects/-/serviceAccounts/{email}", body={}).execute())
bucket_name = os.environ["KEY_FILES_BUCKET"]
bucket_gs = f"gs://{bucket_name}/keys"
key_file = f"{key['name']}.json"
with SimpleStorage(bucket_gs) as storage:
storage.put_file(
file_path=key_file,
content=base64.b64decode(key["privateKeyData"]),
compress=None,
cache_control="no-cache",
)
url = utils.generate_signed_url(bucket_name, f"keys/{key_file}")
msg = f"Key created `{key['name'].split('/')[-1]}`."
msg = f"{msg}\nAvailable <{url}|here> (link valid for"
return f"{msg} {int(os.environ['KEY_LINK_EXPIRATION'])/60}m)."
def test_no_edges():
vertices = np.array([
[ 0, 1, 0 ],
[ 1, 0, 0 ],
], np.float32)
edges = None
vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons', info=info)
vol.skeleton.upload(2, vertices, edges)
skel = vol.skeleton.get(2)
assert skel.id == 2
assert np.all(skel.vertices == vertices)
assert np.all(skel.edges.shape == (0, 2))
assert vol.skeleton.path == 'skeletons'
with SimpleStorage('file:///tmp/cloudvolume/test-skeletons/') as stor:
rawskel = stor.get_file('skeletons/2')
assert len(rawskel) == 8 + 2 * (12 + 0 + 4 + 1)
stor.delete_file('skeletons/2')
def upload_meshes(self, meshes):
if len(meshes) == 0:
return
reader = self.cv.mesh.readers[self.layer_id]
shard_binary = reader.spec.synthesize_shard(meshes)
# the shard filename is derived from the chunk position,
# so any label inside this L2 chunk will do
shard_filename = reader.get_filename(list(meshes.keys())[0])
with SimpleStorage(self.cv.cloudpath) as stor:
stor.put_file(
file_path="{}/initial/{}/{}".format(self.get_mesh_dir(),
self.layer_id,
shard_filename),
content=shard_binary,
compress=None,
content_type="application/octet-stream",
cache_control="no-cache",
)
def get_chunk_edges(
edges_dir: str, chunks_coordinates: List[np.ndarray], cv_threads: int = 1
) -> Dict:
"""
:param edges_dir: cloudvolume storage path
:type str:
:param chunks_coordinates: list of chunk coords for which to load edges
:type List[np.ndarray]:
:param cv_threads: cloudvolume storage client thread count
:type int:
:return: dictionary {"edge_type": Edges}
"""
fnames = []
for chunk_coords in chunks_coordinates:
chunk_str = "_".join(str(coord) for coord in chunk_coords)
# filename format - edges_x_y_z.serialization.compression
fnames.append(f"edges_{chunk_str}.proto.zst")
storage = (
Storage(edges_dir, n_threads=cv_threads)
if cv_threads > 1
else SimpleStorage(edges_dir)
)
chunk_edge_dicts = []
with storage:
files = storage.get_files(fnames)
for _file in files:
# cv error
if _file["error"]:
raise ValueError(_file["error"])
# empty chunk
if not _file["content"]:
continue
edges_dict = _decompress_edges(_file["content"])
chunk_edge_dicts.append(edges_dict)
return concatenate_chunk_edges(chunk_edge_dicts)
def test_skeletons():
# Skeleton of my initials
# z=0: W ; z=1 S
vertices = np.array([
[0, 1, 0],
[1, 0, 0],
[1, 1, 0],
[2, 0, 0],
[2, 1, 0],
[0, 0, 1],
[1, 0, 1],
[1, 1, 1],
[0, 1, 1],
[0, 2, 1],
[1, 2, 1],
], np.float32)
edges = np.array([
[0, 1],
[1, 2],
[2, 3],
[3, 4],
[4, 5],
[5, 6],
[6, 7],
[7, 8],
[8, 9],
[9, 10],
[10, 11],
],
dtype=np.uint32)
radii = np.array([
1.0, 2.5, 3.0, 4.1, 1.2, 5.6, 2000.123123, 15.33332221, 8128.124, -1,
1824.03
],
dtype=np.float32)
vertex_types = np.array([
1,
2,
3,
5,
8,
2,
0,
5,
9,
11,
22,
],
dtype=np.uint8)
vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons', info=info)
vol.skeleton.upload_raw(segid=1,
vertices=vertices,
edges=edges,
radii=radii,
vertex_types=vertex_types)
skel = vol.skeleton.get(1)
assert skel.id == 1
assert np.all(skel.vertices == vertices)
assert np.all(skel.edges == edges)
assert np.all(skel.radii == radii)
assert np.all(skel.vertex_types == vertex_types)
assert vol.skeleton.meta.skeleton_path == 'skeletons'
assert not skel.empty()
with SimpleStorage('file:///tmp/cloudvolume/test-skeletons/') as stor:
rawskel = stor.get_file('skeletons/1')
assert len(rawskel) == 8 + 11 * (12 + 8 + 4 + 1)
stor.delete_file('skeletons/1')
try:
vol.skeleton.get(5)
assert False
except SkeletonDecodeError:
pass
def setup_environment(dry_run, volume_start, volume_stop, volume_size, layer_path,
max_ram_size, output_patch_size,
input_patch_size, channel_num, dtype,
output_patch_overlap, crop_chunk_margin, mip, thumbnail_mip, max_mip,
thumbnail, encoding, voxel_size,
overwrite_info):
"""Prepare storage info files and produce tasks."""
assert not (volume_stop is None and volume_size is None)
if isinstance(volume_start, tuple):
volume_start = Vec(*volume_start)
if isinstance(volume_stop, tuple):
volume_stop = Vec(*volume_stop)
if isinstance(volume_size, tuple):
volume_size = Vec(*volume_size)
if input_patch_size is None:
input_patch_size = output_patch_size
if volume_size is not None:
assert len(volume_size) == 3
assert volume_stop is None
volume_stop = volume_start + volume_size
else:
volume_size = volume_stop - volume_start
print('\noutput volume start: ' + tuple2string(volume_start))
print('output volume stop: ' + tuple2string(volume_stop))
print('output volume size: ' + tuple2string(volume_size))
if output_patch_overlap is None:
# use 50% patch overlap in default
output_patch_overlap = tuple(s//2 for s in output_patch_size)
assert output_patch_overlap[1] == output_patch_overlap[2]
if crop_chunk_margin is None:
crop_chunk_margin = output_patch_overlap
assert crop_chunk_margin[1] == crop_chunk_margin[2]
print('margin size: ' + tuple2string(crop_chunk_margin))
if thumbnail:
# thumnail requires maximum mip level of 5
thumbnail_mip = max(thumbnail_mip, 5)
block_size, output_chunk_size, factor = get_optimized_block_size(
output_patch_size, output_patch_overlap, max_ram_size,
channel_num, max_mip, crop_chunk_margin,
input_patch_size, mip, thumbnail_mip, volume_start
)
if not dry_run:
storage = SimpleStorage(layer_path)
thumbnail_layer_path = os.path.join(layer_path, 'thumbnail')
thumbnail_storage = SimpleStorage(thumbnail_layer_path)
if not overwrite_info:
print('\ncheck that we are not overwriting existing info file.')
assert storage.exists('info')
assert thumbnail_storage.exists('info')
if overwrite_info:
print('create and upload info file to ', layer_path)
# Note that cloudvolume use fortran order rather than C order
info = CloudVolume.create_new_info(channel_num, layer_type='image',
data_type=dtype,
encoding=encoding,
resolution=voxel_size[::-1],
voxel_offset=volume_start[::-1],
volume_size=volume_size[::-1],
chunk_size=block_size[::-1],
max_mip=mip)
vol = CloudVolume(layer_path, info=info)
vol.commit_info()
if overwrite_info:
thumbnail_factor = 2**thumbnail_mip
thumbnail_block_size = (output_chunk_size[0]//factor,
output_chunk_size[1]//thumbnail_factor,
output_chunk_size[2]//thumbnail_factor)
print('thumbnail block size: ' + tuple2string(thumbnail_block_size))
thumbnail_info = CloudVolume.create_new_info(
1, layer_type='image',
data_type='uint8',
encoding='raw',
resolution=voxel_size[::-1],
voxel_offset=volume_start[::-1],
volume_size=volume_size[::-1],
chunk_size=thumbnail_block_size[::-1],
max_mip=thumbnail_mip)
thumbnail_vol = CloudVolume(thumbnail_layer_path, info=thumbnail_info)
thumbnail_vol.commit_info()
print('create a list of bounding boxes...')
roi_start = (volume_start[0],
volume_start[1]//factor,
volume_start[2]//factor)
roi_size = (volume_size[0],
volume_size[1]//factor,
volume_size[2]//factor)
roi_stop = tuple(s+z for s, z in zip(roi_start, roi_size))
# create bounding boxes and ingest to queue
bboxes = BoundingBoxes.from_manual_setup(
output_chunk_size,
roi_start=roi_start, roi_stop=roi_stop)
logging.info(f'total number of tasks: {len(bboxes)}')
logging.debug(f'bounding boxes: {bboxes}')
print(yellow(
'Note that you should reuse the printed out parameters in the production run.' +
' These parameters are not ingested to AWS SQS queue.'))
return bboxes
def setup_environment(dry_run, volume_start, volume_stop, volume_size,
layer_path, max_ram_size, output_patch_size,
input_patch_size, channel_num, dtype,
output_patch_overlap, crop_chunk_margin, mip,
thumbnail_mip, max_mip, queue_name, visibility_timeout,
thumbnail, encoding, voxel_size, overwrite_info,
verbose):
"""Prepare storage info files and produce tasks."""
assert not (volume_stop is None and volume_size is None)
if isinstance(volume_start, tuple):
volume_start = Vec(*volume_start)
if isinstance(volume_stop, tuple):
volume_stop = Vec(*volume_stop)
if isinstance(volume_size, tuple):
volume_size = Vec(*volume_size)
if input_patch_size is None:
input_patch_size = output_patch_size
if volume_size:
assert volume_stop is None
volume_stop = volume_start + volume_size
else:
volume_size = volume_stop - volume_start
print('\noutput volume start: ' + tuple2string(volume_start))
print('output volume stop: ' + tuple2string(volume_stop))
print('output volume size: ' + tuple2string(volume_size))
if output_patch_overlap is None:
# use 50% patch overlap in default
output_patch_overlap = tuple(s // 2 for s in output_patch_size)
assert output_patch_overlap[1] == output_patch_overlap[2]
if crop_chunk_margin is None:
crop_chunk_margin = output_patch_overlap
assert crop_chunk_margin[1] == crop_chunk_margin[2]
print('margin size: ' + tuple2string(crop_chunk_margin))
if thumbnail:
# thumnail requires maximum mip level of 5
thumbnail_mip = max(thumbnail_mip, 5)
patch_stride = tuple(
s - o for s, o in zip(output_patch_size, output_patch_overlap))
# total number of voxels per patch in one stride
patch_voxel_num = np.product(patch_stride)
# use half of the maximum ram size to store output buffer
ideal_total_patch_num = int(max_ram_size * 1e9 / 2 / 4 / channel_num /
patch_voxel_num)
# the xy size should be the same
assert output_patch_size[1] == output_patch_size[2]
# compute the output chunk/block size in cloud storage
# assume that the crop margin size is the same with the patch overlap
patch_num_start = int(ideal_total_patch_num**(1. / 3.) / 2)
patch_num_stop = patch_num_start * 3
# find the patch number solution with minimum cost by bruteforce search
cost = sys.float_info.max
patch_num = None
# patch number in x and y
max_factor = 2**max_mip
factor = 2**mip
for pnxy in range(patch_num_start, patch_num_stop):
if (pnxy * patch_stride[2] + output_patch_overlap[2] -
2 * crop_chunk_margin[2]) % max_factor != 0:
continue
# patch number in z
for pnz in range(patch_num_start, patch_num_stop):
if (pnz * patch_stride[0] + output_patch_overlap[0] -
2 * crop_chunk_margin[0]) % factor != 0:
continue
current_cost = (pnxy * pnxy * pnz / ideal_total_patch_num -
1)**2 #+ (pnxy / pnz - 1) ** 2
if current_cost < cost:
cost = current_cost
patch_num = (pnz, pnxy, pnxy)
print('\n--input-patch-size ', tuple2string(input_patch_size))
print('--output-patch-size ', tuple2string(output_patch_size))
print('--output-patch-overlap ', tuple2string(output_patch_overlap))
print('--output-patch-stride ', tuple2string(patch_stride))
print('--patch-num ', patch_num)
assert mip >= 0
block_mip = (mip + thumbnail_mip) // 2
block_factor = 2**block_mip
output_chunk_size = tuple(n * s + o - 2 * c for n, s, o, c in zip(
patch_num, patch_stride, output_patch_overlap, crop_chunk_margin))
input_chunk_size = tuple(
ocs + ccm * 2 + ips - ops
for ocs, ccm, ips, ops in zip(output_chunk_size, crop_chunk_margin,
input_patch_size, output_patch_size))
expand_margin_size = tuple(
(ics - ocs) // 2
for ics, ocs in zip(input_chunk_size, output_chunk_size))
input_chunk_start = tuple(
vs - ccm - (ips - ops) // 2
for vs, ccm, ips, ops in zip(volume_start, crop_chunk_margin,
input_patch_size, output_patch_size))
block_size = (output_chunk_size[0] // factor,
output_chunk_size[1] // block_factor,
output_chunk_size[2] // block_factor)
print('\n--input-chunk-size ' + tuple2string(input_chunk_size))
print('--input-volume-start ' + tuple2string(input_chunk_start))
print('--output-chunk-size ' + tuple2string(output_chunk_size))
print('cutout expand margin size ' + tuple2string(expand_margin_size))
print('output volume start: ' + tuple2string(volume_start))
print('block size ' + tuple2string(block_size))
print('RAM size of each block: ',
np.prod(output_chunk_size) / 1024 / 1024 / 1024 * 4 * channel_num,
' GB')
voxel_utilization = np.prod(output_chunk_size) / np.prod(
patch_num) / np.prod(output_patch_size)
print('voxel utilization: {:.2f}'.format(voxel_utilization))
if not dry_run:
storage = SimpleStorage(layer_path)
thumbnail_layer_path = os.path.join(layer_path, 'thumbnail')
thumbnail_storage = SimpleStorage(thumbnail_layer_path)
if not overwrite_info:
print('\ncheck that we are not overwriting existing info file.')
assert storage.exists('info')
assert thumbnail_storage.exists('info')
print('create and upload info file to ', layer_path)
# Note that cloudvolume use fortran order rather than C order
info = CloudVolume.create_new_info(channel_num,
layer_type='image',
data_type=dtype,
encoding=encoding,
resolution=voxel_size[::-1],
voxel_offset=volume_start[::-1],
volume_size=volume_size[::-1],
chunk_size=block_size[::-1],
max_mip=mip)
vol = CloudVolume(layer_path, info=info)
if overwrite_info:
vol.commit_info()
thumbnail_factor = 2**thumbnail_mip
thumbnail_block_size = (output_chunk_size[0] // factor,
output_chunk_size[1] // thumbnail_factor,
output_chunk_size[2] // thumbnail_factor)
print('thumbnail block size: ' + tuple2string(thumbnail_block_size))
thumbnail_info = CloudVolume.create_new_info(
1,
layer_type='image',
data_type='uint8',
encoding='raw',
resolution=voxel_size[::-1],
voxel_offset=volume_start[::-1],
volume_size=volume_size[::-1],
chunk_size=thumbnail_block_size[::-1],
max_mip=thumbnail_mip)
thumbnail_vol = CloudVolume(thumbnail_layer_path, info=thumbnail_info)
if overwrite_info:
thumbnail_vol.commit_info()
print('create a list of bounding boxes...')
roi_start = (volume_start[0], volume_start[1] // factor,
volume_start[2] // factor)
roi_size = (volume_size[0], volume_size[1] // factor,
volume_size[2] // factor)
roi_stop = tuple(s + z for s, z in zip(roi_start, roi_size))
# create bounding boxes and ingest to queue
bboxes = create_bounding_boxes(output_chunk_size,
roi_start=roi_start,
roi_stop=roi_stop,
verbose=verbose)
print('total number of tasks: ', len(bboxes))
if verbose > 1:
print('bounding boxes: ', bboxes)
return bboxes
class NormalizeSectionContrastOperator(OperatorBase):
"""Contrast Correction based on LuminanceLevelsTask output.
Note that this operator was modified from Will's ContrastNormalizationTask in igneous:
https://github.com/seung-lab/igneous/blob/master/igneous/tasks.py#L735
"""
def __init__(
self,
levels_path: str,
lower_clip_fraction: float = 0.01,
upper_clip_fraction: float = 0.01,
minval: int = 1,
maxval: int = np.iinfo(np.uint8).max,
#maxval: int = np.iinfo(np.uint32).max,
name: str = 'normalize-contrast',
verbose: bool = True):
"""
levels_path: (str) path of section histogram files.
clip_fraction: (float) the fraction of intensity to be clamped.
minval: (float)
maxval: (float) Note that the original algorithm use default maxval as
float(np.iinfo(np.float32)).max. It is meaningless to use value
larger than the max value of uint8.
"""
super().__init__(name=name, verbose=verbose)
assert 0 <= lower_clip_fraction <= 1
assert 0 <= upper_clip_fraction <= 1
assert lower_clip_fraction + upper_clip_fraction <= 1
self.levels_path = levels_path
self.lower_clip_fraction = float(lower_clip_fraction)
self.upper_clip_fraction = float(upper_clip_fraction)
self.minval = minval
self.maxval = maxval
# intensity value lookup table cache
self.lookup_tables = dict()
self.stor = SimpleStorage(self.levels_path)
self.verbose = verbose
def __call__(self, chunk):
# this is a image, not affinitymap
assert chunk.ndim == 3
# we have to use np.dtype function to make it match
# https://stackoverflow.com/questions/26921836/correct-way-to-test-for-numpy-dtype
assert chunk.dtype is np.dtype(np.uint8)
for z in range(chunk.bbox.minpt[-3], chunk.bbox.maxpt[-3]):
lookup_table = self.fetch_lookup_table(z)
slices = (slice(z, z + 1), *chunk.slices[-2:])
image = chunk.cutout(slices)
image_global_offset = image.global_offset
image = lookup_table[image]
image = Chunk(image, global_offset=image_global_offset)
chunk.save(image)
return chunk
def find_section_clamping_values(self, zlevel):
"""compute the clamping values for each section."""
# remove the np.copy from original code since we only need this once
filtered = zlevel
# remove pure black from frequency counts as
# it has no information in our images
filtered[0] = 0
cdf = np.zeros(shape=(len(filtered), ), dtype=np.uint64)
cdf[0] = filtered[0]
for i in range(1, len(filtered)):
cdf[i] = cdf[i - 1] + filtered[i]
total = cdf[-1]
if total == 0:
return (0, 0)
lower = 0
for i, val in enumerate(cdf):
if float(val) / float(total) > self.lower_clip_fraction:
break
lower = i
upper = 0
for i, val in enumerate(cdf):
if float(val) / float(total) > 1 - self.upper_clip_fraction:
break
upper = i
return lower, upper
def fetch_lookup_table(self, z):
"""
readout the histograms in each corresponding section.
lookup tables are constructed and cached.
"""
if z not in self.lookup_tables:
data = self.stor.get_file(f'{z}')
assert data is not None, f'histogram of section {z} is missing!'
data = json.loads(data.decode('utf-8'))
levels = np.array(data['levels'], dtype=np.uint64)
lower, upper = self.find_section_clamping_values(levels)
if lower == upper:
lookup_table = np.arange(0, 256, dtype=np.uint8)
else:
# compute the lookup table
lookup_table = np.arange(0, 256, dtype=np.float32)
lookup_table = (lookup_table -
float(lower)) * (self.maxval /
(float(upper) - float(lower)))
np.clip(lookup_table, self.minval, self.maxval, out=lookup_table)
lookup_table = np.round(lookup_table)
lookup_table = lookup_table.astype(np.uint8)
self.lookup_tables[z] = lookup_table
return self.lookup_tables[z]
def get_simple_storage():
local_path = current_app.config['STORAGE_CV_PATH']
local_storage = SimpleStorage(local_path)
return local_storage
def threaded_upload_chunks(
meta, cache,
img, mip, chunk_ranges,
compress, cdn_cache, progress,
n_threads=DEFAULT_THREADS,
delete_black_uploads=False,
background_color=0,
green=False,
compress_level=None,
):
if cache.enabled:
mkdir(cache.path)
while img.ndim < 4:
img = img[ ..., np.newaxis ]
remotestor = lambda: SimpleStorage(meta.cloudpath, progress=progress)
localstor = lambda: SimpleStorage('file://' + cache.path, progress=progress)
def do_upload(imgchunk, cloudpath):
encoded = chunks.encode(imgchunk, meta.encoding(mip), meta.compressed_segmentation_block_size(mip))
with remotestor() as cloudstorage:
cloudstorage.put_file(
file_path=cloudpath,
content=encoded,
content_type=content_type(meta.encoding(mip)),
compress=should_compress(meta.encoding(mip), compress, cache),
compress_level=compress_level,
cache_control=cdn_cache_control(cdn_cache),
)
if cache.enabled:
with localstor() as cachestorage:
cachestorage.put_file(
file_path=cloudpath,
content=encoded,
content_type=content_type(meta.encoding(mip)),
compress=should_compress(meta.encoding(mip), compress, cache, iscache=True)
)
def do_delete(cloudpath):
with remotestor() as cloudstorage:
cloudstorage.delete_file(cloudpath)
if cache.enabled:
with localstor() as cachestorage:
cachestorage.delete_file(cloudpath)
def process(startpt, endpt, spt, ept):
if np.array_equal(spt, ept):
return
imgchunk = img[ startpt.x:endpt.x, startpt.y:endpt.y, startpt.z:endpt.z, : ]
# handle the edge of the dataset
clamp_ept = min2(ept, meta.bounds(mip).maxpt)
newept = clamp_ept - spt
imgchunk = imgchunk[ :newept.x, :newept.y, :newept.z, : ]
filename = "{}-{}_{}-{}_{}-{}".format(
spt.x, clamp_ept.x,
spt.y, clamp_ept.y,
spt.z, clamp_ept.z
)
cloudpath = meta.join(meta.key(mip), filename)
if delete_black_uploads:
if np.any(imgchunk != background_color):
do_upload(imgchunk, cloudpath)
else:
do_delete(cloudpath)
else:
do_upload(imgchunk, cloudpath)
schedule_jobs(
fns=( partial(process, *vals) for vals in chunk_ranges ),
concurrency=DEFAULT_THREADS,
progress=('Uploading' if progress else None),
total=len(chunk_ranges),
green=green,
)
