def create_binarized_vol(vol_path_bin,
vol_path_old,
ltype="image",
dtype="float32",
res=0,
parallel=False):
vol = CloudVolume(vol_path_old)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=ltype,
data_type=dtype, # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=vol.scales[res]
["resolution"], # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=[1024, 1024, 1], # units are voxels
volume_size=vol.scales[res]["size"],
)
bin_vol = CloudVolume(vol_path_bin, info=info, parallel=parallel)
[
bin_vol.add_scale((2**i, 2**i, 1), chunk_size=[1024, 1024, 1])
for i in range(6)
]
bin_vol.commit_info()
return bin_vol
def create_cloud_volume(
precomputed_path,
img_size,
voxel_size,
num_downsampling_levels=6,
chunk_size=[1024, 1024, 1],
parallel=False,
layer_type="image",
dtype="uint16",
):
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=dtype, # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=voxel_size, # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=chunk_size, # units are voxels
volume_size=img_size, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(precomputed_path, info=info, parallel=parallel)
[
vol.add_scale((2**i, 2**i, 1), chunk_size=chunk_size)
for i in range(num_downsampling_levels)
]
vol.commit_info()
return vol
def prepare_precomputed(precomputed_path,
offset,
size,
resolution,
chunk_size,
factor=(2, 2, 1),
dtype='uint32'):
cv_args = dict(bounded=True,
fill_missing=False,
autocrop=False,
cache=False,
compress_cache=None,
cdn_cache=False,
progress=False,
provenance=None,
compress=True,
non_aligned_writes=True,
parallel=True)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type='segmentation',
data_type=dtype,
encoding='compressed_segmentation',
resolution=list(resolution),
voxel_offset=np.array(offset),
volume_size=np.array(size),
chunk_size=chunk_size,
max_mip=0,
factor=factor,
)
cv = CloudVolume('file://' + precomputed_path, mip=0, info=info, **cv_args)
cv.commit_info()
return cv
def create_info_file_from_build(layer_path, layer_type, resolution, encoding):
assert layer_type in ('image', 'segmentation', 'affinities')
with Storage(layer_path) as storage:
bounds, build_chunk_size = compute_build_bounding_box(storage)
data_type, num_channels = get_build_data_type_and_shape(storage)
neuroglancer_chunk_size = find_closest_divisor(build_chunk_size,
closest_to=[64, 64, 64])
info = CloudVolume.create_new_info(
num_channels=num_channels,
layer_type=layer_type,
data_type=data_type,
encoding=encoding,
resolution=resolution,
voxel_offset=bounds.minpt.tolist(),
volume_size=bounds.size3(),
mesh=(layer_type == 'segmentation'),
chunk_size=list(map(int, neuroglancer_chunk_size)),
)
vol = CloudVolume(layer_path, mip=0, info=info).commit_info()
vol = create_downsample_scales(layer_path,
mip=0,
ds_shape=build_chunk_size,
axis='z')
return vol.info
def test_fill_missing():
info = CloudVolume.create_new_info(
num_channels=1, # Increase this number when we add more tests for RGB
layer_type='image',
data_type='uint8',
encoding='raw',
resolution=[1, 1, 1],
voxel_offset=[0, 0, 0],
volume_size=[128, 128, 64],
mesh='mesh',
chunk_size=[64, 64, 64],
)
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume', mip=0, info=info)
vol.commit_info()
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume',
mip=0,
fill_missing=True)
assert np.count_nonzero(vol[:]) == 0
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume',
mip=0,
fill_missing=True,
cache=True)
assert np.count_nonzero(vol[:]) == 0
assert np.count_nonzero(vol[:]) == 0
vol.cache.flush()
delete_layer('/tmp/cloudvolume/empty_volume')
def upload_cloud(cloud_dir, volume, layer_type, dtype, resolution, volume_size, voxel_offset=[0,0,0], prov_description=""):
"""
cloud_dir : Cloud directory to upload
volume : Volume to upload
layer_type : 'image' or 'segmentation'
dtype : Has to match volume's data type
resolution : Resolution of voxel
volume_size : Volume size
voxel_offset : Volume offset
prov_description : Provenance description
"""
info = CloudVolume.create_new_info(
num_channels = 1,
layer_type = layer_type, # 'image' or 'segmentation'
data_type = dtype, # can pick any popular uint
encoding = 'raw', # other option: 'jpeg' but it's lossy
resolution = resolution, # X,Y,Z values in nanometers
voxel_offset = voxel_offset, # values X,Y,Z values in voxels
chunk_size = [ 128, 128, 1 ], # rechunk of image X,Y,Z in voxels
volume_size = volume_size, # X,Y,Z size in voxels
)
vol = CloudVolume(cloud_dir, parallel=True, progress=True, cdn_cache=False, info=info)
vol.provenance.description = prov_description
vol.provenance.owners = ['*****@*****.**'] # list of contact email addresses
vol.commit_info() # generates gs://bucket/dataset/layer/info json file
vol.commit_provenance() # generates gs://bucket/dataset/layer/provenance json file
vol[:,:,:] = volume
def create_volume_from_image(image, offset, layer_path, layer_type, resolution, encoding):
assert layer_type in ('image', 'segmentation', 'affinities')
offset = Vec(*offset)
volsize = Vec(*image.shape[:3])
data_type = str(image.dtype)
bounds = Bbox(offset, offset + volsize)
neuroglancer_chunk_size = find_closest_divisor(image.shape[:3], closest_to=[64,64,64])
info = CloudVolume.create_new_info(
num_channels=1, # Increase this number when we add more tests for RGB
layer_type=layer_type,
data_type=data_type,
encoding=encoding,
resolution=resolution,
voxel_offset=bounds.minpt,
volume_size=bounds.size3(),
mesh=(layer_type == 'segmentation'),
chunk_size=neuroglancer_chunk_size,
)
vol = CloudVolume(layer_path, mip=0, info=info)
vol.commit_info()
vol[:,:,:] = image
return vol
def test_compression_methods(compress_method):
path = "file:///tmp/cloudvolume/test" + '-' + str(TEST_NUMBER)
# create a NG volume
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="image",
data_type="uint16", # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=[4, 4, 40], # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
chunk_size=[512, 512, 16], # units are voxels
volume_size=[512 * 4, 512 * 4, 16 * 40],
)
vol = CloudVolume(path, info=info, compress=compress_method)
vol.commit_info()
image = np.random.randint(low=0,
high=2**16 - 1,
size=(512, 512, 16, 1),
dtype="uint16")
vol[0:512, 0:512, 0:16] = image
image_test = vol[0:512, 0:512, 0:16]
delete_layer('/tmp/cloudvolume/test' + '-' + str(TEST_NUMBER))
assert vol.compress == compress_method
assert np.array_equal(image_test, image)
def make_info_file(brain, home_dir, volume_size, type_vol="647", commit=True):
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="image", # "image" or "segmentation"
data_type=
"uint16", # 32 not necessary for Princeton atlas, but was for Allen atlas
encoding=
"raw", # other options: "jpeg", "compressed_segmentation" (req. uint32 or uint64)
resolution=[1630, 1630, 3000
], # X,Y,Z values in nanometers, 40 microns in each dim.
voxel_offset=[0, 0, 0], # values X,Y,Z values in voxels
chunk_size=[1024, 1024, 32], # rechunk of image X,Y,Z in voxels,
volume_size=volume_size, # X,Y,Z size in voxels
)
# If you"re using amazon or the local file system, you can replace "gs" with "s3" or "file"
vol = CloudVolume("file://" + home_dir + "/" + brain + "/" + type_vol,
info=info)
vol.provenance.description = "TP tracing"
vol.provenance.owners = ["*****@*****.**"
] # list of contact email addresses
if commit:
vol.commit_info() # generates gs://bucket/dataset/layer/info json file
vol.commit_provenance(
) # generates gs://bucket/dataset/layer/provenance json file
print("Created CloudVolume info file: ", vol.info_cloudpath)
return vol
def init_precomputed(self,
path,
volume_size,
starting_points=None,
progress_id=None):
info = CloudVolume.create_new_info(
num_channels=self.num_channels,
layer_type=self.layer_type, # 'image' or 'segmentation'
data_type=self.data_type, #
encoding=
'raw', # other options: 'jpeg', 'compressed_segmentation' (req. uint32 or uint64)
resolution=self.scales, # Size of X,Y,Z pixels in nanometers,
voxel_offset=self.offset, # values X,Y,Z values in voxels
chunk_size=self.chunk_size, # rechunk of image X,Y,Z in voxels
volume_size=volume_size, # X,Y,Z size in voxels
)
self.starting_points = starting_points
self.progress_id = progress_id
self.precomputed_vol = CloudVolume(f'file://{path}',
mip=0,
info=info,
compress=True,
progress=False)
self.precomputed_vol.commit_info()
self.precomputed_vol.commit_provenance()
def make_info_file(volume_size,resolution,layer_dir,commit=True):
"""
---PURPOSE---
Make the cloudvolume info file.
---INPUT---
volume_size [Nx,Ny,Nz] in voxels, e.g. [2160,2560,1271]
pix_scale_nm [size of x pix in nm,size of y pix in nm,size of z pix in nm], e.g. [5000,5000,10000]
commit if True, will write the info/provenance file to disk.
if False, just creates it in memory
"""
info = CloudVolume.create_new_info(
num_channels = 1,
layer_type = 'segmentation', # 'image' or 'segmentation'
data_type = 'uint16', #
encoding = 'raw', # other options: 'jpeg', 'compressed_segmentation' (req. uint32 or uint64)
resolution = resolution, # Size of X,Y,Z pixels in nanometers,
voxel_offset = [ 0, 0, 0 ], # values X,Y,Z values in voxels
chunk_size = [ 1024,1024,1 ], # rechunk of image X,Y,Z in voxels -- only used for downsampling task I think
volume_size = volume_size, # X,Y,Z size in voxels
)
vol = CloudVolume(f'file://{layer_dir}', info=info)
vol.provenance.description = "Test on spock for profiling precomputed creation"
vol.provenance.owners = ['*****@*****.**'] # list of contact email addresses
if commit:
vol.commit_info() # generates info json file
vol.commit_provenance() # generates provenance json file
print("Created CloudVolume info file: ",vol.info_cloudpath)
return vol
def create_cloud_volume(precomputed_path,
img_size,
voxel_size,
num_hierarchy_levels=5,
parallel=True):
global args
if args.dtype == 'uint64':
layer_type = 'segmentation'
else:
layer_type = 'image'
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=args.dtype, # Channel images might be 'uint8'
encoding='raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=voxel_size, # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=[512, 512, 1], # units are voxels
volume_size=img_size, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(precomputed_path, info=info, parallel=parallel)
# add mip 1
[
vol.add_scale((2**i, 2**i, 1), chunk_size=[512, 512, 1])
for i in range(num_hierarchy_levels)
]
vol.commit_info()
return vol
def main():
parser = argparse.ArgumentParser(
description='Generate field for training set module.')
parser.add_argument('--pyramid_path', type=str)
parser.add_argument('--checkpoint_name', type=str, default="checkpoint")
parser.add_argument('--image_path', type=str)
parser.add_argument('--prev_field_path', type=str)
# parser.add_argument('--field_path', type=str)
parser.add_argument('--dst_dir', type=str)
parser.add_argument('--gpu', type=str, default="0")
parser.add_argument('--stage', type=int)
parser.add_argument('--src_mip', type=int, help='MIP of input')
parser.add_argument('--dst_mip', type=int, help='MIP of output')
parser.add_argument('--save_intermediary', action='store_true')
parser.add_argument('--port', type=int, default=8888)
parser.set_defaults(redirect_stdout=True)
args = parser.parse_args()
assert (os.path.exists(args.pyramid_path))
assert (os.path.exists(args.image_path))
assert (os.path.exists(args.prev_field_path))
world_size = len(args.gpu.split(','))
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
os.environ['MASTER_ADDR'] = 'localhost'
os.environ['MASTER_PORT'] = str(args.port)
img_dset = h5py.File(args.image_path, 'r')['main']
down_res = Vec(2**args.src_mip, 2**args.src_mip, 1)
up_res = Vec(2**args.dst_mip, 2**args.dst_mip, 1)
for mip in [args.src_mip, args.dst_mip]:
res = Vec(2**mip, 2**mip, 1)
path = os.path.join(args.dst_dir, str(mip))
info = CloudVolume.create_new_info(
num_channels=2,
layer_type='image',
data_type='float32',
encoding='raw',
resolution=res,
voxel_offset=[0, 0, 0],
chunk_size=[img_dset.shape[-1], img_dset.shape[-2], 1],
volume_size=[
img_dset.shape[-1], img_dset.shape[-2], img_dset.shape[0]
])
cv = CloudVolume(path, mip=0, info=info, cdn_cache=False)
cv.commit_info()
generate_shards_distributed(world_size=world_size,
pyramid_path=args.pyramid_path,
stage=args.stage,
checkpoint_name=args.checkpoint_name,
img_path=args.image_path,
prev_field_path=args.prev_field_path,
dst_dir=args.dst_dir,
src_mip=args.src_mip,
dst_mip=args.dst_mip)
def ingest(args):
"""Ingest CATMAID tiles with same row,col index across z range
Args:
args: ArgParse object from main
"""
mip = args.mip
row = args.row
col = args.col
chunk_size = Vec(*args.chunk_size)
bbox = Bbox(bbox_start, bbox_start + bbox_size)
chunk_size = [1024, 1024, 1]
x_start = row*chunk_size[0]
x_stop = (row+1)*chunk_size[0]
y_start = col*chunk_size[1]
y_stop = (col+1)*chunk_size[1]
z_start = args.z_start
z_stop = args.z_stop
info = CloudVolume.create_new_info(
num_channels = 1,
layer_type = 'image',
data_type = 'uint8',
encoding = 'raw',
resolution = [args.resolution[0]*2**mip,
args.resolution[1]*2**mip,
args.resolution[2]]
voxel_offset = [x_start,
y_start,
z_start],
chunk_size = chunk_size,
volume_size = [chunk_size[0],
chunk_size[1],
z_stop - z_start]
)
x_range = range(x_start, x_stop)
y_range = range(y_start, y_stop)
z_range = range(z_start, z_stop)
url_base = args.url_base
ex_url = '{}/{}/{}/{}/{}.jpg'.format(url_base, mip, z_start, row, col)
vol = CloudVolume(args.dst_path, info=info)
vol.provenance.description = 'Cutout from CATMAID'
vol.provenance.processing.append({
'method': {
'task': 'ingest',
'src_path': url_base,
'dst_path': args.dst_path,
'row': row,
'col': col,
'z_range': [z_start, z_stop],
'chunk_size': chunk_size.tolist()
'mip': args.mip,
},
'by': args.owner,
'date': strftime('%Y-%m-%d%H:%M %Z'),
})
def crtinfo(layer_type, dtype, encoding):
return CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=dtype,
encoding=encoding,
resolution=resolution,
voxel_offset=voxel_offset,
volume_size=volume_size,
chunk_size=[56, 56, 56],
)
def create_new_info(
self,
num_channels,
layer_type,
data_type,
resolution,
volume_size,
voxel_offset=(0, 0, 0),
encoding="raw",
chunk_size=(64, 64, 64),
mesh=None,
skeletons=None,
compressed_segmentation_block_size=(8, 8, 8),
max_mip=0,
factor=(2, 2, 1),
):
"""
Creates the info JSON necessary for a new cloudvolume resource.
Args:
Required:
num_channels: (int) 1 for grayscale, 3 for RGB
layer_type: (str) typically "image" or "segmentation"
data_type: (str) e.g. "uint8", "uint16", "uint32", "float32"
resolution: int (x,y,z), x,y,z voxel dimensions in nanometers
volume_size: int (x,y,z), extent of dataset in cartesian space from voxel_offset
Optional:
voxel_offset: int (x,y,z), beginning of dataset in positive cartesian space
encoding: (str) "raw" for binaries like numpy arrays, "jpeg"
mesh: (str) name of mesh directory, typically "mesh"
skeletons: (str) name of skeletons directory, typically "skeletons"
chunk_size: int (x,y,z), dimensions of each downloadable 3D image chunk in voxels
compressed_segmentation_block_size: (x,y,z) dimensions of each compressed sub-block
(only used when encoding is 'compressed_segmentation')
max_mip: (int), the maximum mip level id.
factor: (tuple), the downsampling factor for each mip level
Returns: dict representing a single mip level that's JSON encodable
"""
return CloudVolume.create_new_info(
num_channels,
layer_type,
data_type,
encoding,
resolution,
voxel_offset,
volume_size,
mesh,
skeletons,
chunk_size,
compressed_segmentation_block_size,
max_mip,
factor,
)
def create_image_layer(destination, num_resolutions):
info = CloudVolume.create_new_info(num_channels=1,
layer_type='image',
data_type='uint8',
encoding='raw',
resolution=[1, 1, 1],
voxel_offset=[0, 0, 0],
chunk_size=[5, 5, 5],
volume_size=[10, 10, 10])
vol = CloudVolume(destination, compress=False, info=info, parallel=True)
print(vol.info)
vol.commit_info()
return vol
def create_skeleton_layer(s3_bucket, skel_res, img_dims, num_res=7):
"""Creates segmentation layer for skeletons
Arguments:
s3_bucket {str} -- path to precomputed skeleton destination
skel_res {list} -- x,y,z dimensions of highest res voxel size (nm)
img_dims {list} -- x,y,z voxel dimensions of tiff images
Keyword Arguments:
num_res {int} -- number of image resolutions to be downsampled
Returns:
vol {cloudvolume.CloudVolume} -- CloudVolume to upload skeletons to
"""
# create cloudvolume info
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="segmentation",
data_type="uint64", # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
# Voxel scaling, units are in nanometers
resolution=skel_res,
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=[int(i / 4) for i in img_dims],
# chunk_size=[128, 128, 64], # units are voxels
volume_size=[i * 2 ** (num_res - 1) for i in img_dims], # units are voxels
skeletons="skeletons",
)
skel_info = {
"@type": "neuroglancer_skeletons",
"transform": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0],
"vertex_attributes": [
{"id": "radius", "data_type": "float32", "num_components": 1},
{"id": "vertex_types", "data_type": "float32", "num_components": 1},
{"id": "vertex_color", "data_type": "float32", "num_components": 4},
],
}
# get cloudvolume info
vol = CloudVolume(s3_bucket, info=info, parallel=True)
[vol.add_scale((2 ** i, 2 ** i, 2 ** i)) for i in range(num_res)] # num_res - 1
vol.commit_info()
# upload skeleton info to /skeletons/ dir
with storage.SimpleStorage(vol.cloudpath) as stor:
stor.put_json(str(Path("skeletons") / "info"), skel_info)
return vol
def create_cloud_volume(
precomputed_path,
img_size,
voxel_size,
num_mips,
chunk_size,
parallel=False,
layer_type="image",
dtype="uint16",
):
"""Create Neuroglancer precomputed volume S3
Args:
precomputed_path (str): S3 Path to location where precomputed layer will be stored
img_size (list of int): Size of the image (in 3D) to be uploaded
voxel_size ([type]): Voxel size in nanometers
num_mips (int, optional): Number of downsampling levels in X and Y. Defaults to 6.
chunk_size (list, optional): Size of each chunk stored on S3. Defaults to [1024, 1024, 1].
parallel (bool, optional): Whether or not the returned CloudVlue object will use parallel threads. Defaults to False.
layer_type (str, optional): Neuroglancer type of layer. Can be image or segmentation. Defaults to "image".
dtype (str, optional): Datatype of precomputed volume. Defaults to "uint16".
Returns:
cloudvolume.CloudVolume: CloudVolume object associated with this precomputed volume
"""
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=layer_type,
data_type=dtype, # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=voxel_size, # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=chunk_size, # units are voxels
volume_size=img_size, # e.g. a cubic millimeter dataset
)
vol = CloudVolume(precomputed_path, info=info, parallel=parallel)
[vol.add_scale((2 ** i, 2 ** i, 1), chunk_size=chunk_size) for i in range(num_mips)]
vol.commit_info()
return vol
def create_precomputed_volume(s3_resource, **kwargs):
"""Use CloudVolume to create the precomputed info file"""
vol_size = [e - o for e, o in zip(kwargs["extent"], kwargs["offset"])]
max_mip = (kwargs["num_hierarchy_levels"] -
1 if kwargs["downsample_status"] == "DOWNSAMPLED" else 0)
factor = (2, 2,
1) if kwargs["hierarchy_method"] == "anisotropic" else (2, 2, 2)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=kwargs["layer_type"],
data_type=kwargs["dtype"], # Channel images might be 'uint8'
encoding=kwargs[
"encoding"], # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=kwargs["scale"], # Voxel scaling, units are in nanometers
voxel_offset=kwargs[
"offset"], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=kwargs["chunk_size"], # units are voxels
volume_size=vol_size, # units are voxels
# undocumented param that creates the info w/ that many scales
max_mip=max_mip,
factor=factor,
)
# Don't use cloudvolume to submit the JSON, just use boto3 directly
layer_path = kwargs["layer_path"]
infokey = f"{layer_path}/info"
resp = ngprecomputed.save_obj(
s3_resource,
kwargs["dest_bucket"],
infokey,
json.dumps(info),
storage_class="STANDARD",
cache_control="no-cache",
content_type="application/json",
public=kwargs["public"],
owner_id=kwargs["owner_id"],
)
assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
def get_dst_cloudvolume(spec, dst_path, res, parallel=1, data_type="float32"):
"""Create CloudVolume where result will be written
Args:
spec (dict)
dst_path (str)
res (Vec): MIP0
parallel (int)
Returns:
CloudVolume
"""
mip = spec["dst_mip"]
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="image",
data_type=data_type,
encoding="raw",
resolution=res,
voxel_offset=[0, 0, 0],
chunk_size=[spec["x_size"], spec["y_size"], 1],
volume_size=[
spec["x_size"] * 2**mip,
spec["y_size"] * 2**mip,
2 * len(spec["pairs"]),
],
)
dst = CloudVolume(dst_path,
mip=0,
info=info,
cdn_cache=False,
parallel=parallel)
for m in range(1, mip + 1):
factor = Vec(2**m, 2**m, 1)
dst.add_scale(factor=factor,
chunk_size=[spec["x_size"], spec["y_size"], 1])
dst.commit_info()
dst = CloudVolume(dst_path, mip=mip, cdn_cache=False, parallel=parallel)
return dst
def make_test_dataset(path):
mip = 0
volume_size = Vec(1, 1, 6)
chunk_size = Vec(1, 1, 1)
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="image",
data_type="uint8",
encoding="raw",
resolution=[1, 1, 1],
voxel_offset=[0, 0, 0],
chunk_size=chunk_size,
volume_size=volume_size,
)
I = MiplessCloudVolume(path=path, info=info, overwrite=True)
# Make image with
img = np.zeros(volume_size, dtype=np.uint8)
img[:, :, 1] = 1
img[:, :, 2] = 1
img[:, :, 4] = 1
I[mip][:, :, :] = img
def make_info_file():
info = CloudVolume.create_new_info(
num_channels = 1,
layer_type = 'segmentation', # 'image' or 'segmentation'
data_type = 'uint32', # can pick any popular uint
encoding = 'raw', # other options: 'jpeg', 'compressed_segmentation' (req. uint32 or uint64)
resolution = [ 25000, 25000, 25000 ], # X,Y,Z values in nanometers, 40 microns in each dim
voxel_offset = [ 0, 0, 0 ], # values X,Y,Z values in voxels
chunk_size = [ 1024, 1024, 1 ], # rechunk of image X,Y,Z in voxels
volume_size = [320, 528, 456], # X,Y,Z size in voxels
)
# If you're using amazon or the local file system, you can replace 'gs' with 's3' or 'file'
vol = CloudVolume('file:///home/ahoag/ngdemo/demo_bucket/atlas/allenatlas_2017', info=info)
vol.provenance.description = "Segmentation volume for the 3D labeled allen atlas"
vol.provenance.owners = ['*****@*****.**'] # list of contact email addresses
vol.commit_info() # generates gs://bucket/dataset/layer/info json file
vol.commit_provenance() # generates gs://bucket/dataset/layer/provenance json file
print("Created CloudVolume info file: ",vol.info_cloudpath)
return vol
def init_volume(self, path):
info = CloudVolume.create_new_info(
num_channels=self.volume.shape[2]
if len(self.volume.shape) > 2 else 1,
layer_type=self.layer_type,
data_type=self.
data_type, # str(self.volume.dtype), # Channel images might be 'uint8'
encoding=
'raw', # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=self.scales, # Voxel scaling, units are in nanometers
voxel_offset=self.offset, # x,y,z offset in voxels from the origin
chunk_size=self.chunk_size, # units are voxels
volume_size=self.volume.
shape[:3], # e.g. a cubic millimeter dataset
)
self.precomputed_vol = CloudVolume(f'file://{path}',
mip=0,
info=info,
compress=True,
progress=False)
self.precomputed_vol.commit_info()
def test_background_color():
info = CloudVolume.create_new_info(
num_channels=1,
layer_type='image',
data_type='uint8',
encoding='raw',
resolution=[1, 1, 1],
voxel_offset=[0, 0, 0],
volume_size=[128, 128, 1],
mesh='mesh',
chunk_size=[64, 64, 1],
)
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume', mip=0, info=info)
vol.commit_info()
vol.cache.flush()
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume',
mip=0,
background_color=1,
fill_missing=True)
assert np.count_nonzero(vol[:] - 1) == 0
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume',
mip=0,
background_color=1,
fill_missing=True,
bounded=False)
assert np.count_nonzero(vol[0:129, 0:129, 0:1] - 1) == 0
vol = CloudVolume('file:///tmp/cloudvolume/empty_volume',
mip=0,
background_color=1,
fill_missing=True,
bounded=False,
parallel=2)
assert np.count_nonzero(vol[0:129, 0:129, 0:1] - 1) == 0
vol.cache.flush()
delete_layer('/tmp/cloudvolume/empty_volume')
def local_to_cloud(data,
cloud_path,
layer_type=None,
resolution=None,
scale=0):
'''currently support
layer_type: 'image' or 'segmentation'
resolution: tuple of 3 '''
data = np.moveaxis(data, 0, 2)
if not os.path.exists(cloud_path):
os.makedirs(cloud_path)
info = CloudVolume.create_new_info(
1,
layer_type=layer_type,
data_type=str(data.dtype),
encoding='raw',
resolution=list(resolution),
voxel_offset=(0, 0, 0),
volume_size=data.shape,
)
info = build_pyramid_info(info, scale)
if layer_type == 'segmentation':
info['mesh'] = 'mesh'
with open(os.path.join(cloud_path, 'info'), 'w') as f:
json.dump(info, f)
for i in range(0, scale):
vol = CloudVolume('file://' + cloud_path, mip=i,
compress='') # Basic Example
if i > 0:
data = zoom(data, 0.5)
x, y, z = vol.volume_size
data = data[0:x, 0:y, 0:z]
print(vol.volume_size, data.shape)
vol[:, :, :] = data
def create_image_layer(s3_bucket, tif_dimensions, voxel_size, num_resolutions):
"""Creates segmentation layer for skeletons
Arguments:
s3_bucket {str} -- path to SWC file
voxel_size {list} -- 3 floats for voxel size in nm
num_resolutions {int} -- number of resolutions for the image
Returns:
vols {list} -- List of num_resolutions CloudVolume objects, starting from lowest resolution
"""
# create cloudvolume info
info = CloudVolume.create_new_info(
num_channels=1,
layer_type="image",
data_type="uint16", # Channel images might be 'uint8'
encoding="raw", # raw, jpeg, compressed_segmentation, fpzip, kempressed
resolution=voxel_size, # Voxel scaling, units are in nanometers
voxel_offset=[0, 0, 0], # x,y,z offset in voxels from the origin
# Pick a convenient size for your underlying chunk representation
# Powers of two are recommended, doesn't need to cover image exactly
chunk_size=[int(d / 4) for d in tif_dimensions], # units are voxels
# USING MAXIMUM VOLUME size
volume_size=[i * 2**(num_resolutions - 1) for i in tif_dimensions],
)
# get cloudvolume info
vol = CloudVolume(s3_bucket, info=info, parallel=False) # compress = False
# scales resolution up, volume size down
[vol.add_scale((2**i, 2**i, 2**i))
for i in range(num_resolutions)] # ignore chunk size
vol.commit_info()
vols = [
CloudVolume(s3_bucket, mip=i,
parallel=False) # parallel False, compress
for i in range(num_resolutions - 1, -1, -1)
]
return vols
def test_redirects():
info = CloudVolume.create_new_info(
num_channels=1, # Increase this number when we add more tests for RGB
layer_type='image',
data_type='uint8',
encoding='raw',
resolution=[1, 1, 1],
voxel_offset=[0, 0, 0],
volume_size=[128, 128, 64],
mesh='mesh',
chunk_size=[64, 64, 64],
)
vol = CloudVolume('file:///tmp/cloudvolume/redirects_0', mip=0, info=info)
vol.commit_info()
vol.refresh_info()
vol.info['redirect'] = 'file:///tmp/cloudvolume/redirects_0'
vol.commit_info()
vol.refresh_info()
del vol.info['redirect']
for i in range(0, 10):
info['redirect'] = 'file:///tmp/cloudvolume/redirects_' + str(i + 1)
vol = CloudVolume('file:///tmp/cloudvolume/redirects_' + str(i),
mip=0,
info=info)
vol.commit_info()
else:
del vol.info['redirect']
vol.commit_info()
vol = CloudVolume('file:///tmp/cloudvolume/redirects_0', mip=0)
assert vol.cloudpath == 'file:///tmp/cloudvolume/redirects_9'
info['redirect'] = 'file:///tmp/cloudvolume/redirects_10'
vol = CloudVolume('file:///tmp/cloudvolume/redirects_9', mip=0, info=info)
vol.commit_info()
try:
CloudVolume('file:///tmp/cloudvolume/redirects_0', mip=0)
assert False
except exceptions.TooManyRedirects:
pass
vol = CloudVolume('file:///tmp/cloudvolume/redirects_9', max_redirects=0)
del vol.info['redirect']
vol.commit_info()
vol = CloudVolume('file:///tmp/cloudvolume/redirects_5', max_redirects=0)
vol.info['redirect'] = 'file:///tmp/cloudvolume/redirects_1'
vol.commit_info()
try:
vol = CloudVolume('file:///tmp/cloudvolume/redirects_5')
assert False
except exceptions.CyclicRedirect:
pass
vol.info['redirect'] = 'file:///tmp/cloudvolume/redirects_6'
vol.commit_info()
vol = CloudVolume('file:///tmp/cloudvolume/redirects_1')
try:
vol[:, :, :] = 1
assert False
except exceptions.ReadOnlyException:
pass
for i in range(0, 10):
delete_layer('/tmp/cloudvolume/redirects_' + str(i))
def ingest(args):
"""
Ingest an HDF file to a CloudVolume bucket
"""
if args.local_hdf_path:
hdf_file = h5py.File(args.local_hdf_path, "r")
else:
with Storage(args.cloud_src_path) as storage:
hdf_file = h5py.File(storage.get_file(args.cloud_hdf_filename),
"r")
cur_hdf_group = hdf_file
for group_name in args.hdf_keys_to_dataset:
cur_hdf_group = cur_hdf_group[group_name]
hdf_dataset = cur_hdf_group
if args.zyx:
dataset_shape = np.array(
[hdf_dataset.shape[2], hdf_dataset.shape[1], hdf_dataset.shape[0]])
else:
dataset_shape = np.array([*hdf_dataset.shape])
if args.layer_type == "image":
data_type = "uint8"
else:
data_type = "uint64"
voxel_offset = args.voxel_offset
info = CloudVolume.create_new_info(
num_channels=1,
layer_type=args.layer_type,
data_type=data_type,
encoding="raw",
resolution=args.resolution,
voxel_offset=voxel_offset,
chunk_size=args.chunk_size,
volume_size=dataset_shape,
)
provenance = {
"description": args.provenance_description,
"owners": [args.owner]
}
vol = CloudVolume(args.dst_path, info=info, provenance=provenance)
vol.commit_info()
vol.commit_provenance()
all_files = set()
for x in np.arange(voxel_offset[0], voxel_offset[0] + dataset_shape[0],
args.chunk_size[0]):
for y in np.arange(voxel_offset[1], voxel_offset[1] + dataset_shape[1],
args.chunk_size[1]):
for z in np.arange(voxel_offset[2],
voxel_offset[2] + dataset_shape[2],
args.chunk_size[2]):
all_files.add(tuple((x, y, z)))
progress_dir = mkdir(
"progress/") # unlike os.mkdir doesn't crash on prexisting
done_files = set()
for done_file in os.listdir(progress_dir):
done_files.add(tuple(done_file.split(",")))
to_upload = all_files.difference(done_files)
for chunk_start_tuple in to_upload:
chunk_start = np.array(list(chunk_start_tuple))
end_of_dataset = np.array(voxel_offset) + dataset_shape
chunk_end = chunk_start + np.array(args.chunk_size)
chunk_end = Vec(*chunk_end)
chunk_end = Vec.clamp(chunk_end, Vec(0, 0, 0), end_of_dataset)
chunk_hdf_start = chunk_start - voxel_offset
chunk_hdf_end = chunk_end - voxel_offset
if args.zyx:
chunk = hdf_dataset[chunk_hdf_start[2]:chunk_hdf_end[2],
chunk_hdf_start[1]:chunk_hdf_end[1],
chunk_hdf_start[0]:chunk_hdf_end[0], ]
chunk = chunk.T
else:
chunk = hdf_dataset[chunk_hdf_start[0]:chunk_hdf_end[0],
chunk_hdf_start[1]:chunk_hdf_end[1],
chunk_hdf_start[2]:chunk_hdf_end[2], ]
print("Processing ", chunk_start_tuple)
array = np.array(chunk, dtype=np.dtype(data_type), order="F")
vol[chunk_start[0]:chunk_end[0], chunk_start[1]:chunk_end[1],
chunk_start[2]:chunk_end[2], ] = array
touch(os.path.join(progress_dir, str(chunk_start_tuple)))
import os
import shutil
from cloudvolume import CloudVolume, chunks, Storage, Skeleton
from cloudvolume.storage import SimpleStorage
from cloudvolume.lib import mkdir, Bbox, Vec, jsonify
from cloudvolume.datasource.precomputed.sharding import ShardingSpecification
from cloudvolume.exceptions import SkeletonDecodeError, SkeletonAttributeMixingError
info = CloudVolume.create_new_info(
num_channels=1, # Increase this number when we add more tests for RGB
layer_type='segmentation',
data_type='uint16',
encoding='raw',
resolution=[1, 1, 1],
voxel_offset=(0, 0, 0),
skeletons=True,
volume_size=(100, 100, 100),
chunk_size=(64, 64, 64),
)
skel_info = {
"@type":
"neuroglancer_skeletons",
"transform": [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0],
"vertex_attributes": [{
"id": "radius",
"data_type": "float32",
"num_components": 1
}],
