def test_caching():
vol = CloudVolume('file:///tmp/cloudvolume/test-skeletons',
info=info,
cache=True)
vol.cache.flush()
skel = Skeleton(
[
(0, 0, 0),
(1, 0, 0),
(2, 0, 0),
(0, 1, 0),
(0, 2, 0),
(0, 3, 0),
],
edges=[(0, 1), (1, 2), (3, 4), (4, 5), (3, 5)],
segid=666,
)
vol.skeleton.upload(skel)
assert vol.cache.list_skeletons() == ['666.gz']
skel.id = 1
with open(os.path.join(vol.cache.path, 'skeletons/1'), 'wb') as f:
f.write(skel.to_precomputed())
cached_skel = vol.skeleton.get(1)
assert cached_skel == skel
vol.cache.flush()
def execute(self):
corgie_logger.info(
f"Generate new skeleton vertices task for id {self.skeleton_id_str}"
)
skeleton = get_skeleton(self.src_path, self.skeleton_id_str)
if self.vertex_sort:
vertex_sort = skeleton.vertices[:, 2].argsort()
else:
vertex_sort = np.arange(0, len(skeleton.vertices))
number_vertices = len(skeleton.vertices)
index_points = list(range(0, number_vertices, self.task_vertex_size))
cf = CloudFiles(f"{self.dst_path}")
array_filenames = []
for i in range(len(index_points)):
start_index = index_points[i]
if i + 1 == len(index_points):
end_index = number_vertices
else:
end_index = index_points[i + 1]
array_filenames.append(
f"intermediary_arrays/{self.skeleton_id_str}:{start_index}-{end_index}"
)
array_files = cf.get(array_filenames)
# Dict to make sure arrays are concatenated in correct order
array_dict = {}
for array_file in array_files:
array_dict[array_file["path"]] = pickle.loads(
array_file["content"])
array_arrays = []
for array_filename in array_filenames:
array_arrays.append(array_dict[array_filename])
array_arrays = np.concatenate(array_arrays)
# Restore the correct order of the vertices
restore_sort = vertex_sort.argsort()
new_vertices = array_arrays[restore_sort]
new_skeleton = Skeleton(
vertices=new_vertices,
edges=skeleton.edges,
radii=skeleton.radius,
vertex_types=skeleton.vertex_types,
space=skeleton.space,
transform=skeleton.transform,
)
cf.put(
path=self.skeleton_id_str,
content=new_skeleton.to_precomputed(),
compress="gzip",
)
def uploadshardedskeletons(skelsource, skelseglist, skelnamelist, path):
"""Upload sharded skeletons to a local server.
Parameters
----------
skelsource : List containing cloud volume skeletons
skelseglist : List containing the segids(skid)
skelnamelist : List containing the names of skeletons
path : path to the local data server
Returns
-------
cv : cloudvolume class object
"""
info = {
"@type":
"neuroglancer_skeletons",
"transform":
skelsource[0].transform.flatten(),
"vertex_attributes": [{
"id": "radius",
"data_type": "float32",
"num_components": 1
}],
"scales":
"um"
}
path = 'file://' + path + '/precomputed'
cv = CloudVolume(path, info=info)
# prepare for info file
cv.skeleton.meta.info['@type'] = 'neuroglancer_skeletons'
cv.skeleton.meta.info['transform'] = skelsource[0].transform.flatten()
cv.skeleton.meta.info['vertex_attributes'] = [{
'id': 'radius',
'data_type': 'float32',
'num_components': 1
}]
# prepare sharding info
spec = ShardingSpecification(
'neuroglancer_uint64_sharded_v1',
preshift_bits=9,
hash='murmurhash3_x86_128',
minishard_bits=6,
shard_bits=15,
minishard_index_encoding='raw',
data_encoding='raw',
)
cv.skeleton.meta.info['sharding'] = spec.to_dict()
cv.skeleton.meta.info['segment_properties'] = 'seg_props'
cv.skeleton.meta.commit_info()
precomputedskels = {}
for skelidx in range(len(skelsource)):
skelid = int(skelsource[skelidx].id)
skel = Skeleton(skelsource[skelidx].vertices,
edges=skelsource[skelidx].edges,
segid=skelid,
extra_attributes=[{
"id": "radius",
"data_type": "float32",
"num_components": 1,
}]).physical_space()
precomputedskels[skelid] = skel.to_precomputed()
shardfiles = spec.synthesize_shards(precomputedskels)
shardedfilepath = os.path.join(cv.basepath, os.path.basename(path),
cv.skeleton.meta.skeleton_path)
for fname in shardfiles.keys():
with open(shardedfilepath + '/' + fname, 'wb') as f:
f.write(shardfiles[fname])
segfilepath = os.path.join(cv.basepath, os.path.basename(path),
cv.skeleton.meta.skeleton_path, 'seg_props')
if not os.path.exists(segfilepath):
os.makedirs(segfilepath)
print('creating:', segfilepath)
allsegproplist = []
for segid in skelseglist:
segpropdict = {}
segpropdict['id'] = segid
segpropdict['type'] = 'label'
segpropdict['values'] = skelnamelist
allsegproplist.append(segpropdict)
seginfo = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": skelseglist,
"properties": allsegproplist
}
}
segfile = os.path.join(segfilepath, 'info')
with open(segfile, 'w') as segfile:
json.dump(seginfo, segfile)
return cv
def uploadskeletons(skelsource, skelseglist, skelnamelist, path):
"""Upload skeleton (of cloudvolume class) to a local server.
Parameters
----------
skelsource : List containing cloud volume skeletons
skelseglist : List containing the segids(skid)
skelnamelist : List containing the names of skeletons
path : path to the local data server
Returns
-------
cv : cloudvolume class object
"""
info = {
"@type":
"neuroglancer_skeletons",
"transform":
skelsource[0].transform.flatten(),
"vertex_attributes": [{
"id": "radius",
"data_type": "float32",
"num_components": 1
}],
"scales":
"um"
}
path = 'file://' + path + '/precomputed'
cv = CloudVolume(path, info=info)
# prepare for info file
cv.skeleton.meta.info['@type'] = 'neuroglancer_skeletons'
cv.skeleton.meta.info['transform'] = skelsource[0].transform.flatten()
cv.skeleton.meta.info['vertex_attributes'] = [{
'id': 'radius',
'data_type': 'float32',
'num_components': 1
}]
del cv.skeleton.meta.info['sharding']
del cv.skeleton.meta.info['spatial_index']
cv.skeleton.meta.info['segment_properties'] = 'seg_props'
cv.skeleton.meta.commit_info()
files = [
os.path.join(cv.skeleton.meta.skeleton_path, str(skel.id))
for skel in skelsource
]
for fileidx in range(len(files)):
fullfilepath = files[fileidx]
fullfilepath = os.path.join(cv.basepath, os.path.basename(path),
fullfilepath)
uploadskel = Skeleton(vertices=skelsource[fileidx].vertices,
edges=skelsource[fileidx].edges)
print(fullfilepath)
with open(fullfilepath, 'wb') as f:
f.write(uploadskel.to_precomputed())
segfilepath = os.path.join(cv.basepath, os.path.basename(path),
cv.skeleton.meta.skeleton_path, 'seg_props')
if not os.path.exists(segfilepath):
os.makedirs(segfilepath)
print('creating:', segfilepath)
allsegproplist = []
for segid in skelseglist:
segpropdict = {}
segpropdict['id'] = segid
segpropdict['type'] = 'label'
segpropdict['values'] = skelnamelist
allsegproplist.append(segpropdict)
seginfo = {
"@type": "neuroglancer_segment_properties",
"inline": {
"ids": skelseglist,
"properties": allsegproplist
}
}
segfile = os.path.join(segfilepath, 'info')
with open(segfile, 'w') as segfile:
json.dump(seginfo, segfile)
return cv
