def get_synapses_dvid(dvid_server, dvid_uuid, dvid_annot, dvid_roi):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid,
'flyem_syn_eval','flyem_syn_eval')
synapses_json = dvid_node.custom_request(
'%s/roi/%s' % (dvid_annot, dvid_roi),
None, ConnectionMethod.GET)
return synapses_json.decode()
def writer(element_pairs):
from libdvid import DVIDNodeService, Vertex, Edge
# write graph information
node_service = DVIDNodeService(str(server), str(uuid))
if element_pairs is None:
return
vertices = []
edges = []
for element_pair in element_pairs:
edge, weight = element_pair
v1, v2 = edge
if v2 == -1:
vertices.append(Vertex(v1, weight))
else:
edges.append(Edge(v1, v2, weight))
if len(vertices) > 0:
node_service.update_vertices(str(graph_name), vertices)
if len(edges) > 0:
node_service.update_edges(str(graph_name), edges)
return []
def _init_version(self, uuid):
# create connection to repo
self.nodeconn = DVIDNodeService(str(self.server), str(uuid))
# fetch all repo information
status, data, errmsg = self.rawconn.make_request("/repo/" + uuid + "/info", ConnectionMethod.GET)
self.repoinfo = json.loads(data)
self.allinstances = {}
# load all versions in repo
self.alluuids = set()
dag = self.repoinfo["DAG"]["Nodes"]
for uuidt, nodedata in dag.items():
self.alluuids.add(str(nodedata["UUID"]))
for instancename, val in self.repoinfo["DataInstances"].items():
# name is not necessarily unique to a repo
self.allinstances[(str(instancename), str(val["Base"]["DataUUID"]))] = str(val["Base"]["TypeName"])
# datainstances that should be hidden (array of names)
try:
self.hidden_instances = set(self.nodeconn.get_json(self.MetaLocation, self.RestrictionName))
except:
self.hidden_instances = set()
nodeids = {}
# check if locked note
dag = self.repoinfo["DAG"]["Nodes"]
for tuuid, nodedata in dag.items():
nodeids[str(nodedata["VersionID"])] = nodedata
if tuuid.startswith(uuid):
self.uuid = str(tuuid)
self.current_node = nodedata
self.locked = nodedata["Locked"]
# load all ancestors
ancestors = set()
# commit history uuid, commit note in order from oldest to newest
self.loghistory = []
currnode = self.current_node
while True:
ancestors.add(str(currnode["UUID"]))
self.loghistory.append((str(currnode["UUID"]), currnode["Note"]))
if len(currnode["Parents"]) > 0:
currnode = nodeids[str(currnode["Parents"][0])]
else:
break
self.loghistory.reverse()
# load all instances
self.activeinstances = {}
if not self.locked:
tempuuid = self.loghistory[-1][0]
self.loghistory[-1] = (tempuuid, "(open node)")
for instancename, val in self.repoinfo["DataInstances"].items():
if str(val["Base"]["RepoUUID"]) in ancestors:
self.activeinstances[str(instancename)] = str(val["Base"]["TypeName"])
def get_labels2():
# fetch second label volume
node_service2 = DVIDNodeService(str(server2), str(uuid2))
# retrieve data from roi start position
return node_service2.get_labels3D(
str(label_name2), (size1, size2, size3), (subvolume.roi[0], subvolume.roi[1], subvolume.roi[2])
)
def setUpClass(cls):
"""
Override. Called by nosetests.
"""
# Choose names
cls.dvid_repo = "datasetA"
cls.data_name = "random_data"
cls.volume_location = "/repos/{dvid_repo}/volumes/{data_name}".format( **cls.__dict__ )
cls.data_uuid = get_testrepo_root_uuid()
cls.node_location = "/repos/{dvid_repo}/nodes/{data_uuid}".format( **cls.__dict__ )
# Generate some test data
#data = numpy.random.randint(0, 255, (128, 256, 512, 1)).astype( numpy.uint8 )
data = numpy.zeros((128, 256, 512, 1), dtype=numpy.uint8)
data.flat[:] = numpy.arange( numpy.prod((128, 256, 512, 1)) )
cls.original_data = data
cls.voxels_metadata = VoxelsMetadata.create_default_metadata(data.shape, data.dtype, "zyxc", 1.0, "")
# Write it to a new data instance
node_service = DVIDNodeService(TEST_DVID_SERVER, cls.data_uuid)
node_service.create_grayscale8(cls.data_name)
node_service.put_gray3D( cls.data_name, data[...,0], (0,0,0) )
def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
num_at_once = 100000
n_tot = len(tbars_json)
for ii in range(0, n_tot, num_at_once):
jj = np.minimum(ii + num_at_once, n_tot)
data = json.dumps(tbars_json[ii:jj])
oo = dvid_node.custom_request('%s/elements' % dvid_annot,
data.encode('utf-8'),
ConnectionMethod.POST)
def retrieve_node_service(server, uuid, resource_server, resource_port, appname="sparkservices"):
"""Create a DVID node service object"""
server = str(server)
# refresh dvid server meta if localhost (since it is exclusive or points to global db)
"""
if server.startswith("http://127.0.0.1") or \
server.startswith("http://localhost") or \
server.startswith("127.0.0.1") or server.startswith("localhost"):
import os
if not os.path.exists("/tmp/reloaded.hack"):
addr = server + "/api/server/reload-metadata"
if not server.startswith("http://"):
addr = "http://" + addr
session = default_dvid_session()
session.post(addr)
open("/tmp/reloaded.hack", 'w').close()
"""
from libdvid import DVIDNodeService
import os
username = os.environ["USER"]
if resource_server != "":
node_service = DVIDNodeService(server, str(uuid), username, appname, str(resource_server), resource_port)
else:
node_service = DVIDNodeService(server, str(uuid), username, appname)
return node_service
def get_blocksize(dvid_server, uuid, dataname):
"""Gets block size for supplied data name.
Note:
Does not check for existence of body and whether
it is an array type. The block size is always
assumes to be isotropic.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
dataname (str): data instance
Returns:
(z,y,x) blocksize.
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
ns = DVIDNodeService(str(dvid_server), str(uuid))
info = ns.get_typeinfo(dataname)
x, y, z = info["Extended"]["BlockSize"] # DVID ordered x,y,z
return (z, y, x)
def get_blocksize(dvid_server, uuid, dataname):
"""Gets block size for supplied data name.
Note:
Does not check for existence of body and whether
it is an array type. The block size is always
assumes to be isotropic.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
dataname (str): data instance
Returns:
(z,y,x) blocksize.
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
ns = DVIDNodeService(str(dvid_server), str(uuid))
info = ns.get_typeinfo(dataname)
x,y,z = info["Extended"]["BlockSize"] # DVID ordered x,y,z
return (z,y,x)
class IoDVID(object):
def __init__(self, server_address, uuid, key):
assert WITH_DVID, "Need dvid"
self.ds = DVIDNodeService(server_address, uuid)
self.key = key
# get the shape the dvid way...
endpoint = "/" + self.key + "/info"
attributes = self.ds.custom_request(endpoint, "", ConnectionMethod.GET)
# TODO do we need to increase by 1 here ?
self._shape = tuple(mp + 1 for mp in attributes["MaxPoint"])
def read(self, bb):
offset = tuple(b.start for b in bb)
shape = tuple(b.stop - b.start for b in bb)
return self.ds.get_gray3D(self.key, shape, offset)
def write(self, out, out_bb):
raise NotImplementedError("Writing to DVID is not yet implemented!")
@property
def shape(self):
return self._shape
def close(self):
pass
def get_segmask():
node_service = DVIDNodeService(str(pdconf["dvid-server"]), str(pdconf["uuid"]))
# retrieve data from roi start position
return node_service.get_labels3D(
str(pdconf["segmentation-name"]),
(size1, size2, size3),
(subvolume.roi[0] - border, subvolume.roi[1] - border, subvolume.roi[2] - border),
)
def test_custom_request(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**", "test_custom_app")
response_body = node_service.custom_request("log", b"",
ConnectionMethod.GET)
# This shouldn't raise an exception
json_data = json.loads(response_body)
def create_new( hostname, uuid, data_name, voxels_metadata ):
node_service = DVIDNodeService(hostname, uuid)
typename = voxels_metadata.determine_dvid_typename()
if typename == "grayscale8":
node_service.create_grayscale8(data_name)
elif typename == "labels64":
node_service.create_labelblk("test_labels_3d")
else:
raise RuntimeError("Don't know how to create a data instance of type '{}'".format(typename))
def storeFeaturesForFrame(self, features, timeframe):
"""
Stores feature data
"""
assert(self.server_address is not None)
assert(self.uuid is not None)
node_service = DVIDNodeService(self.server_address, self.uuid)
node_service.create_keyvalue(self.keyvalue_store)
node_service.put(self.keyvalue_store, "frame-{}".format(timeframe), json.dumps(features))
def loadFeaturesForFrame(self, features, timeframe):
"""
loads feature data
"""
assert(self.server_address is not None)
assert(self.uuid is not None)
node_service = DVIDNodeService(self.server_address, self.uuid)
node_service.create_keyvalue(self.keyvalue_store)
return json.loads(node_service.get(self.keyvalue_store, "frame-{}".format(timeframe)))
def __init__(self, server_address, uuid, key):
assert WITH_DVID, "Need dvid"
self.ds = DVIDNodeService(server_address, uuid)
self.key = key
# get the shape the dvid way...
endpoint = "/" + self.key + "/info"
attributes = self.ds.custom_request(endpoint, "", ConnectionMethod.GET)
# TODO do we need to increase by 1 here ?
self._shape = tuple(mp + 1 for mp in attributes["MaxPoint"])
def getTimeRange(self, Resource, PathInResource):
"""
Count Label images to derive the total number of frames
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return tuple of (first frame, last frame)
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
return config["time_range"]
def get_labels():
# extract labels 64
node_service = DVIDNodeService(str(server), str(uuid))
# retrieve data from roi start position
return node_service.get_labels3D(
str(label_name),
(size1, size2, size3),
(subvolume.roi[0], subvolume.roi[1], subvolume.roi[2]),
roi=str(roiname),
)
def test_copy_roi(self):
expected_blocks = [(1,2,3),(2,3,4),(4,5,6)]
# We aren't really exercising transferring between servers here. Too bad.
src_info = (TEST_DVID_SERVER, self.uuid, 'src_roi')
dest_info = (TEST_DVID_SERVER, self.uuid, 'dest_roi')
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid, "unittest@test_roi_utils.py", "Test_roi_utils")
written_blocks = node_service.get_roi("src_roi")
assert set(expected_blocks) == set(written_blocks)
def getTimeRange(self, Resource, PathInResource):
"""
Count Label images to derive the total number of frames
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return tuple of (first frame, last frame)
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
return config["time_range"]
def execute(self):
from DVIDSparkServices.reconutils import SimpleGraph
from libdvid import DVIDNodeService
from pyspark import SparkContext
from pyspark import StorageLevel
if "chunk-size" in self.config_data["options"]:
self.chunksize = self.config_data["options"]["chunk-size"]
# grab ROI
distrois = self.sparkdvid_context.parallelize_roi(self.config_data["dvid-info"]["roi"], self.chunksize)
num_partitions = distrois.getNumPartitions()
# map ROI to label volume (1 pixel overlap)
label_chunks = self.sparkdvid_context.map_labels64(
distrois, self.config_data["dvid-info"]["label-name"], 1, self.config_data["dvid-info"]["roi"]
)
# map labels to graph data -- external program (eventually convert neuroproof metrics and graph to a python library) ?!
sg = SimpleGraph.SimpleGraph(self.config_data["options"])
# extract graph
graph_elements = label_chunks.flatMap(sg.build_graph)
# group data for vertices and edges
graph_elements_red = graph_elements.reduceByKey(lambda a, b: a + b)
# repartition by first vertex to better group edges together
graph_elements_red = graph_elements_red.partitionBy(num_partitions, lambda a: hash(a[0]))
graph_elements_red.persist(StorageLevel.MEMORY_ONLY) # ??
graph_vertices = graph_elements_red.filter(sg.is_vertex)
graph_edges = graph_elements_red.filter(sg.is_edge)
# create graph
node_service = DVIDNodeService(
str(self.config_data["dvid-info"]["dvid-server"]), str(self.config_data["dvid-info"]["uuid"])
)
node_service.create_graph(str(self.config_data["dvid-info"]["graph-name"]))
# dump graph -- should this be wrapped through utils or through sparkdvid ??
# will this result in too many request (should they be accumulated) ??
# currently looking at one partitioning at a time to try to group requests
self.sparkdvid_context.foreachPartition_graph_elements(
graph_vertices, self.config_data["dvid-info"]["graph-name"]
)
self.sparkdvid_context.foreachPartition_graph_elements(graph_edges, self.config_data["dvid-info"]["graph-name"])
if "debug" in self.config_data["options"] and self.config_data["options"]["debug"]:
num_elements = graph_elements.count()
print "DEBUG: ", num_elements
graph_elements_red.unpersist()
def get_labels():
# extract labels 64
node_service = DVIDNodeService(str(server), str(uuid))
# retrieve data from roi start position considering border
return node_service.get_labels3D(
str(label_name),
(size1, size2, size3),
(subvolume.roi[0] - border, subvolume.roi[1] - border, subvolume.roi[2] - border),
compress=True,
roi=str(roiname),
)
def put_labels():
node_service = DVIDNodeService(str(server), str(uuid))
# send data from roi start position
node_service.put_labels3D(
str(label_name),
seg,
(subvolume.roi.x1, subvolume.roi.y1, subvolume.roi.z1),
compress=True,
roi=str(roi_name),
mutate=mutate,
)
def get_gray():
node_service = DVIDNodeService(str(server), str(uuid))
return node_service.get_gray3D(
str(gray_name),
(size1, size2, size3),
(
subvolume.roi.x1 - subvolume.border,
subvolume.roi.y1 - subvolume.border,
subvolume.roi.z1 - subvolume.border,
),
)
def test_isdatainstance(self):
"""Tests is_datainstance function.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
ns = DVIDNodeService(dvidserver, uuid)
ns.create_labelblk("labels")
self.assertTrue(is_datainstance(dvidserver, uuid, "labels"))
self.assertFalse(is_datainstance(dvidserver, uuid, "labels2"))
def getImageShape(self, Resource, PathInResource):
"""
Derive Image Shape from label image.
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return list with image dimensions
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
self.shape = config["shape"]
return self.shape
def get_output_vol(index):
output_service = DVIDNodeService(
str(config['outputs'][index]['dvid']['server']),
str(config['outputs'][index]['dvid']['uuid']))
output_name = config['outputs'][index]['dvid']['segmentation-name']
output_bb_xyz = config['outputs'][index]['geometry']['bounding-box']
output_bb_zyx = np.array(output_bb_xyz)[:, ::-1]
output_shape = output_bb_zyx[1] - output_bb_zyx[0]
output_volume = output_service.get_labels3D(output_name, output_shape,
output_bb_zyx[0])
return output_volume
def copy_roi(src_info, dest_info):
src_service = DVIDNodeService(src_info.server, src_info.uuid)
dest_service = DVIDNodeService(dest_info.server, dest_info.uuid)
# If necessary, create the ROI on the destination server
try:
info = dest_service.get_typeinfo(dest_info.name)
except DVIDException:
dest_service.create_roi(dest_info.name)
roi_blocks = src_service.get_roi(src_info.name)
dest_service.post_roi(dest_info.name, roi_blocks)
def getImageShape(self, Resource, PathInResource):
"""
Derive Image Shape from label image.
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return list with image dimensions
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
self.shape = config["shape"]
return self.shape
def test_copy_roi(self):
expected_blocks = [(1, 2, 3), (2, 3, 4), (4, 5, 6)]
# We aren't really exercising transferring between servers here. Too bad.
src_info = (TEST_DVID_SERVER, self.uuid, 'src_roi')
dest_info = (TEST_DVID_SERVER, self.uuid, 'dest_roi')
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"unittest@test_roi_utils.py",
"Test_roi_utils")
written_blocks = node_service.get_roi("src_roi")
assert set(expected_blocks) == set(written_blocks)
def test_keyvalue(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_keyvalue("test_keyvalue")
node_service.put("test_keyvalue", "kkkk", "vvvv")
readback_value = node_service.get("test_keyvalue", "kkkk")
self.assertEqual(readback_value, "vvvv")
with self.assertRaises(RuntimeError):
node_service.put("test_keyvalue", "kkkk", 123) # 123 is not a buffer.
def _update_subvol_widget(self, node_uuid, dataname, typename):
"""
Update the subvolume widget with the min/max extents of the given node and dataname.
Note: The node and dataname do not necessarily have to match the currently
selected node and dataname.
This enables the right-click behavior, which can be used to
limit your data volume to the size of a different data volume.
"""
error_msg = None
try:
if typename == "roi":
node_service = DVIDNodeService(self._hostname, str(node_uuid))
roi_blocks_zyx = numpy.array(
node_service.get_roi(str(dataname)))
maxindex = tuple(DVID_BLOCK_WIDTH *
(1 + numpy.max(roi_blocks_zyx, axis=0)))
minindex = (0, 0, 0) # Rois are always 3D
axiskeys = "zyx"
# If the current selection is a dataset, then include a channel dimension
if self.get_selection().typename != "roi":
axiskeys = "zyxc"
minindex = minindex + (0, )
maxindex = maxindex + (
1,
) # FIXME: This assumes that the selected data has only 1 channel...
else:
# Query the server
raw_metadata = VoxelsAccessor.get_metadata(
self._hostname, node_uuid, dataname)
voxels_metadata = VoxelsMetadata(raw_metadata)
maxindex = voxels_metadata.shape
minindex = voxels_metadata.minindex
axiskeys = voxels_metadata.axiskeys
# If the current selection is a roi, then remove the channel dimension
if self.get_selection().typename == "roi":
axiskeys = "zyx"
minindex = minindex[:-1]
maxindex = maxindex[:-1]
except (DVIDException, ErrMsg) as ex:
error_msg = str(ex)
log_exception(logger)
else:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
if error_msg:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
QMessageBox.critical(self, "DVID Error", error_msg)
self._subvol_widget.initWithExtents("", (), (), ())
return
self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex,
maxindex)
def getImageDataAtTimeFrame(self, Resource, PathInResource, axes, timeframe):
"""
Loads image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return numpy array of image data at timeframe.
"""
node_service = DVIDNodeService(Resource, PathInResource)
if (self.shape == None):
self.getImageShape(Resource, PathInResource)
raw_frame = node_service.get_gray3D(self._getRawImageName(timeframe), tuple(self.shape), (0,0,0))
return raw_frame
def __init__(self, hostname, uuid, data_name,
query_args=None,
throttle=None,
retry_timeout=60.0,
retry_interval=1.0,
warning_interval=30.0,
_metadata=None,
_access_type="raw"):
"""
:param uuid: The node uuid
:param data_name: The name of the volume
:param hostname: The DVID server hostname e.g. 'localhost:8000'
:param throttle: Enable the DVID 'throttle' flag for all get/post requests
:param retry_timeout: Total time to spend repeating failed requests before giving up.
(Set to 0 to prevent retries.)
:param retry_interval: Time to wait before repeating a failed get/post.
:param warning_interval: If the retry period exceeds this interval (but hasn't
hit the retry_timeout yet), a warning is emitted.
:param _metadata: If provided, used as the metadata for the accessor. Otherwise, the server is queried to obtain this volume's metadata.
.. note:: When DVID is overloaded, it may indicate its busy status by returning a ``503``
(service unavailable) error in response to a get/post request. In that case,
the get/post methods below will automatically repeat the failed request until
the `retry_timeout` is reached.
"""
self.hostname = hostname
self.uuid = uuid
self.data_name = data_name
self._retry_timeout = retry_timeout
self._retry_interval = retry_interval
self._warning_interval = warning_interval
self._query_args = query_args or {}
self._access_type = _access_type
self._node_service = DVIDNodeService(hostname, uuid)
# Special case: throttle can be set explicity via the keyword or implicitly via the query_args.
# Make sure they are consistent.
if 'throttle' in self._query_args:
if self._query_args['throttle'] == 'on':
assert throttle is None or throttle is True
self._throttle = True
if self._query_args['throttle'] == 'off':
assert throttle is None or throttle is False
self._throttle = False
elif throttle is None:
self._throttle = False
else:
self._throttle = throttle
# Request this volume's metadata from DVID if necessary
self.voxels_metadata = _metadata or VoxelsAccessor.get_metadata( hostname, uuid, data_name )
def ingest_data(self):
"""
Ideally, this would be run within the __init__ function,
but operators should never raise non-fatal exceptions within Operator.__init__()
(See OperatorMetaClass.__call__)
This serves as an alternative init function, from which we are allowed to raise exceptions.
"""
try:
node_service = DVIDNodeService(self._hostname, self._uuid)
roi_blocks_zyx = numpy.array(node_service.get_roi(self._roi_name))
except DVIDException as ex:
if ex.status == http.client.NOT_FOUND:
raise OpDvidRoi.DatasetReadError("DVIDException: " +
ex.message)
raise
except ErrMsg as ex:
raise OpDvidRoi.DatasetReadError("ErrMsg: " + str(ex))
# Store a dense array of the block flags
# Each voxel of this dataset represents a dvid block, so this volume
# is 32x smaller than the full-res data in every dimension.
# TODO: For now, we only respect the high-side of the bounding box, and force the low-side to 0,0,0.
shape = tuple(1 + numpy.max(roi_blocks_zyx, axis=0))
slice_shape = (1, ) + shape[1:]
self._mem_file = h5py.File(str(id(self)),
driver='core',
backing_store=False,
mode='w')
dset = self._mem_file.create_dataset('data',
shape=shape,
dtype=numpy.uint8,
chunks=slice_shape,
compression='lzf')
# Allocate temporary array slice for writing into the dataset
z_slice = numpy.ndarray(slice_shape, numpy.uint8)
# Group the block indexes by z-coordinate
# FIXME: This might get slow for lots of blocks because of all the calls to this lambda.
# Consider upgrading to cytools.groupby instead
group_iter = groupby(roi_blocks_zyx,
lambda block_index: block_index[0])
# Set a pixel for each block index, one z-slice at a time
for z, block_indexes_iter in group_iter:
block_indexes = numpy.array(list(block_indexes_iter))
block_indexes = block_indexes[:, 1:] # drop z index
z_slice[:] = 0
z_slice[0][tuple(block_indexes.transpose())] = 1
dset[z, :, :] = z_slice[0, ...]
self._dset = dset
def getLabelImageForFrame(self, Resource, PathInResource, timeframe):
"""
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return numpy array of image data at timeframe.
"""
if (self.shape == None):
self.getImageShape(Resource, PathInResource)
node_service = DVIDNodeService(Resource, PathInResource)
seg_frame = np.array(node_service.get_labels3D(self._getSegmentationName(timeframe),
tuple(self.shape), (0,0,0))).astype(np.uint32)
return seg_frame
def test_DataInstance(self):
"""Tests DataInstance class.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
ns = DVIDNodeService(dvidserver, uuid)
ns.create_labelblk("labels")
ns.create_grayscale8("gray")
ns.create_keyvalue("kv")
try:
temp = DataInstance(dvidserver, uuid, "blah")
except ValueError:
# correct caught error
self.assertTrue(True)
labels = DataInstance(dvidserver, uuid, "labels")
gray = DataInstance(dvidserver, uuid, "gray")
kv = DataInstance(dvidserver, uuid, "kv")
self.assertTrue(labels.is_array())
self.assertTrue(labels.is_labels())
self.assertTrue(gray.is_array())
self.assertFalse(gray.is_labels())
self.assertFalse(kv.is_array())
self.assertFalse(kv.is_labels())
def set_syncs(dvid_server,
dvid_uuid,
dvid_annot,
dvid_segm=None,
dvid_labelsz=None):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, os.getenv('USER'),
'fpl')
dvid_conn = DVIDConnection(dvid_server, os.getenv('USER'), 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
if dvid_labelsz is not None:
# create labelsz if necessary
try:
dvid_node.custom_request('%s/info' % dvid_labelsz, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'labelsz',
'dataname': dvid_labelsz
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
# sync synapses to segmentation
if dvid_segm is not None:
syn_sync_json = json.dumps({'sync': dvid_segm})
dvid_node.custom_request('%s/sync' % dvid_annot,
syn_sync_json.encode('utf-8'),
ConnectionMethod.POST)
# sync labelsz to synapses
if dvid_labelsz is not None:
lsz_sync_json = json.dumps({'sync': dvid_annot})
dvid_node.custom_request('%s/sync' % dvid_labelsz,
lsz_sync_json.encode('utf-8'),
ConnectionMethod.POST)
def is_datainstance(dvid_server, uuid, name):
"""Checks if datainstance name exists.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
"""
try:
ns = DVIDNodeService(str(dvid_server), str(uuid))
info = ns.get_typeinfo(name)
except DVIDException:
# returns exception if it does not exist
return False
return True
def is_datainstance(dvid_server, uuid, name):
"""Checks if datainstance name exists.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
"""
try:
ns = DVIDNodeService(str(dvid_server), str(uuid))
info = ns.get_typeinfo(name)
except DVIDException:
# returns exception if it does not exist
return False
return True
def getImageDataAtTimeFrame(self, Resource, PathInResource, axes,
timeframe):
"""
Loads image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return numpy array of image data at timeframe.
"""
node_service = DVIDNodeService(Resource, PathInResource)
if (self.shape == None):
self.getImageShape(Resource, PathInResource)
raw_frame = node_service.get_gray3D(self._getRawImageName(timeframe),
tuple(self.shape), (0, 0, 0))
return raw_frame
def test_encode_labelarray_volume():
vol = np.random.randint(1000, 2000, size=(128, 128, 128), dtype=np.uint64)
vol[:64, :64, :64] = 0
vol[-64:, -64:, -64:] = 0
encoded = encode_labelarray_volume((512, 1024, 2048), vol, gzip_level=0)
encoded2 = encode_labelarray_volume((512, 1024, 2048), vol, gzip_level=9)
assert len(encoded) > len(encoded2)
inflated = DVIDNodeService.inflate_labelarray_blocks3D_from_raw(
encoded, (128, 128, 128), (512, 1024, 2048))
assert (inflated == vol).all()
inflated = DVIDNodeService.inflate_labelarray_blocks3D_from_raw(
encoded2, (128, 128, 128), (512, 1024, 2048))
assert (inflated == vol).all()
def evaluate_pr(tbars_pd, tbars_gt,
dist_thresh=27, conf_thresholds=None,
segm_dvid=None):
tbars_pd = tbars_pd.get_tbars()
tbars_gt = tbars_gt.get_tbars(1.)
if conf_thresholds is None:
cc = np.unique(tbars_pd.conf)
conf_thresholds = (cc[1:]+cc[:-1])/2
if segm_dvid is not None:
segm_dvid_node = DVIDNodeService(
segm_dvid[0], segm_dvid[1],
'flyem_syn_eval', 'flyem_syn_eval')
segm_dvid_segm = segm_dvid[2]
lbls_pd = get_labels(segm_dvid_node, segm_dvid_segm, tbars_pd)
lbls_gt = get_labels(segm_dvid_node, segm_dvid_segm, tbars_gt)
else:
lbls_pd = None
lbls_gt = None
result = obj_pr_curve(tbars_pd, tbars_gt,
dist_thresh, conf_thresholds,
lbls_pd, lbls_gt)
return result
def test_roi_partition(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_roi("test_roi_partition")
blocks = [(0,0,0),(1,1,1),(2,2,2),(3,3,3)]
node_service.post_roi("test_roi_partition", blocks)
substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32)])
self.assertEqual( packing_factor, float(len(blocks))/(len(substacks) * 4**3) )
blocks += [(4,0,0)]
node_service.post_roi("test_roi_partition", blocks)
substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32), SubstackXYZ(128,0,0,4*32)])
def fri_get_image_generator(get_image_args, qq):
if not get_image_args:
return
dvid_server = get_image_args[0][1]
dvid_uuid = get_image_args[0][2]
instance_name = get_image_args[0][7]
if dvid_server[:5] == 'gs://': # DICED google bucket
using_diced = True
if dvid_server.find(',') >= 0:
store = DicedStore(*dvid_server.split(','))
else:
store = DicedStore(dvid_server)
repo = store.open_repo(uuid=dvid_uuid)
dvid_node = repo.get_array(instance_name)
else:
using_diced = False
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
for gg in get_image_args:
while qq.qsize() >= 2:
time.sleep(10)
qq.put(fri_get_image(gg, dvid_node, using_diced, instance_name))
if using_diced:
store._shutdown_store()
def getLabelImageForFrame(self, Resource, PathInResource, timeframe):
"""
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return numpy array of image data at timeframe.
"""
if (self.shape == None):
self.getImageShape(Resource, PathInResource)
node_service = DVIDNodeService(Resource, PathInResource)
seg_frame = np.array(
node_service.get_labels3D(self._getSegmentationName(timeframe),
tuple(self.shape),
(0, 0, 0))).astype(np.uint32)
return seg_frame
def open_repo(self, name=None, uuid=None):
"""Open repository of the specified name (or by unique identifier).
If only the name is specified, the root node is taken by default.
Args:
name (str): name of repository to open
uuid (str): unique identifier for repository
Returns:
A repository object if it exists.
Exception:
Raises a DicedException if repo does not exist.
"""
if uuid is not None:
try:
ns = DVIDNodeService(self._server, uuid)
return DicedRepo(self._server, uuid, self)
except DVIDException:
# repo does not exist
raise DicedException("uuid does not exist")
elif name is not None:
repoid = self.get_repouuid(name)
return DicedRepo(self._server, repoid, self)
def download_to_h5( hostname, uuid, instance, roi, output_filepath, dset_name=None, compression='lzf', overlap_px=0):
"""
"""
ns = DVIDNodeService(hostname, uuid)
va = VoxelsAccessor(hostname, uuid, instance, throttle=True)
dset_name = dset_name or instance
assert roi, "Must provide a ROI"
logger.info("Downloading {hostname}/api/node/{uuid}/{instance}?roi={roi} to {output_filepath}/{dset_name}".format(**locals()))
substacks, _packing_factor = ns.get_roi_partition(roi, SUBSTACK_SIZE / DVID_BLOCK_SIZE)
# Substack tuples are (size, z, y, x)
substacks_zyx = np.array(substacks)[:, 1:]
# If the user specified an 'overlap', we add it to all substacks.
# Technically, this isn't very efficient, because a lot of overlapping
# pixels on the interior of the ROI will be fetched twice.
substacks_zyx[:,0] -= overlap_px
substacks_zyx[:,1] += overlap_px
roi_bb = ( np.min(substacks_zyx, axis=0),
np.max(substacks_zyx, axis=0)+SUBSTACK_SIZE )
with h5py.File(output_filepath, 'a') as output_file:
try:
del output_file[dset_name]
except KeyError:
pass
dset = output_file.create_dataset( dset_name, shape=roi_bb[1], dtype=va.dtype, chunks=True, compression=compression )
for i, substack_zyx in enumerate(substacks_zyx):
logger.info("Substack {}/{} {}: Downloading...".format( i, len(substacks_zyx), list(substack_zyx) ))
# Append a singleton channel axis
substack_bb = np.array(( tuple(substack_zyx) + (0,),
tuple(substack_zyx + SUBSTACK_SIZE) + (1,) ))
# Includes singleton channel
substack_data = va.get_ndarray(*substack_bb)
logger.info("Substack {}/{} {}: Writing...".format( i, len(substacks_zyx), list(substack_zyx) ))
dset[bb_to_slicing(*substack_bb[:,:-1])] = substack_data[...,0]
logger.info("DONE Downloading {hostname}/api/node/{uuid}/{instance}?roi={roi} to {output_filepath}/{dset_name}".format(**locals()))
def test_keyvalue(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_keyvalue("test_keyvalue")
node_service.put("test_keyvalue", "kkkk", "vvvv")
readback_value = node_service.get("test_keyvalue", "kkkk")
self.assertEqual(readback_value, "vvvv")
with self.assertRaises(RuntimeError):
node_service.put("test_keyvalue", "kkkk", 123) # 123 is not a buffer.
def ingest_data(self):
"""
Ideally, this would be run within the __init__ function,
but operators should never raise non-fatal exceptions within Operator.__init__()
(See OperatorMetaClass.__call__)
This serves as an alternative init function, from which we are allowed to raise exceptions.
"""
try:
node_service = DVIDNodeService(self._hostname, self._uuid)
roi_blocks_zyx = numpy.array( node_service.get_roi(self._roi_name) )
except DVIDException as ex:
if ex.status == httplib.NOT_FOUND:
raise OpDvidRoi.DatasetReadError("DVIDException: " + ex.message)
raise
except ErrMsg as ex:
raise OpDvidRoi.DatasetReadError("ErrMsg: " + str(ex))
# Store a dense array of the block flags
# Each voxel of this dataset represents a dvid block, so this volume
# is 32x smaller than the full-res data in every dimension.
# TODO: For now, we only respect the high-side of the bounding box, and force the low-side to 0,0,0.
shape = tuple(1 + numpy.max( roi_blocks_zyx, axis=0 ))
slice_shape = (1,) + shape[1:]
self._mem_file = h5py.File(str(id(self)), driver='core', backing_store=False, mode='w')
dset = self._mem_file.create_dataset('data',
shape=shape,
dtype=numpy.uint8,
chunks=slice_shape,
compression='lzf')
# Allocate temporary array slice for writing into the dataset
z_slice = numpy.ndarray( slice_shape, numpy.uint8 )
# Group the block indexes by z-coordinate
# FIXME: This might get slow for lots of blocks because of all the calls to this lambda.
# Consider upgrading to cytools.groupby instead
group_iter = groupby(roi_blocks_zyx, lambda block_index: block_index[0])
# Set a pixel for each block index, one z-slice at a time
for z, block_indexes_iter in group_iter:
block_indexes = numpy.array(list(block_indexes_iter))
block_indexes = block_indexes[:,1:] # drop z index
z_slice[:] = 0
z_slice[0][ tuple(block_indexes.transpose()) ] = 1
dset[z,:,:] = z_slice[0,...]
self._dset = dset
def _update_subvol_widget(self, node_uuid, dataname, typename):
"""
Update the subvolume widget with the min/max extents of the given node and dataname.
Note: The node and dataname do not necessarily have to match the currently
selected node and dataname.
This enables the right-click behavior, which can be used to
limit your data volume to the size of a different data volume.
"""
error_msg = None
try:
if typename == "roi":
node_service = DVIDNodeService(self._hostname, str(node_uuid))
roi_blocks_xyz = numpy.array(node_service.get_roi(str(dataname)))
maxindex = tuple(DVID_BLOCK_WIDTH * (1 + numpy.max(roi_blocks_xyz, axis=0)))
minindex = (0, 0, 0) # Rois are always 3D
axiskeys = "xyz"
# If the current selection is a dataset, then include a channel dimension
if self.get_selection().typename != "roi":
axiskeys = "cxyz"
minindex = (0,) + minindex
maxindex = (1,) + maxindex # FIXME: This assumes that the selected data has only 1 channel...
else:
# Query the server
raw_metadata = VoxelsAccessor.get_metadata(self._hostname, node_uuid, dataname)
voxels_metadata = VoxelsMetadata(raw_metadata)
maxindex = voxels_metadata.shape
minindex = voxels_metadata.minindex
axiskeys = voxels_metadata.axiskeys
# If the current selection is a roi, then remove the channel dimension
if self.get_selection().typename == "roi":
axiskeys = "xyz"
minindex = minindex[1:]
maxindex = maxindex[1:]
except (DVIDException, ErrMsg) as ex:
error_msg = str(ex)
log_exception(logger)
else:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
if error_msg:
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(False)
QMessageBox.critical(self, "DVID Error", error_msg)
self._subvol_widget.initWithExtents("", (), (), ())
return
self._subvol_widget.initWithExtents(axiskeys, maxindex, minindex, maxindex)
def test_sparselabelmask(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**", "test_sparselabelmask")
node_service.create_labelarray("test_sparselabelmask", 64)
# Diagonal Stripes
data = numpy.indices(
(128, 128, 128),
dtype=numpy.uint16).sum(axis=0).astype(numpy.uint64)
data[:] //= 5
data_downsampled = data[::2, ::2, ::2].copy(
'C') # cheap downsampling...
TEST_LABEL = data[
64, 64,
64] # This is a big diagonal that passes through all but the first block.
assert TEST_LABEL in data
assert data.flags['C_CONTIGUOUS']
# Load two scales
node_service.put_labelblocks3D("test_sparselabelmask", data, (0, 0, 0))
node_service.put_labelblocks3D("test_sparselabelmask",
data_downsampled, (0, 0, 0),
scale=1)
# Retrieve a body at full scale
block_starts, block_masks = node_service.get_sparselabelmask(
TEST_LABEL, "test_sparselabelmask", 0)
assert len(block_starts) == len(block_masks) == 7
assert block_masks[0].dtype == numpy.bool
block_stops = block_starts + 64
for start, stop, mask in zip(block_starts, block_stops, block_masks):
expected_mask = (data[bb_to_slicing(start, stop)] == TEST_LABEL)
assert (mask == expected_mask).all()
# Retrieve that body at half scale
block_starts, block_masks = node_service.get_sparselabelmask(
TEST_LABEL, "test_sparselabelmask", 1)
assert len(block_starts) == len(block_masks) == 1
expected_mask = (data_downsampled == TEST_LABEL)
assert (block_starts[0] == (0, 0, 0)).all()
assert (block_masks[0] == expected_mask).all()
def test_labels_3d(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid, "*****@*****.**", "test_label_app")
node_service.create_labelblk("test_labels_3d")
lblksize = node_service.get_blocksize("test_labels_3d")
self.assertEqual(32, lblksize)
node_service.create_labelblk("test_labels_3d64", "", 64)
lblksize64 = node_service.get_blocksize("test_labels_3d64")
self.assertEqual(64, lblksize64)
data = numpy.random.randint(0, 2**63-1, (128,128,128)).astype(numpy.uint64)
assert data.flags['C_CONTIGUOUS']
node_service.put_labels3D( "test_labels_3d", data, (0,0,0) )
retrieved_data = node_service.get_labels3D( "test_labels_3d", (30,31,32), (20,20,20) )
self.assertTrue( (retrieved_data == data[20:50, 20:51, 20:52]).all() )
def parallelize_roi(self, roi, chunk_size, border=0, find_neighbors=False):
"""Creates an RDD from subvolumes found in an ROI.
This is analogous to the Spark parallelize function.
It currently defines the number of partitions as the
number of subvolumes.
TODO: implement general partitioner given other
input such as bounding box coordinates.
Args:
roi (str): name of DVID ROI at current server and uuid
chunk_size (int): the desired dimension of the subvolume
border (int): size of the border surrounding the subvolume
find_neighbors (bool): whether to identify neighbors
Returns:
RDD as [(subvolume id, subvolume)] and # of subvolumes
"""
# function will export and should include dependencies
subvolumes = [] # x,y,z,x2,y2,z2
from libdvid import DVIDNodeService, SubstackXYZ
# extract roi for a given chunk size
node_service = DVIDNodeService(str(self.dvid_server), str(self.uuid))
substacks, packing_factor = node_service.get_roi_partition(str(roi), chunk_size / self.BLK_SIZE)
# create roi array giving unique substack ids
for substack_id, substack in enumerate(substacks):
# use substack id as key
subvolumes.append((substack_id, Subvolume(substack_id, substack, chunk_size, border)))
# grab all neighbors for each substack
if find_neighbors:
# inefficient search for all boundaries
for i in range(0, len(subvolumes) - 1):
for j in range(i + 1, len(subvolumes)):
subvolumes[i][1].recordborder(subvolumes[j][1])
# Potential TODO: custom partitioner for grouping close regions
return self.sc.parallelize(subvolumes, len(subvolumes))
def test_roi_partition(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_roi("test_roi_partition")
blocks = [(0,0,0),(1,1,1),(2,2,2),(3,3,3)]
node_service.post_roi("test_roi_partition", blocks)
substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32)])
self.assertEqual( packing_factor, float(len(blocks))/(len(substacks) * 4**3) )
blocks += [(4,0,0)]
node_service.post_roi("test_roi_partition", blocks)
substacks, packing_factor = node_service.get_roi_partition("test_roi_partition", 4)
self.assertEqual(substacks, [SubstackXYZ(0,0,0,4*32), SubstackXYZ(128,0,0,4*32)])
def set_sync(dvid_server, uuid, srcname, destname):
"""Sets a sync on srcname to point to destname.
Note: only a limited number of syncs are possible.
libdvid-cpp will throw an error if a sync is not possible.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
srcname (str): data instance with new sync
destname (str): data instance pointed to by new sync
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
ns = DVIDNodeService(str(dvid_server), str(uuid))
data = {"sync": destname}
ns.custom_request(srcname + "/sync", json.dumps(data).encode('utf-8'), ConnectionMethod.POST)
def setUpClass(cls):
"""
Override. Called by nosetests.
"""
cls.uuid = get_testrepo_root_uuid()
cls.roi_name = "OpDvidRoi_test_roi"
node_service = DVIDNodeService(TEST_DVID_SERVER, cls.uuid)
node_service.create_roi(cls.roi_name)
# Create an upside-down L-shaped roi, something like this:
#
# 1 1 1 1
# 1 1 1 1
# 1 1 0 0
# 1 1 0 0
cls.expected_data = numpy.ones((64, 128, 128), dtype=numpy.uint8, order="C")
cls.expected_data[:, 64:, 64:] = 0
block_values = cls.expected_data[::32, ::32, ::32]
coords = numpy.transpose(numpy.nonzero(block_values))
node_service.post_roi(cls.roi_name, coords)
def __init__(self, dvidserver, uuid, dataname):
"""Initialization.
Args:
dvidserver (string): address for dvid server
uuid (string): uuid for DVID instance
dataname (string): name of data instance
"""
self.server = dvidserver
self.uuid = uuid
self.name = dataname
# check DVID existence and get meta
try:
node_service = DVIDNodeService(str(dvidserver), str(uuid))
self.info = node_service.get_typeinfo(dataname)
except DVIDException:
raise ValueError("Instance not available")
self.datatype = str(self.info["Base"]["TypeName"])
def copy_roi( src_info, dest_info ):
src_service = DVIDNodeService(src_info.server, src_info.uuid)
dest_service = DVIDNodeService(dest_info.server, dest_info.uuid)
# If necessary, create the ROI on the destination server
try:
info = dest_service.get_typeinfo(dest_info.name)
except DVIDException:
dest_service.create_roi(dest_info.name)
roi_blocks = src_service.get_roi(src_info.name)
dest_service.post_roi(dest_info.name, roi_blocks)
