def retrieve_node_service(server, uuid, resource_server, resource_port, appname="sparkservices"):
"""Create a DVID node service object"""
server = str(server)
# refresh dvid server meta if localhost (since it is exclusive or points to global db)
"""
if server.startswith("http://127.0.0.1") or \
server.startswith("http://localhost") or \
server.startswith("127.0.0.1") or server.startswith("localhost"):
import os
if not os.path.exists("/tmp/reloaded.hack"):
addr = server + "/api/server/reload-metadata"
if not server.startswith("http://"):
addr = "http://" + addr
session = default_dvid_session()
session.post(addr)
open("/tmp/reloaded.hack", 'w').close()
"""
from libdvid import DVIDNodeService
import os
username = os.environ["USER"]
if resource_server != "":
node_service = DVIDNodeService(server, str(uuid), username, appname, str(resource_server), resource_port)
else:
node_service = DVIDNodeService(server, str(uuid), username, appname)
return node_service
def copy_roi(src_info, dest_info):
src_service = DVIDNodeService(src_info.server, src_info.uuid)
dest_service = DVIDNodeService(dest_info.server, dest_info.uuid)
# If necessary, create the ROI on the destination server
try:
info = dest_service.get_typeinfo(dest_info.name)
except DVIDException:
dest_service.create_roi(dest_info.name)
roi_blocks = src_service.get_roi(src_info.name)
dest_service.post_roi(dest_info.name, roi_blocks)
def setUpClass(cls):
"""
Override. Called by nosetests.
"""
# Choose names
cls.dvid_repo = "datasetA"
cls.data_name = "random_data"
cls.volume_location = "/repos/{dvid_repo}/volumes/{data_name}".format(
**cls.__dict__)
cls.data_uuid = get_testrepo_root_uuid()
cls.node_location = "/repos/{dvid_repo}/nodes/{data_uuid}".format(
**cls.__dict__)
# Generate some test data
#data = numpy.random.randint(0, 255, (128, 256, 512, 1)).astype( numpy.uint8 )
data = numpy.zeros((128, 256, 512, 1), dtype=numpy.uint8)
data.flat[:] = numpy.arange(numpy.prod((128, 256, 512, 1)))
cls.original_data = data
cls.voxels_metadata = VoxelsMetadata.create_default_metadata(
data.shape, data.dtype, "zyxc", 1.0, "")
# Write it to a new data instance
node_service = DVIDNodeService(TEST_DVID_SERVER, cls.data_uuid)
node_service.create_grayscale8(cls.data_name)
node_service.put_gray3D(cls.data_name, data[..., 0], (0, 0, 0))
def evaluate_pr(tbars_pd, tbars_gt,
dist_thresh=27, conf_thresholds=None,
segm_dvid=None):
tbars_pd = tbars_pd.get_tbars()
tbars_gt = tbars_gt.get_tbars(1.)
if conf_thresholds is None:
cc = np.unique(tbars_pd.conf)
conf_thresholds = (cc[1:]+cc[:-1])/2
if segm_dvid is not None:
segm_dvid_node = DVIDNodeService(
segm_dvid[0], segm_dvid[1],
'flyem_syn_eval', 'flyem_syn_eval')
segm_dvid_segm = segm_dvid[2]
lbls_pd = get_labels(segm_dvid_node, segm_dvid_segm, tbars_pd)
lbls_gt = get_labels(segm_dvid_node, segm_dvid_segm, tbars_gt)
else:
lbls_pd = None
lbls_gt = None
result = obj_pr_curve(tbars_pd, tbars_gt,
dist_thresh, conf_thresholds,
lbls_pd, lbls_gt)
return result
def test_roi_2(self):
# Create X-shaped ROI
# This ensures that the x-run encoding within our ROI message works properly
# (There used to be a bug in post_roi() that would have broken this test.)
_ = 0
# 8x8 blocks = 256x256 px
roi_mask_yx = numpy.array([[1, _, _, _, _, _, _, 1],
[1, 1, _, _, _, _, 1, 1],
[_, 1, 1, _, _, 1, 1, _],
[_, _, 1, 1, 1, 1, _, _],
[_, _, _, 1, 1, _, _, _],
[_, _, 1, 1, 1, 1, _, _],
[_, 1, 1, _, _, 1, 1, _],
[1, 1, _, _, _, _, 1, 1]])
roi_mask_zyx = numpy.zeros((8, 8, 8))
roi_mask_zyx[:] = roi_mask_yx[None, :, :]
roi_block_indexes = numpy.transpose(roi_mask_zyx.nonzero())
ns = DVIDNodeService(TEST_DVID_SERVER, self.uuid, "*****@*****.**",
"test_roi_2_app")
ns.create_roi('test-diagonal-256')
ns.post_roi('test-diagonal-256', roi_block_indexes)
fetched_blocks = ns.get_roi("test-diagonal-256")
fetched_blocks = numpy.array(fetched_blocks)
# Both arrays happen to be sorted already
assert (fetched_blocks == roi_block_indexes).all()
def open_repo(self, name=None, uuid=None):
"""Open repository of the specified name (or by unique identifier).
If only the name is specified, the root node is taken by default.
Args:
name (str): name of repository to open
uuid (str): unique identifier for repository
Returns:
A repository object if it exists.
Exception:
Raises a DicedException if repo does not exist.
"""
if uuid is not None:
try:
ns = DVIDNodeService(self._server, uuid)
return DicedRepo(self._server, uuid, self)
except DVIDException:
# repo does not exist
raise DicedException("uuid does not exist")
elif name is not None:
repoid = self.get_repouuid(name)
return DicedRepo(self._server, repoid, self)
def get_synapses_dvid(dvid_server, dvid_uuid, dvid_annot, dvid_roi):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid,
'flyem_syn_eval','flyem_syn_eval')
synapses_json = dvid_node.custom_request(
'%s/roi/%s' % (dvid_annot, dvid_roi),
None, ConnectionMethod.GET)
return synapses_json.decode()
def _init_version(self, uuid):
# create connection to repo
self.nodeconn = DVIDNodeService(str(self.server), str(uuid))
# fetch all repo information
status, data, errmsg = self.rawconn.make_request("/repo/" + uuid + "/info", ConnectionMethod.GET)
self.repoinfo = json.loads(data)
self.allinstances = {}
# load all versions in repo
self.alluuids = set()
dag = self.repoinfo["DAG"]["Nodes"]
for uuidt, nodedata in dag.items():
self.alluuids.add(str(nodedata["UUID"]))
for instancename, val in self.repoinfo["DataInstances"].items():
# name is not necessarily unique to a repo
self.allinstances[(str(instancename), str(val["Base"]["DataUUID"]))] = str(val["Base"]["TypeName"])
# datainstances that should be hidden (array of names)
try:
self.hidden_instances = set(self.nodeconn.get_json(self.MetaLocation, self.RestrictionName))
except:
self.hidden_instances = set()
nodeids = {}
# check if locked note
dag = self.repoinfo["DAG"]["Nodes"]
for tuuid, nodedata in dag.items():
nodeids[str(nodedata["VersionID"])] = nodedata
if tuuid.startswith(uuid):
self.uuid = str(tuuid)
self.current_node = nodedata
self.locked = nodedata["Locked"]
# load all ancestors
ancestors = set()
# commit history uuid, commit note in order from oldest to newest
self.loghistory = []
currnode = self.current_node
while True:
ancestors.add(str(currnode["UUID"]))
self.loghistory.append((str(currnode["UUID"]), currnode["Note"]))
if len(currnode["Parents"]) > 0:
currnode = nodeids[str(currnode["Parents"][0])]
else:
break
self.loghistory.reverse()
# load all instances
self.activeinstances = {}
if not self.locked:
tempuuid = self.loghistory[-1][0]
self.loghistory[-1] = (tempuuid, "(open node)")
for instancename, val in self.repoinfo["DataInstances"].items():
if str(val["Base"]["RepoUUID"]) in ancestors:
self.activeinstances[str(instancename)] = str(val["Base"]["TypeName"])
def get_blocksize(dvid_server, uuid, dataname):
"""Gets block size for supplied data name.
Note:
Does not check for existence of body and whether
it is an array type. The block size is always
assumes to be isotropic.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
dataname (str): data instance
Returns:
(z,y,x) blocksize.
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
ns = DVIDNodeService(str(dvid_server), str(uuid))
info = ns.get_typeinfo(dataname)
x, y, z = info["Extended"]["BlockSize"] # DVID ordered x,y,z
return (z, y, x)
def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
num_at_once = 100000
n_tot = len(tbars_json)
for ii in range(0, n_tot, num_at_once):
jj = np.minimum(ii + num_at_once, n_tot)
data = json.dumps(tbars_json[ii:jj])
oo = dvid_node.custom_request('%s/elements' % dvid_annot,
data.encode('utf-8'),
ConnectionMethod.POST)
def fri_get_image_generator(get_image_args, qq):
if not get_image_args:
return
dvid_server = get_image_args[0][1]
dvid_uuid = get_image_args[0][2]
instance_name = get_image_args[0][7]
if dvid_server[:5] == 'gs://': # DICED google bucket
using_diced = True
if dvid_server.find(',') >= 0:
store = DicedStore(*dvid_server.split(','))
else:
store = DicedStore(dvid_server)
repo = store.open_repo(uuid=dvid_uuid)
dvid_node = repo.get_array(instance_name)
else:
using_diced = False
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
for gg in get_image_args:
while qq.qsize() >= 2:
time.sleep(10)
qq.put(fri_get_image(gg, dvid_node, using_diced, instance_name))
if using_diced:
store._shutdown_store()
def test_roi(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**", "test_roi_app")
node_service.create_roi("test_roi")
node_service.post_roi("test_roi", [(1, 2, 3), (2, 3, 4), (4, 5, 6)])
roi_blocks = node_service.get_roi("test_roi")
self.assertEqual(roi_blocks, [(1, 2, 3), (2, 3, 4), (4, 5, 6)])
def test_DataInstance(self):
"""Tests DataInstance class.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
ns = DVIDNodeService(dvidserver, uuid)
ns.create_labelblk("labels")
ns.create_grayscale8("gray")
ns.create_keyvalue("kv")
try:
temp = DataInstance(dvidserver, uuid, "blah")
except ValueError:
# correct caught error
self.assertTrue(True)
labels = DataInstance(dvidserver, uuid, "labels")
gray = DataInstance(dvidserver, uuid, "gray")
kv = DataInstance(dvidserver, uuid, "kv")
self.assertTrue(labels.is_array())
self.assertTrue(labels.is_labels())
self.assertTrue(gray.is_array())
self.assertFalse(gray.is_labels())
self.assertFalse(kv.is_array())
self.assertFalse(kv.is_labels())
def test_roi_ptquery(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**", "test_ptquery2 app")
node_service.create_roi("test_roi")
node_service.post_roi("test_roi", [(1, 2, 3), (2, 3, 4), (4, 5, 6)])
query_results = node_service.roi_ptquery("test_roi",
[(0, 0, 0), (32, 64, 32 * 3)])
self.assertEqual(query_results, [False, True])
def test_custom_request(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**", "test_custom_app")
response_body = node_service.custom_request("log", b"",
ConnectionMethod.GET)
# This shouldn't raise an exception
json_data = json.loads(response_body)
def test_labels_3d_volsync(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_labelblk("test_labels_3d2", "test_labels_3d2_vol")
data = numpy.random.randint(0, 2**63-1, (128,128,128)).astype(numpy.uint64).transpose()
assert data.flags['F_CONTIGUOUS']
node_service.put_labels3D( "test_labels_3d2", data, (0,0,0) )
retrieved_data = node_service.get_labels3D( "test_labels_3d2", (30,31,32), (20,20,20) )
self.assertTrue( (retrieved_data == data[20:50, 20:51, 20:52]).all() )
def test_grayscale_3d(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_grayscale8("test_grayscale_3d")
data = numpy.random.randint(0, 255, (128,128,128)).astype(numpy.uint8).transpose()
assert data.flags['F_CONTIGUOUS']
node_service.put_gray3D( "test_grayscale_3d", data, (0,0,0) )
retrieved_data = node_service.get_gray3D( "test_grayscale_3d", (30,31,32), (20,20,20) )
self.assertTrue( (retrieved_data == data[20:50, 20:51, 20:52]).all() )
def storeFeaturesForFrame(self, features, timeframe):
"""
Stores feature data
"""
assert(self.server_address is not None)
assert(self.uuid is not None)
node_service = DVIDNodeService(self.server_address, self.uuid)
node_service.create_keyvalue(self.keyvalue_store)
node_service.put(self.keyvalue_store, "frame-{}".format(timeframe), json.dumps(features))
def test_keyvalue(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid)
node_service.create_keyvalue("test_keyvalue")
node_service.put("test_keyvalue", "kkkk", "vvvv")
readback_value = node_service.get("test_keyvalue", "kkkk")
self.assertEqual(readback_value, "vvvv")
with self.assertRaises(RuntimeError):
node_service.put("test_keyvalue", "kkkk", 123) # 123 is not a buffer.
def setUpClass(cls):
cls.uuid = get_testrepo_root_uuid()
node_service = DVIDNodeService(TEST_DVID_SERVER, cls.uuid,
"unittest@test_roi_utils.py",
"Test_roi_utils")
node_service.create_roi("src_roi")
node_service.post_roi("src_roi", [(1, 2, 3), (2, 3, 4), (4, 5, 6)])
roi_blocks = node_service.get_roi("src_roi")
def loadFeaturesForFrame(self, features, timeframe):
"""
loads feature data
"""
assert(self.server_address is not None)
assert(self.uuid is not None)
node_service = DVIDNodeService(self.server_address, self.uuid)
node_service.create_keyvalue(self.keyvalue_store)
return json.loads(node_service.get(self.keyvalue_store, "frame-{}".format(timeframe)))
def create_new( hostname, uuid, data_name, voxels_metadata ):
node_service = DVIDNodeService(hostname, uuid)
typename = voxels_metadata.determine_dvid_typename()
if typename == "grayscale8":
node_service.create_grayscale8(data_name)
elif typename == "labels64":
node_service.create_labelblk("test_labels_3d")
else:
raise RuntimeError("Don't know how to create a data instance of type '{}'".format(typename))
def test_labelarray_get_volumes(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**",
"test_labelarray_get_volumes")
node_service.create_labelarray("test_labelarray64_get", 64)
data = numpy.random.randint(0, 10,
(128, 128, 128)).astype(numpy.uint64)
assert data.flags['C_CONTIGUOUS']
# Try reading from empty data (before anything is loaded in the instance)
# Should be all zeros
retrieved_data = node_service.get_labels3D("test_labelarray64_get",
(64, 128, 128), (0, 0, 0))
assert (retrieved_data == 0).all()
# Now populate the instance with some data
node_service.put_labelblocks3D("test_labelarray64_get", data,
(0, 0, 0))
retrieved_data = node_service.get_labels3D("test_labelarray64_get",
(30, 31, 32), (20, 20, 20))
assert (retrieved_data == data[20:50, 20:51, 20:52]).all()
# First with the ordinary GET
retrieved_data = node_service.get_labels3D("test_labelarray64_get",
(64, 64, 64), (0, 0, 0))
assert (retrieved_data == data[:64, :64, :64]).all()
retrieved_data = node_service.get_labels3D("test_labelarray64_get",
(128, 128, 128), (0, 0, 0))
assert (retrieved_data == data).all()
# Again with the blockwise GET
retrieved_data = node_service.get_labelarray_blocks3D(
"test_labelarray64_get", (64, 64, 64), (0, 0, 0))
assert (retrieved_data == data[:64, :64, :64]).all()
retrieved_data = node_service.get_labelarray_blocks3D(
"test_labelarray64_get", (128, 128, 128), (0, 0, 0))
assert (retrieved_data == data).all()
# Try the requests-based alternative (request raw via requests.get, then inflate afterwards)
r = requests.get(
f'http://{TEST_DVID_SERVER}/api/node/{self.uuid}/test_labelarray64_get/blocks/128_128_128/0_0_0?compression=blocks'
)
r.raise_for_status()
inflated_data = DVIDNodeService.inflate_labelarray_blocks3D_from_raw(
r.content, (128, 128, 128), (0, 0, 0))
assert (inflated_data == data).all()
# What happens if we request a block that doesn't exist on the server?
missing = node_service.get_labelarray_blocks3D("test_labelarray64_get",
(128, 128, 128),
(1024, 1024, 1024),
scale=6)
assert missing.shape == (128, 128, 128)
assert (missing == 0).all()
def __init__(self, server_address, uuid, key):
assert WITH_DVID, "Need dvid"
self.ds = DVIDNodeService(server_address, uuid)
self.key = key
# get the shape the dvid way...
endpoint = "/" + self.key + "/info"
attributes = self.ds.custom_request(endpoint, "", ConnectionMethod.GET)
# TODO do we need to increase by 1 here ?
self._shape = tuple(mp + 1 for mp in attributes["MaxPoint"])
def getTimeRange(self, Resource, PathInResource):
"""
Count Label images to derive the total number of frames
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return tuple of (first frame, last frame)
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
return config["time_range"]
def test_labelarray_put_single_block(self):
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"*****@*****.**",
"test_labelarray_put_single_block")
node_service.create_labelarray("test_labelarray64", 64)
data = numpy.random.randint(0, 10, (64, 64, 64)).astype(numpy.uint64)
node_service.put_labelblocks3D("test_labelarray64", data, (0, 0, 64))
retrieved_data = node_service.get_labels3D("test_labelarray64",
(64, 64, 64), (0, 0, 64))
assert (retrieved_data == data[:64, :64, :64]).all()
def get_output_vol(index):
output_service = DVIDNodeService(
str(config['outputs'][index]['dvid']['server']),
str(config['outputs'][index]['dvid']['uuid']))
output_name = config['outputs'][index]['dvid']['segmentation-name']
output_bb_xyz = config['outputs'][index]['geometry']['bounding-box']
output_bb_zyx = np.array(output_bb_xyz)[:, ::-1]
output_shape = output_bb_zyx[1] - output_bb_zyx[0]
output_volume = output_service.get_labels3D(output_name, output_shape,
output_bb_zyx[0])
return output_volume
def test_copy_roi(self):
expected_blocks = [(1, 2, 3), (2, 3, 4), (4, 5, 6)]
# We aren't really exercising transferring between servers here. Too bad.
src_info = (TEST_DVID_SERVER, self.uuid, 'src_roi')
dest_info = (TEST_DVID_SERVER, self.uuid, 'dest_roi')
node_service = DVIDNodeService(TEST_DVID_SERVER, self.uuid,
"unittest@test_roi_utils.py",
"Test_roi_utils")
written_blocks = node_service.get_roi("src_roi")
assert set(expected_blocks) == set(written_blocks)
def getImageShape(self, Resource, PathInResource):
"""
Derive Image Shape from label image.
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return list with image dimensions
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
self.shape = config["shape"]
return self.shape
def test_isdatainstance(self):
"""Tests is_datainstance function.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
ns = DVIDNodeService(dvidserver, uuid)
ns.create_labelblk("labels")
self.assertTrue(is_datainstance(dvidserver, uuid, "labels"))
self.assertFalse(is_datainstance(dvidserver, uuid, "labels2"))
