def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
num_at_once = 100000
n_tot = len(tbars_json)
for ii in range(0, n_tot, num_at_once):
jj = np.minimum(ii + num_at_once, n_tot)
data = json.dumps(tbars_json[ii:jj])
oo = dvid_node.custom_request('%s/elements' % dvid_annot,
data.encode('utf-8'),
ConnectionMethod.POST)
def test_get(self):
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request(
"/server/info", ConnectionMethod.GET)
self.assertEqual(status, 200)
self.assertEqual(error_message, "")
# This shouldn't raise an exception
json_data = json.loads(body)
def test_garbage_request_throws(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2")
try:
status, body, error_message = connection.make_request(
"/does/not/exist", ConnectionMethod.GET)
except DVIDException as ex:
assert ex.status == 404
else:
assert False, "Bad requests are supposed to throw exceptions!"
def test_garbage_request(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2")
status, body, error_message = connection.make_request(
"/does/not/exist", ConnectionMethod.GET, checkHttpErrors=False)
# No connection errors, just missing data.
self.assertEqual(error_message, "")
self.assertEqual(status, 404)
self.assertNotEqual(body, "")
def test_garbage_request(self):
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request(
"/does/not/exist", ConnectionMethod.GET)
# No connection errors, just missing data.
self.assertEqual(error_message, "")
self.assertEqual(status, 404)
self.assertNotEqual(body, "")
def delete_all_data_instances(uuid):
connection = DVIDConnection(TEST_DVID_SERVER)
repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid )
status, body, error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET)
assert status == http.client.OK, "Request for {} returned status {}".format(repo_info_uri, status)
assert error_message == ""
repo_info = json.loads(body)
for instance_name in list(repo_info["DataInstances"].keys()):
status, body, error_message = connection.make_request( "/api/repo/{uuid}/{dataname}?imsure=true"
.format( uuid=uuid, dataname=str(instance_name) ),
ConnectionMethod.DELETE )
def delete_all_data_instances(uuid):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp")
repo_info_uri = "/repo/{uuid}/info".format(uuid=uuid)
status, body, _error_message = connection.make_request(
repo_info_uri, ConnectionMethod.GET)
repo_info = json.loads(body)
for instance_name in repo_info["DataInstances"].keys():
status, body, _error_message = connection.make_request(
"/repo/{uuid}/{dataname}?imsure=true".format(
uuid=uuid, dataname=str(instance_name)),
ConnectionMethod.DELETE,
checkHttpErrors=False)
def get_metadata( hostname, uuid, data_name ):
"""
Query the voxels metadata for the given node/data_name.
"""
connection = DVIDConnection(hostname)
rest_query = "/node/{uuid}/{data_name}/metadata".format( uuid=uuid, data_name=data_name )
status, response_body, _err_msg = connection.make_request( rest_query, ConnectionMethod.GET )
try:
json_data = json.loads(response_body)
except ValueError:
raise RuntimeError("Response body could not be parsed as valid json:\n"
"GET " + rest_query + "\n" + response_body)
return VoxelsMetadata( json_data )
def set_syncs(dvid_server,
dvid_uuid,
dvid_annot,
dvid_segm=None,
dvid_labelsz=None):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, os.getenv('USER'),
'fpl')
dvid_conn = DVIDConnection(dvid_server, os.getenv('USER'), 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
if dvid_labelsz is not None:
# create labelsz if necessary
try:
dvid_node.custom_request('%s/info' % dvid_labelsz, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'labelsz',
'dataname': dvid_labelsz
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
# sync synapses to segmentation
if dvid_segm is not None:
syn_sync_json = json.dumps({'sync': dvid_segm})
dvid_node.custom_request('%s/sync' % dvid_annot,
syn_sync_json.encode('utf-8'),
ConnectionMethod.POST)
# sync labelsz to synapses
if dvid_labelsz is not None:
lsz_sync_json = json.dumps({'sync': dvid_annot})
dvid_node.custom_request('%s/sync' % dvid_labelsz,
lsz_sync_json.encode('utf-8'),
ConnectionMethod.POST)
def get_testrepo_root_uuid():
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET)
assert status == http.client.OK, "Request for /repos/info returned status {}".format( status )
assert error_message == ""
repos_info = json.loads(body)
test_repos = [uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo']
if test_repos:
uuid = test_repos[0][0]
return str(uuid)
else:
from libdvid import DVIDServerService
server = DVIDServerService(TEST_DVID_SERVER)
uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse.");
return str(uuid)
def get_testrepo_root_uuid():
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp")
status, body, _error_message = connection.make_request(
"/repos/info", ConnectionMethod.GET)
repos_info = json.loads(body)
test_repos = [
uuid_repo_info for uuid_repo_info in list(repos_info.items())
if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo'
]
if test_repos:
uuid = test_repos[0][0]
return str(uuid)
else:
from libdvid import DVIDServerService
server = DVIDServerService(TEST_DVID_SERVER)
uuid = server.create_new_repo(
"testrepo", "This repo is for unit tests to use and abuse.")
return str(uuid)
def _handle_new_hostname(self):
"""
Called by 'Connect' button.
Connect to the server, download the server info and repo info,
and populate the GUI widgets with the data.
"""
new_hostname = str(self._hostname_combobox.currentText())
if '://' in new_hostname:
new_hostname = new_hostname.split('://')[1]
error_msg = None
self._server_info = None
self._repos_info = None
self._current_repo = None
self._hostname = None
try:
# Query the server
connection = DVIDConnection(new_hostname)
self._server_info = ContentsBrowser._get_server_info(connection)
self._repos_info = ContentsBrowser._get_repos_info(connection)
self._hostname = new_hostname
self._connection = connection
except DVIDException as ex:
error_msg = "libdvid.DVIDException: {}".format(ex.message)
except ErrMsg as ex:
error_msg = "libdvid.ErrMsg: {}".format(ex.message)
if error_msg:
QMessageBox.critical(self, "Connection Error", error_msg)
self._populate_node_list(None)
else:
self._connect_button.setEnabled(False)
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
enable_contents = self._repos_info is not None
self._data_groupbox.setEnabled(enable_contents)
self._node_groupbox.setEnabled(enable_contents)
self._new_data_groupbox.setEnabled(enable_contents)
self._populate_hostinfo_table()
self._populate_repo_tree()
def test_zz_quickstart_usage(self):
import json
import numpy
from libdvid import DVIDConnection, ConnectionMethod
from libdvid.voxels import VoxelsAccessor, VoxelsMetadata
# Open a connection to DVID
connection = DVIDConnection("127.0.0.1:8000")
# Get detailed dataset info: /api/repos/info (note: /api is prepended automatically)
status, body, _error_message = connection.make_request(
"/repos/info", ConnectionMethod.GET)
dataset_details = json.loads(body)
# print(json.dumps( dataset_details, indent=4 ))
# Create a new remote volume (assuming you already know the uuid of the node)
uuid = UUID
voxels_metadata = VoxelsMetadata.create_default_metadata(
(0, 0, 0, 1), numpy.uint8, 'zyxc', 1.0, "")
VoxelsAccessor.create_new("127.0.0.1:8000", uuid, "my_volume",
voxels_metadata)
# Use the VoxelsAccessor convenience class to manipulate a particular data volume
accessor = VoxelsAccessor("127.0.0.1:8000", uuid, "my_volume")
# print(dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape)
# Add some data (must be block-aligned)
# Must include all channels.
updated_data = numpy.ones((256, 192, 128, 1), dtype=numpy.uint8)
accessor[256:512, 32:224, 0:128, 0] = updated_data
# OR:
#accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data )
# Read from it (First axis is channel.)
cutout_array = accessor[300:330, 40:120, 10:110, 0]
# OR:
cutout_array = accessor.get_ndarray((300, 40, 10, 0),
(330, 120, 110, 1))
assert isinstance(cutout_array, numpy.ndarray)
assert cutout_array.shape == (30, 80, 100, 1)
def __init__(self, server, uuid, dicedstore, readonly=False):
self.server = server
# create connection object to DVID
self.rawconn = DVIDConnection(server)
# hold reference
self.dicedstore = dicedstore
self.uuid = None
# DVID version node connection
self.nodeconn = None
# all instances to ignore for ls
self.hidden_instances = None
# fetch all repo information
self.repoinfo = None
# whether current version is read only
self.locked = None
# meta for current node
self.current_node = None
# instances available at version
self.allinstances = None
self.activeinstances = None
# initialize version specific data
self._init_version(uuid)
# create meta types if not currently available
# This operation is not available if the DVID server is running in
# readonly mode, but most of this library will still work if it is.
# Supplying readonly=True in the constructor will bypass this initialisation.
if not readonly:
if self.MetaLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.MetaLocation)
if self.FilesLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.FilesLocation)
def create_rawarray8(dvid_server,
uuid,
name,
blocksize=(64, 64, 64),
compression=Compression.DEFAULT):
"""Create 8 bit labels data structure.
Note:
Currenly using uint8blk only. Does not check whether
the type already exists. DVIDExceptions are uncaught.
libdvid-cpp can be used directly but this also supports
setting the compression type.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
blocksize (3 int tuple): block size z,y,x
compression (Compression enum): compression to be used
minimal_extents: box [(z0,y0,x0), (z1,y1,x1)].
If provided, data extents will be at least this large (possibly larger).
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
conn = DVIDConnection(dvid_server)
typename = "uint8blk"
logger.info(
"Creating {typename} instance: {uuid}/{name}".format(**locals()))
endpoint = "/repo/" + uuid + "/instance"
blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0])
data = {"typename": typename, "dataname": name, "BlockSize": blockstr}
if compression != Compression.DEFAULT:
data["Compression"] = compression.value
conn.make_request(endpoint, ConnectionMethod.POST,
json.dumps(data).encode('utf-8'))
def test_sycns(self):
"""Test sync check and setting a sync.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
create_label_instance(dvidserver, uuid, "labels")
# check if labels is listening to labels2
self.assertFalse(has_sync(dvidserver, uuid, "labels", "bodies"))
# create labelvol and sync to it
conn = DVIDConnection(dvidserver)
endpoint = "/repo/" + uuid + "/instance"
data = {"typename": "labelvol", "dataname": "bodies"}
conn.make_request(endpoint, ConnectionMethod.POST,
json.dumps(data).encode())
set_sync(dvidserver, uuid, "labels", "bodies")
self.assertTrue(has_sync(dvidserver, uuid, "labels", "bodies"))
def list_repos(self):
"""List all repositories in the store.
Returns:
A list of (name, uuid) tuples
"""
data = None
try:
conn = DVIDConnection(self._server)
code, data, errmsg = conn.make_request("/repos/info",
ConnectionMethod.GET)
except DVIDException as err:
raise DicedException("Failed to access /repos/info on DVID")
jdata = json.loads(data)
res = []
for _key, val in jdata.items():
res.append((str(val["Alias"]), str(val["Root"])))
return res
def __init__(self, server, uuid, dicedstore):
self.server = server
# create connection object to DVID
self.rawconn = DVIDConnection(server)
# hold reference
self.dicedstore = dicedstore
self.uuid = None
# DVID version node connection
self.nodeconn = None
# all instances to ignore for ls
self.hidden_instances = None
# fetch all repo information
self.repoinfo = None
# whether current version is read only
self.locked = None
# meta for current node
self.current_node = None
# instances available at version
self.allinstances = None
self.activeinstances = None
# initialize version specific data
self._init_version(uuid)
# create meta types if not currently available
if self.MetaLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.MetaLocation)
if self.FilesLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.FilesLocation)
def test_default_user(self):
# For backwards compatibility, make sure we can create a
# DVIDNodeService without supplying a user name
connection = DVIDConnection(TEST_DVID_SERVER)
def create_label_instance(dvid_server,
uuid,
name,
levels=0,
blocksize=(64, 64, 64),
compression=Compression.DEFAULT,
enable_index=True,
typename='labelarray',
tags=[]):
"""
Create 64 bit labels data structure.
Note:
Does not check whether
the type already exists. DVIDExceptions are uncaught.
libdvid-cpp can be used directly but this also supports
setting the compression type.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
blocksize (3 int tuple): block size z,y,x
compression (Compression enum): compression to be used
enable_index: Whether or not to support indexing on this label instance
Should usually be True, except for benchmarking purposes.
typename: What instance type to create ('labelarray' or 'labelmap')
tags: A list of arbitrary strings to include in the 'Tags' field of the instance.
Returns:
True if the labels instance didn't already exist on the server
False if it already existed. (In which case this function has no effect.)
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller, except for the 'already exists' exception.
"""
conn = DVIDConnection(dvid_server)
logger.info(
"Creating {typename} instance: {uuid}/{name}".format(**locals()))
endpoint = "/repo/" + uuid + "/instance"
blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0])
data = {
"typename": typename,
"dataname": name,
"BlockSize": blockstr,
"IndexedLabels": str(enable_index).lower(),
"CountLabels": str(enable_index).lower(),
"MaxDownresLevel": str(levels)
}
if tags:
tagstr = ','.join(tags)
data["Tags"] = tagstr
if compression != Compression.DEFAULT:
data["Compression"] = compression.value
try:
conn.make_request(endpoint, ConnectionMethod.POST,
json.dumps(data).encode('utf-8'))
except DVIDException as ex:
if 'already exists' in ex.message:
pass
else:
raise
return True
