def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl')
dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
num_at_once = 100000
n_tot = len(tbars_json)
for ii in range(0, n_tot, num_at_once):
jj = np.minimum(ii + num_at_once, n_tot)
data = json.dumps(tbars_json[ii:jj])
oo = dvid_node.custom_request('%s/elements' % dvid_annot,
data.encode('utf-8'),
ConnectionMethod.POST)
def create_label_instance(dvid_server, uuid, name, levels=0, blocksize=(64,64,64),
compression=Compression.DEFAULT, enable_index=True, typename='labelarray', tags=[] ):
"""
Create 64 bit labels data structure.
Note:
Does not check whether
the type already exists. DVIDExceptions are uncaught.
libdvid-cpp can be used directly but this also supports
setting the compression type.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
blocksize (3 int tuple): block size z,y,x
compression (Compression enum): compression to be used
enable_index: Whether or not to support indexing on this label instance
Should usually be True, except for benchmarking purposes.
typename: What instance type to create ('labelarray' or 'labelmap')
tags: A list of arbitrary strings to include in the 'Tags' field of the instance.
Returns:
True if the labels instance didn't already exist on the server
False if it already existed. (In which case this function has no effect.)
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller, except for the 'already exists' exception.
"""
conn = DVIDConnection(dvid_server)
logger.info("Creating {typename} instance: {uuid}/{name}".format( **locals() ))
endpoint = "/repo/" + uuid + "/instance"
blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0])
data = { "typename": typename,
"dataname": name,
"BlockSize": blockstr,
"IndexedLabels": str(enable_index).lower(),
"CountLabels": str(enable_index).lower(),
"MaxDownresLevel": str(levels) }
if tags:
tagstr = ','.join(tags)
data["Tags"] = tagstr
if compression != Compression.DEFAULT:
data["Compression"] = compression.value
try:
conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8'))
except DVIDException as ex:
if 'already exists' in ex.message:
pass
else:
raise
return True
def test_get(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp")
status, body, error_message = connection.make_request( "/server/info", ConnectionMethod.GET);
self.assertEqual(status, 200)
self.assertEqual(error_message, "")
# This shouldn't raise an exception
json_data = json.loads(body)
def test_get(self):
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request(
"/server/info", ConnectionMethod.GET)
self.assertEqual(status, 200)
self.assertEqual(error_message, "")
# This shouldn't raise an exception
json_data = json.loads(body)
def delete_all_data_instances(uuid):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp")
repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid )
status, body, _error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET)
repo_info = json.loads(body)
for instance_name in repo_info["DataInstances"].keys():
status, body, _error_message = connection.make_request( "/repo/{uuid}/{dataname}?imsure=true"
.format( uuid=uuid, dataname=str(instance_name) ),
ConnectionMethod.DELETE, checkHttpErrors=False )
def test_garbage_request(self):
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET);
# No connection errors, just missing data.
self.assertEqual(error_message, "")
self.assertEqual(status, 404)
self.assertNotEqual(body, "")
def test_garbage_request(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2")
status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET, checkHttpErrors=False);
# No connection errors, just missing data.
self.assertEqual(error_message, "")
self.assertEqual(status, 404)
self.assertNotEqual(body, "")
def test_garbage_request_throws(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2")
try:
status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET);
except DVIDException as ex:
assert ex.status == 404
else:
assert False, "Bad requests are supposed to throw exceptions!"
def test_garbage_request_throws(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2")
try:
status, body, error_message = connection.make_request(
"/does/not/exist", ConnectionMethod.GET)
except DVIDException as ex:
assert ex.status == 404
else:
assert False, "Bad requests are supposed to throw exceptions!"
def test_garbage_request(self):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2")
status, body, error_message = connection.make_request(
"/does/not/exist", ConnectionMethod.GET, checkHttpErrors=False)
# No connection errors, just missing data.
self.assertEqual(error_message, "")
self.assertEqual(status, 404)
self.assertNotEqual(body, "")
def test_garbage_request(self):
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request(
"/does/not/exist", ConnectionMethod.GET)
# No connection errors, just missing data.
self.assertEqual(error_message, "")
self.assertEqual(status, 404)
self.assertNotEqual(body, "")
def delete_all_data_instances(uuid):
connection = DVIDConnection(TEST_DVID_SERVER)
repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid )
status, body, error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET)
assert status == httplib.OK, "Request for {} returned status {}".format(repo_info_uri, status)
assert error_message == ""
repo_info = json.loads(body)
for instance_name in repo_info["DataInstances"].keys():
status, body, error_message = connection.make_request( "/api/repo/{uuid}/{dataname}?imsure=true"
.format( uuid=uuid, dataname=str(instance_name) ),
ConnectionMethod.DELETE )
def delete_all_data_instances(uuid):
connection = DVIDConnection(TEST_DVID_SERVER)
repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid )
status, body, error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET)
assert status == http.client.OK, "Request for {} returned status {}".format(repo_info_uri, status)
assert error_message == ""
repo_info = json.loads(body)
for instance_name in list(repo_info["DataInstances"].keys()):
status, body, error_message = connection.make_request( "/api/repo/{uuid}/{dataname}?imsure=true"
.format( uuid=uuid, dataname=str(instance_name) ),
ConnectionMethod.DELETE )
def delete_all_data_instances(uuid):
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp")
repo_info_uri = "/repo/{uuid}/info".format(uuid=uuid)
status, body, _error_message = connection.make_request(
repo_info_uri, ConnectionMethod.GET)
repo_info = json.loads(body)
for instance_name in repo_info["DataInstances"].keys():
status, body, _error_message = connection.make_request(
"/repo/{uuid}/{dataname}?imsure=true".format(
uuid=uuid, dataname=str(instance_name)),
ConnectionMethod.DELETE,
checkHttpErrors=False)
def get_metadata( hostname, uuid, data_name ):
"""
Query the voxels metadata for the given node/data_name.
"""
connection = DVIDConnection(hostname)
rest_query = "/node/{uuid}/{data_name}/metadata".format( uuid=uuid, data_name=data_name )
status, response_body, _err_msg = connection.make_request( rest_query, ConnectionMethod.GET )
try:
json_data = json.loads(response_body)
except ValueError:
raise RuntimeError("Response body could not be parsed as valid json:\n"
"GET " + rest_query + "\n" + response_body)
return VoxelsMetadata( json_data )
def set_syncs(dvid_server,
dvid_uuid,
dvid_annot,
dvid_segm=None,
dvid_labelsz=None):
dvid_node = DVIDNodeService(dvid_server, dvid_uuid, os.getenv('USER'),
'fpl')
dvid_conn = DVIDConnection(dvid_server, os.getenv('USER'), 'fpl')
# create annotation if necessary
try:
dvid_node.custom_request('%s/info' % dvid_annot, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'annotation',
'dataname': dvid_annot
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
if dvid_labelsz is not None:
# create labelsz if necessary
try:
dvid_node.custom_request('%s/info' % dvid_labelsz, None,
ConnectionMethod.GET)
except DVIDException as e:
post_json = json.dumps({
'typename': 'labelsz',
'dataname': dvid_labelsz
})
status, body, error_message = dvid_conn.make_request(
'/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST,
post_json.encode('utf-8'))
# sync synapses to segmentation
if dvid_segm is not None:
syn_sync_json = json.dumps({'sync': dvid_segm})
dvid_node.custom_request('%s/sync' % dvid_annot,
syn_sync_json.encode('utf-8'),
ConnectionMethod.POST)
# sync labelsz to synapses
if dvid_labelsz is not None:
lsz_sync_json = json.dumps({'sync': dvid_annot})
dvid_node.custom_request('%s/sync' % dvid_labelsz,
lsz_sync_json.encode('utf-8'),
ConnectionMethod.POST)
def get_testrepo_root_uuid():
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET)
assert status == http.client.OK, "Request for /repos/info returned status {}".format( status )
assert error_message == ""
repos_info = json.loads(body)
test_repos = [uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo']
if test_repos:
uuid = test_repos[0][0]
return str(uuid)
else:
from libdvid import DVIDServerService
server = DVIDServerService(TEST_DVID_SERVER)
uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse.");
return str(uuid)
def get_testrepo_root_uuid():
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET)
assert status == httplib.OK, "Request for /repos/info returned status {}".format( status )
assert error_message == ""
repos_info = json.loads(body)
test_repos = filter( lambda (uuid, repo_info): repo_info and repo_info['Alias'] == 'testrepo',
repos_info.items() )
if test_repos:
uuid = test_repos[0][0]
return str(uuid)
else:
from libdvid import DVIDServerService
server = DVIDServerService(TEST_DVID_SERVER)
uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse.");
return str(uuid)
def get_testrepo_root_uuid():
connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp")
status, body, _error_message = connection.make_request(
"/repos/info", ConnectionMethod.GET)
repos_info = json.loads(body)
test_repos = [
uuid_repo_info for uuid_repo_info in list(repos_info.items())
if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo'
]
if test_repos:
uuid = test_repos[0][0]
return str(uuid)
else:
from libdvid import DVIDServerService
server = DVIDServerService(TEST_DVID_SERVER)
uuid = server.create_new_repo(
"testrepo", "This repo is for unit tests to use and abuse.")
return str(uuid)
def get_testrepo_root_uuid():
connection = DVIDConnection(TEST_DVID_SERVER)
status, body, error_message = connection.make_request("/repos/info", ConnectionMethod.GET)
assert status == http.client.OK, "Request for /repos/info returned status {}".format(status)
assert error_message == ""
repos_info = json.loads(body)
test_repos = [
uuid_repo_info
for uuid_repo_info in list(repos_info.items())
if uuid_repo_info[1] and uuid_repo_info[1]["Alias"] == "testrepo"
]
if test_repos:
uuid = test_repos[0][0]
return str(uuid)
else:
from libdvid import DVIDServerService
server = DVIDServerService(TEST_DVID_SERVER)
uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse.")
return str(uuid)
def __init__(self, server, uuid, dicedstore, readonly=False):
self.server = server
# create connection object to DVID
self.rawconn = DVIDConnection(server)
# hold reference
self.dicedstore = dicedstore
self.uuid = None
# DVID version node connection
self.nodeconn = None
# all instances to ignore for ls
self.hidden_instances = None
# fetch all repo information
self.repoinfo = None
# whether current version is read only
self.locked = None
# meta for current node
self.current_node = None
# instances available at version
self.allinstances = None
self.activeinstances = None
# initialize version specific data
self._init_version(uuid)
# create meta types if not currently available
# This operation is not available if the DVID server is running in
# readonly mode, but most of this library will still work if it is.
# Supplying readonly=True in the constructor will bypass this initialisation.
if not readonly:
if self.MetaLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.MetaLocation)
if self.FilesLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.FilesLocation)
def test_zz_quickstart_usage(self):
import json
import numpy
from libdvid import DVIDConnection, ConnectionMethod
from libdvid.voxels import VoxelsAccessor, VoxelsMetadata
# Open a connection to DVID
connection = DVIDConnection("127.0.0.1:8000")
# Get detailed dataset info: /api/repos/info (note: /api is prepended automatically)
status, body, _error_message = connection.make_request(
"/repos/info", ConnectionMethod.GET)
dataset_details = json.loads(body)
# print(json.dumps( dataset_details, indent=4 ))
# Create a new remote volume (assuming you already know the uuid of the node)
uuid = UUID
voxels_metadata = VoxelsMetadata.create_default_metadata(
(0, 0, 0, 1), numpy.uint8, 'zyxc', 1.0, "")
VoxelsAccessor.create_new("127.0.0.1:8000", uuid, "my_volume",
voxels_metadata)
# Use the VoxelsAccessor convenience class to manipulate a particular data volume
accessor = VoxelsAccessor("127.0.0.1:8000", uuid, "my_volume")
# print(dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape)
# Add some data (must be block-aligned)
# Must include all channels.
updated_data = numpy.ones((256, 192, 128, 1), dtype=numpy.uint8)
accessor[256:512, 32:224, 0:128, 0] = updated_data
# OR:
#accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data )
# Read from it (First axis is channel.)
cutout_array = accessor[300:330, 40:120, 10:110, 0]
# OR:
cutout_array = accessor.get_ndarray((300, 40, 10, 0),
(330, 120, 110, 1))
assert isinstance(cutout_array, numpy.ndarray)
assert cutout_array.shape == (30, 80, 100, 1)
def create_rawarray8(dvid_server,
uuid,
name,
blocksize=(64, 64, 64),
compression=Compression.DEFAULT):
"""Create 8 bit labels data structure.
Note:
Currenly using uint8blk only. Does not check whether
the type already exists. DVIDExceptions are uncaught.
libdvid-cpp can be used directly but this also supports
setting the compression type.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
blocksize (3 int tuple): block size z,y,x
compression (Compression enum): compression to be used
minimal_extents: box [(z0,y0,x0), (z1,y1,x1)].
If provided, data extents will be at least this large (possibly larger).
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
conn = DVIDConnection(dvid_server)
typename = "uint8blk"
logger.info(
"Creating {typename} instance: {uuid}/{name}".format(**locals()))
endpoint = "/repo/" + uuid + "/instance"
blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0])
data = {"typename": typename, "dataname": name, "BlockSize": blockstr}
if compression != Compression.DEFAULT:
data["Compression"] = compression.value
conn.make_request(endpoint, ConnectionMethod.POST,
json.dumps(data).encode('utf-8'))
def test_sycns(self):
"""Test sync check and setting a sync.
"""
service = DVIDServerService(dvidserver)
uuid = service.create_new_repo("foo", "bar")
create_label_instance(dvidserver, uuid, "labels")
# check if labels is listening to labels2
self.assertFalse(has_sync(dvidserver, uuid, "labels", "bodies"))
# create labelvol and sync to it
conn = DVIDConnection(dvidserver)
endpoint = "/repo/" + uuid + "/instance"
data = {"typename": "labelvol", "dataname": "bodies"}
conn.make_request(endpoint, ConnectionMethod.POST,
json.dumps(data).encode())
set_sync(dvidserver, uuid, "labels", "bodies")
self.assertTrue(has_sync(dvidserver, uuid, "labels", "bodies"))
def list_repos(self):
"""List all repositories in the store.
Returns:
A list of (name, uuid) tuples
"""
data = None
try:
conn = DVIDConnection(self._server)
code, data, errmsg = conn.make_request("/repos/info",
ConnectionMethod.GET)
except DVIDException as err:
raise DicedException("Failed to access /repos/info on DVID")
jdata = json.loads(data)
res = []
for _key, val in jdata.items():
res.append((str(val["Alias"]), str(val["Root"])))
return res
def test_zz_quickstart_usage(self):
import json
import numpy
from libdvid import DVIDConnection, ConnectionMethod
from libdvid.voxels import VoxelsAccessor, VoxelsMetadata
# Open a connection to DVID
connection = DVIDConnection( "localhost:8000" )
# Get detailed dataset info: /api/repos/info (note: /api is prepended automatically)
status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET)
dataset_details = json.loads(body)
# print json.dumps( dataset_details, indent=4 )
# Create a new remote volume (assuming you already know the uuid of the node)
uuid = UUID
voxels_metadata = VoxelsMetadata.create_default_metadata( (1,0,0,0), numpy.uint8, 'cxyz', 1.0, "" )
VoxelsAccessor.create_new( "localhost:8000", uuid, "my_volume", voxels_metadata )
# Use the VoxelsAccessor convenience class to manipulate a particular data volume
accessor = VoxelsAccessor( "localhost:8000", uuid, "my_volume" )
# print dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape
# Add some data (must be block-aligned)
# Must include all channels.
# Must be FORTRAN array, using FORTRAN indexing order conventions
# (Use order='F', and make sure you're indexing it as cxyz)
updated_data = numpy.ones( (1,128,192,256), dtype=numpy.uint8, order='F' )
updated_data = numpy.asfortranarray(updated_data)
accessor[:, 0:128, 32:224, 256:512] = updated_data
# OR:
#accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data )
# Read from it (First axis is channel.)
cutout_array = accessor[:, 10:110, 40:120, 300:330]
# OR:
cutout_array = accessor.get_ndarray( (0,10,40,300), (1,110,120,330) )
assert isinstance(cutout_array, numpy.ndarray)
assert cutout_array.shape == (1,100,80,30)
def test_zz_quickstart_usage(self):
import json
import numpy
from libdvid import DVIDConnection, ConnectionMethod
from libdvid.voxels import VoxelsAccessor, VoxelsMetadata
# Open a connection to DVID
connection = DVIDConnection( "127.0.0.1:8000" )
# Get detailed dataset info: /api/repos/info (note: /api is prepended automatically)
status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET)
dataset_details = json.loads(body)
# print(json.dumps( dataset_details, indent=4 ))
# Create a new remote volume (assuming you already know the uuid of the node)
uuid = UUID
voxels_metadata = VoxelsMetadata.create_default_metadata( (0,0,0,1), numpy.uint8, 'zyxc', 1.0, "" )
VoxelsAccessor.create_new( "127.0.0.1:8000", uuid, "my_volume", voxels_metadata )
# Use the VoxelsAccessor convenience class to manipulate a particular data volume
accessor = VoxelsAccessor( "127.0.0.1:8000", uuid, "my_volume" )
# print(dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape)
# Add some data (must be block-aligned)
# Must include all channels.
updated_data = numpy.ones( (256,192,128,1), dtype=numpy.uint8)
accessor[256:512, 32:224, 0:128, 0] = updated_data
# OR:
#accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data )
# Read from it (First axis is channel.)
cutout_array = accessor[300:330, 40:120, 10:110, 0]
# OR:
cutout_array = accessor.get_ndarray( (300,40,10,0), (330,120,110,1) )
assert isinstance(cutout_array, numpy.ndarray)
assert cutout_array.shape == (30,80,100,1)
def __init__(self, server, uuid, dicedstore):
self.server = server
# create connection object to DVID
self.rawconn = DVIDConnection(server)
# hold reference
self.dicedstore = dicedstore
self.uuid = None
# DVID version node connection
self.nodeconn = None
# all instances to ignore for ls
self.hidden_instances = None
# fetch all repo information
self.repoinfo = None
# whether current version is read only
self.locked = None
# meta for current node
self.current_node = None
# instances available at version
self.allinstances = None
self.activeinstances = None
# initialize version specific data
self._init_version(uuid)
# create meta types if not currently available
if self.MetaLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.MetaLocation)
if self.FilesLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.FilesLocation)
def create_rawarray8(dvid_server, uuid, name, blocksize=(64,64,64),
compression=Compression.DEFAULT ):
"""Create 8 bit labels data structure.
Note:
Currenly using uint8blk only. Does not check whether
the type already exists. DVIDExceptions are uncaught.
libdvid-cpp can be used directly but this also supports
setting the compression type.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
blocksize (3 int tuple): block size z,y,x
compression (Compression enum): compression to be used
minimal_extents: box [(z0,y0,x0), (z1,y1,x1)].
If provided, data extents will be at least this large (possibly larger).
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller.
"""
conn = DVIDConnection(dvid_server)
typename = "uint8blk"
logger.info("Creating {typename} instance: {uuid}/{name}".format( **locals() ))
endpoint = "/repo/" + uuid + "/instance"
blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0])
data = {"typename": typename, "dataname": name, "BlockSize": blockstr}
if compression != Compression.DEFAULT:
data["Compression"] = compression.value
conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8'))
def _handle_new_hostname(self):
"""
Called by 'Connect' button.
Connect to the server, download the server info and repo info,
and populate the GUI widgets with the data.
"""
new_hostname = str(self._hostname_combobox.currentText())
if '://' in new_hostname:
new_hostname = new_hostname.split('://')[1]
error_msg = None
self._server_info = None
self._repos_info = None
self._current_repo = None
self._hostname = None
try:
# Query the server
connection = DVIDConnection(new_hostname)
self._server_info = ContentsBrowser._get_server_info(connection)
self._repos_info = ContentsBrowser._get_repos_info(connection)
self._hostname = new_hostname
self._connection = connection
except DVIDException as ex:
error_msg = "libdvid.DVIDException: {}".format(ex.message)
except ErrMsg as ex:
error_msg = "libdvid.ErrMsg: {}".format(ex.message)
if error_msg:
QMessageBox.critical(self, "Connection Error", error_msg)
self._populate_node_list(None)
else:
self._connect_button.setEnabled(False)
self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True)
enable_contents = self._repos_info is not None
self._data_groupbox.setEnabled(enable_contents)
self._node_groupbox.setEnabled(enable_contents)
self._new_data_groupbox.setEnabled(enable_contents)
self._populate_hostinfo_table()
self._populate_repo_tree()
class DicedRepo(object):
"""Provides access to a version of the specified repo.
Note:
If the version of the repo has been locked (i.e., is not open),
creating datainstance and writing data is disabled. If one
wants to write to the repo, a new node should be branched.
There are several several metadata conventions that are
used by diced.
* files are always located in '.files'
* .meta contains various information about repo
* .meta/restrictions: is a list of instance names to be ignored by default (set outside of diced)
* .meta/instance:{instancename:datauuid}: contains '{"numdims": <num>}'
* .meta/neuroglancer: contains a list of instances compatible with neuroglancer
"""
# block size constants
BLKSIZE3D = 64
BLKSIZE2D = 512
BLKSIZE1D = 262144
# keep track of all internal DVID datatypes supported
SupportedTypes = {"uint8blk" : ArrayDtype.uint8, "uint16blk": ArrayDtype.uint16,
"uint32blk": ArrayDtype.uint32, "uint64blk": ArrayDtype.uint64,
"labelblk": ArrayDtype.uint64}
RawTypeMappings = {ArrayDtype.uint8: "uint8blk", ArrayDtype.uint16: "uint16blk",
ArrayDtype.uint32: "uint32blk", ArrayDtype.uint64: "uint64blk"}
LabelTypeMappings = {ArrayDtype.uint8: "labelblk", ArrayDtype.uint16: "labelblk",
ArrayDtype.uint32: "labelblk", ArrayDtype.uint64: "labelblk"}
LabelTypes = set(["labelblk"])
MetaLocation = ".meta"
FilesLocation = ".files"
RestrictionName = "restrictions"
InstanceMetaPrefix = "instance:"
def __init__(self, server, uuid, dicedstore):
self.server = server
# create connection object to DVID
self.rawconn = DVIDConnection(server)
# hold reference
self.dicedstore = dicedstore
self.uuid = None
# DVID version node connection
self.nodeconn = None
# all instances to ignore for ls
self.hidden_instances = None
# fetch all repo information
self.repoinfo = None
# whether current version is read only
self.locked = None
# meta for current node
self.current_node = None
# instances available at version
self.allinstances = None
self.activeinstances = None
# initialize version specific data
self._init_version(uuid)
# create meta types if not currently available
if self.MetaLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.MetaLocation)
if self.FilesLocation not in self.activeinstances:
self.nodeconn.create_keyvalue(self.FilesLocation)
def change_version(self, uuid):
"""Change the current node version.
Args:
uuid (str): version node to use
Raises:
DicedException if UUID not found
"""
for tuuid in self.alluuids:
if tuuid.startswith(uuid):
self._init_version(uuid)
return
raise DicedException("UUID not found")
def get_current_version(self):
"""Retrieve the current version ID.
Returns:
String for UUID
"""
return self.uuid
def get_array(self, name):
"""Retrive a DicedArray object.
Returns:
DicedArray
Raises:
DicedException if the array does not exist
"""
if name in self.activeinstances:
# check if accepted type
typename = self.activeinstances[name]
if typename in self.SupportedTypes: # only support arrays
islabel3D = typename in self.LabelTypes
numdims = 3
# check if num dims specified
try:
datauuid = self.repoinfo["DataInstances"][name]["Base"]["DataUUID"]
data = self.nodeconn.get_json(self.MetaLocation, str(self.InstanceMetaPrefix+name+":"+datauuid))
numdims = data["numdims"]
except:
pass
return DicedArray(name, self.dicedstore, self.locked, self.nodeconn,
numdims, self.SupportedTypes[typename], islabel3D)
else:
raise DicedException("Instance name: " + name + " has an unsupported type " + typename)
raise DicedException("Instance name: " + name + " not found in version " + self.uuid)
def create_array(self, name, dtype, dims=3, islabel3D=False,
lossycompression=False, versioned=True):
"""Create a new array in the repo at this version.
Args:
name (str): unique name for this array
dtype (ArrayDtype): datatype (not relevant if labels)
dims (int): number of dimensions (support 1,2,3)
islabel3D (bool): treat as label data (usually highly compressible) (always 64bit)
versioned (bool): allow array to be versioned
lossycompresion (bool): use lossy compression (only if not islabel3D)
Returns:
new DicedArray object
Raises:
DicedException if node is locked or name is already used.
"""
if self.locked:
raise DicedException("Cannot create instance on locked node")
for (tname, uuid) in self.allinstances:
if tname == name:
raise DicedException("Name already exists in repo")
if islabel3D and dims != 3:
raise DicedException("islabel3D only supported for 3D data")
if islabel3D and dtype != ArrayDtype.uint64:
raise DicedException("islabel3D only works with 64 bit data")
endpoint = "/repo/" + self.uuid + "/instance"
typename = self.RawTypeMappings[dtype]
if islabel3D:
typename = self.LabelTypeMappings[dtype]
# handle blocksize
blockstr = str(self.BLKSIZE3D) + "," + str(self.BLKSIZE3D) + "," + str(self.BLKSIZE3D)
if dims == 2:
blockstr = str(self.BLKSIZE2D) + "," + str(self.BLKSIZE2D) + "," + str(1)
if dims == 1:
blockstr = str(self.BLKSIZE1D) + "," + str(1) + "," + str(1)
data = {"typename": typename, "dataname": name, "BlockSize": blockstr}
if not islabel3D and lossycompression:
data["Compression"] = "jpeg"
self.rawconn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8'))
# update current node meta
self._init_version(self.uuid)
# use '.meta' keyvalue to store array size (since not internal to DVID yet)
self.nodeconn.put(self.MetaLocation, self.InstanceMetaPrefix+name+":"+str(self.repoinfo["DataInstances"][name]["Base"]["DataUUID"]), json.dumps({"numdims": dims}).encode('utf-8'))
return DicedArray(name, self.dicedstore, False, self.nodeconn,
dims, dtype, islabel3D)
def get_commit_log(self):
"""Retrieve a list of commit log messages.
Returns:
[(str, str] array of uuid, log messages
"""
return self.loghistory
def upload_filedata(self, dataname, data):
"""Upload file data to this repo version.
Args:
dataname (str): name of file
data (str): data to store
Raises:
DicedException if locked
"""
if self.locked:
raise DicedException("Node already locked")
if isinstance(data, unicode):
data = data.encode('utf-8')
self.nodeconn.put(self.FilesLocation, dataname, data)
def download_filedata(self, dataname):
"""Download file data.
Args:
dataname (str): name of file
Returns:
Data for file as a string
Raises:
DicedException if file does not exist
"""
data = None
try:
data = self.nodeconn.get(self.FilesLocation, dataname)
except DVIDException:
raise DicedException("file does not exist")
return data
def list_instances(self, showhidden=False):
"""Lists data store in this version of the repo.
This will show all the array data. The user can
choose to show everything shown in DVID.
Args:
showhidden (boolean): show all instances (even non-array)
Return:
[ (instance name, ArrayDtype) ] if showhidden is True
the function returns [ (instance name, type name string) ].
"""
res = []
for instance, typename in self.activeinstances.items():
if showhidden:
res.append((instance, typename))
elif typename in self.SupportedTypes and instance not in self.hidden_instances:
res.append((instance, self.SupportedTypes[typename]))
return res
def list_files(self):
"""List all the files for this version node.
Returns:
List of strings for file names
"""
json_text = self.nodeconn.custom_request("/" + self.FilesLocation + "/keys/0/z", b"",
ConnectionMethod.GET)
return json.loads(json_text)
def create_branch(self, message):
"""Create a new branch from this locked node.
Args:
message (str): commit message
Returns:
string for new version UUID
Raises:
DicedExcpetion if node is not locked.
"""
if not self.locked:
raise DicedException("Must lock node before branching")
res = self.nodeconn.custom_request("/branch",
json.dumps({"note": message}).encode('utf-8'),
ConnectionMethod.POST)
res = json.loads(res)
# add new uuid (no need to reinit everything)
self.alluuids.add(str(res["child"]))
return str(res["child"])
def lock_node(self, message):
"""Lock node.
Args:
message (str): commit message
Returns:
string for new version UUID
Raises:
DicedExcpetion if node is already locked.
"""
if self.locked:
raise DicedException("Node already locked")
self.nodeconn.custom_request("/commit",
json.dumps({"note": message}).encode('utf-8'),
ConnectionMethod.POST)
# no need to reinit everything
self.locked = True
def delete_file(self, filename):
"""Delete file from this version.
This will only delete the file from the current version.
Args:
filename (str): name of file
Raises:
DicedExcpetion if node is already locked.
"""
if self.locked:
raise DicedException("Node already locked")
self.nodeconn.custom_request("/" + self.FilesLocation + "/key/" + filename,
b"", ConnectionMethod.DELETE)
def delete_array(self, dataname):
"""Delete array from repo (not just version!) -- this cannot be undone!
Note:
For large arrays this could be very time-consuming.
While this is non-blocking, currently, DVID
will not resume the deletion on restart, so it is
possible for data to still be stored even if it
is superficially removed. For now, the user should
ensure DicedStore is open for a some time after
issue the command.
TODO:
Implement a restartable background delete.
"""
addr = self.dicedstore._server.split(':')[0]
rpcaddress = addr + ":" + str(self.dicedstore.rpcport)
deletecall = subprocess.Popen(['dvid', '-rpc='+rpcaddress, 'repo', self.uuid, 'delete', dataname], stdout=None)
deletecall.communicate()
self._init_version(self.uuid)
def _init_version(self, uuid):
# create connection to repo
self.nodeconn = DVIDNodeService(str(self.server), str(uuid))
# fetch all repo information
status, data, errmsg = self.rawconn.make_request("/repo/" + uuid + "/info", ConnectionMethod.GET)
self.repoinfo = json.loads(data)
self.allinstances = {}
# load all versions in repo
self.alluuids = set()
dag = self.repoinfo["DAG"]["Nodes"]
for uuidt, nodedata in dag.items():
self.alluuids.add(str(nodedata["UUID"]))
for instancename, val in self.repoinfo["DataInstances"].items():
# name is not necessarily unique to a repo
self.allinstances[(str(instancename), str(val["Base"]["DataUUID"]))] = str(val["Base"]["TypeName"])
# datainstances that should be hidden (array of names)
try:
self.hidden_instances = set(self.nodeconn.get_json(self.MetaLocation, self.RestrictionName))
except:
self.hidden_instances = set()
nodeids = {}
# check if locked note
dag = self.repoinfo["DAG"]["Nodes"]
for tuuid, nodedata in dag.items():
nodeids[str(nodedata["VersionID"])] = nodedata
if tuuid.startswith(uuid):
self.uuid = str(tuuid)
self.current_node = nodedata
self.locked = nodedata["Locked"]
# load all ancestors
ancestors = set()
# commit history uuid, commit note in order from oldest to newest
self.loghistory = []
currnode = self.current_node
while True:
ancestors.add(str(currnode["UUID"]))
self.loghistory.append((str(currnode["UUID"]), currnode["Note"]))
if len(currnode["Parents"]) > 0:
currnode = nodeids[str(currnode["Parents"][0])]
else:
break
self.loghistory.reverse()
# load all instances
self.activeinstances = {}
if not self.locked:
tempuuid = self.loghistory[-1][0]
self.loghistory[-1] = (tempuuid, "(open node)")
for instancename, val in self.repoinfo["DataInstances"].items():
if str(val["Base"]["RepoUUID"]) in ancestors:
self.activeinstances[str(instancename)] = str(val["Base"]["TypeName"])
def test_default_user(self):
# For backwards compatibility, make sure we can create a
# DVIDNodeService without supplying a user name
connection = DVIDConnection(TEST_DVID_SERVER)
def create_label_instance(dvid_server,
uuid,
name,
levels=0,
blocksize=(64, 64, 64),
compression=Compression.DEFAULT,
enable_index=True,
typename='labelarray',
tags=[]):
"""
Create 64 bit labels data structure.
Note:
Does not check whether
the type already exists. DVIDExceptions are uncaught.
libdvid-cpp can be used directly but this also supports
setting the compression type.
Args:
dvid_server (str): location of dvid server
uuid (str): version id
name (str): data instance name
blocksize (3 int tuple): block size z,y,x
compression (Compression enum): compression to be used
enable_index: Whether or not to support indexing on this label instance
Should usually be True, except for benchmarking purposes.
typename: What instance type to create ('labelarray' or 'labelmap')
tags: A list of arbitrary strings to include in the 'Tags' field of the instance.
Returns:
True if the labels instance didn't already exist on the server
False if it already existed. (In which case this function has no effect.)
Raises:
DVIDExceptions are not caught in this function and will be
transferred to the caller, except for the 'already exists' exception.
"""
conn = DVIDConnection(dvid_server)
logger.info(
"Creating {typename} instance: {uuid}/{name}".format(**locals()))
endpoint = "/repo/" + uuid + "/instance"
blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0])
data = {
"typename": typename,
"dataname": name,
"BlockSize": blockstr,
"IndexedLabels": str(enable_index).lower(),
"CountLabels": str(enable_index).lower(),
"MaxDownresLevel": str(levels)
}
if tags:
tagstr = ','.join(tags)
data["Tags"] = tagstr
if compression != Compression.DEFAULT:
data["Compression"] = compression.value
try:
conn.make_request(endpoint, ConnectionMethod.POST,
json.dumps(data).encode('utf-8'))
except DVIDException as ex:
if 'already exists' in ex.message:
pass
else:
raise
return True
