def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot): dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl') dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl') # create annotation if necessary try: dvid_node.custom_request('%s/info' % dvid_annot, None, ConnectionMethod.GET) except DVIDException as e: post_json = json.dumps({ 'typename': 'annotation', 'dataname': dvid_annot }) status, body, error_message = dvid_conn.make_request( '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST, post_json.encode('utf-8')) num_at_once = 100000 n_tot = len(tbars_json) for ii in range(0, n_tot, num_at_once): jj = np.minimum(ii + num_at_once, n_tot) data = json.dumps(tbars_json[ii:jj]) oo = dvid_node.custom_request('%s/elements' % dvid_annot, data.encode('utf-8'), ConnectionMethod.POST)
def test_get(self): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/server/info", ConnectionMethod.GET) self.assertEqual(status, 200) self.assertEqual(error_message, "") # This shouldn't raise an exception json_data = json.loads(body)
def test_garbage_request_throws(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2") try: status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET) except DVIDException as ex: assert ex.status == 404 else: assert False, "Bad requests are supposed to throw exceptions!"
def test_garbage_request(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2") status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET, checkHttpErrors=False) # No connection errors, just missing data. self.assertEqual(error_message, "") self.assertEqual(status, 404) self.assertNotEqual(body, "")
def test_garbage_request(self): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET) # No connection errors, just missing data. self.assertEqual(error_message, "") self.assertEqual(status, 404) self.assertNotEqual(body, "")
def delete_all_data_instances(uuid): connection = DVIDConnection(TEST_DVID_SERVER) repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid ) status, body, error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET) assert status == http.client.OK, "Request for {} returned status {}".format(repo_info_uri, status) assert error_message == "" repo_info = json.loads(body) for instance_name in list(repo_info["DataInstances"].keys()): status, body, error_message = connection.make_request( "/api/repo/{uuid}/{dataname}?imsure=true" .format( uuid=uuid, dataname=str(instance_name) ), ConnectionMethod.DELETE )
def delete_all_data_instances(uuid): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") repo_info_uri = "/repo/{uuid}/info".format(uuid=uuid) status, body, _error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET) repo_info = json.loads(body) for instance_name in repo_info["DataInstances"].keys(): status, body, _error_message = connection.make_request( "/repo/{uuid}/{dataname}?imsure=true".format( uuid=uuid, dataname=str(instance_name)), ConnectionMethod.DELETE, checkHttpErrors=False)
def get_metadata( hostname, uuid, data_name ): """ Query the voxels metadata for the given node/data_name. """ connection = DVIDConnection(hostname) rest_query = "/node/{uuid}/{data_name}/metadata".format( uuid=uuid, data_name=data_name ) status, response_body, _err_msg = connection.make_request( rest_query, ConnectionMethod.GET ) try: json_data = json.loads(response_body) except ValueError: raise RuntimeError("Response body could not be parsed as valid json:\n" "GET " + rest_query + "\n" + response_body) return VoxelsMetadata( json_data )
def set_syncs(dvid_server, dvid_uuid, dvid_annot, dvid_segm=None, dvid_labelsz=None): dvid_node = DVIDNodeService(dvid_server, dvid_uuid, os.getenv('USER'), 'fpl') dvid_conn = DVIDConnection(dvid_server, os.getenv('USER'), 'fpl') # create annotation if necessary try: dvid_node.custom_request('%s/info' % dvid_annot, None, ConnectionMethod.GET) except DVIDException as e: post_json = json.dumps({ 'typename': 'annotation', 'dataname': dvid_annot }) status, body, error_message = dvid_conn.make_request( '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST, post_json.encode('utf-8')) if dvid_labelsz is not None: # create labelsz if necessary try: dvid_node.custom_request('%s/info' % dvid_labelsz, None, ConnectionMethod.GET) except DVIDException as e: post_json = json.dumps({ 'typename': 'labelsz', 'dataname': dvid_labelsz }) status, body, error_message = dvid_conn.make_request( '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST, post_json.encode('utf-8')) # sync synapses to segmentation if dvid_segm is not None: syn_sync_json = json.dumps({'sync': dvid_segm}) dvid_node.custom_request('%s/sync' % dvid_annot, syn_sync_json.encode('utf-8'), ConnectionMethod.POST) # sync labelsz to synapses if dvid_labelsz is not None: lsz_sync_json = json.dumps({'sync': dvid_annot}) dvid_node.custom_request('%s/sync' % dvid_labelsz, lsz_sync_json.encode('utf-8'), ConnectionMethod.POST)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) assert status == http.client.OK, "Request for /repos/info returned status {}".format( status ) assert error_message == "" repos_info = json.loads(body) test_repos = [uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo'] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse."); return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) repos_info = json.loads(body) test_repos = [ uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo' ] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo( "testrepo", "This repo is for unit tests to use and abuse.") return str(uuid)
def _handle_new_hostname(self): """ Called by 'Connect' button. Connect to the server, download the server info and repo info, and populate the GUI widgets with the data. """ new_hostname = str(self._hostname_combobox.currentText()) if '://' in new_hostname: new_hostname = new_hostname.split('://')[1] error_msg = None self._server_info = None self._repos_info = None self._current_repo = None self._hostname = None try: # Query the server connection = DVIDConnection(new_hostname) self._server_info = ContentsBrowser._get_server_info(connection) self._repos_info = ContentsBrowser._get_repos_info(connection) self._hostname = new_hostname self._connection = connection except DVIDException as ex: error_msg = "libdvid.DVIDException: {}".format(ex.message) except ErrMsg as ex: error_msg = "libdvid.ErrMsg: {}".format(ex.message) if error_msg: QMessageBox.critical(self, "Connection Error", error_msg) self._populate_node_list(None) else: self._connect_button.setEnabled(False) self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True) enable_contents = self._repos_info is not None self._data_groupbox.setEnabled(enable_contents) self._node_groupbox.setEnabled(enable_contents) self._new_data_groupbox.setEnabled(enable_contents) self._populate_hostinfo_table() self._populate_repo_tree()
def test_zz_quickstart_usage(self): import json import numpy from libdvid import DVIDConnection, ConnectionMethod from libdvid.voxels import VoxelsAccessor, VoxelsMetadata # Open a connection to DVID connection = DVIDConnection("127.0.0.1:8000") # Get detailed dataset info: /api/repos/info (note: /api is prepended automatically) status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) dataset_details = json.loads(body) # print(json.dumps( dataset_details, indent=4 )) # Create a new remote volume (assuming you already know the uuid of the node) uuid = UUID voxels_metadata = VoxelsMetadata.create_default_metadata( (0, 0, 0, 1), numpy.uint8, 'zyxc', 1.0, "") VoxelsAccessor.create_new("127.0.0.1:8000", uuid, "my_volume", voxels_metadata) # Use the VoxelsAccessor convenience class to manipulate a particular data volume accessor = VoxelsAccessor("127.0.0.1:8000", uuid, "my_volume") # print(dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape) # Add some data (must be block-aligned) # Must include all channels. updated_data = numpy.ones((256, 192, 128, 1), dtype=numpy.uint8) accessor[256:512, 32:224, 0:128, 0] = updated_data # OR: #accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data ) # Read from it (First axis is channel.) cutout_array = accessor[300:330, 40:120, 10:110, 0] # OR: cutout_array = accessor.get_ndarray((300, 40, 10, 0), (330, 120, 110, 1)) assert isinstance(cutout_array, numpy.ndarray) assert cutout_array.shape == (30, 80, 100, 1)
def __init__(self, server, uuid, dicedstore, readonly=False): self.server = server # create connection object to DVID self.rawconn = DVIDConnection(server) # hold reference self.dicedstore = dicedstore self.uuid = None # DVID version node connection self.nodeconn = None # all instances to ignore for ls self.hidden_instances = None # fetch all repo information self.repoinfo = None # whether current version is read only self.locked = None # meta for current node self.current_node = None # instances available at version self.allinstances = None self.activeinstances = None # initialize version specific data self._init_version(uuid) # create meta types if not currently available # This operation is not available if the DVID server is running in # readonly mode, but most of this library will still work if it is. # Supplying readonly=True in the constructor will bypass this initialisation. if not readonly: if self.MetaLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.MetaLocation) if self.FilesLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.FilesLocation)
def create_rawarray8(dvid_server, uuid, name, blocksize=(64, 64, 64), compression=Compression.DEFAULT): """Create 8 bit labels data structure. Note: Currenly using uint8blk only. Does not check whether the type already exists. DVIDExceptions are uncaught. libdvid-cpp can be used directly but this also supports setting the compression type. Args: dvid_server (str): location of dvid server uuid (str): version id name (str): data instance name blocksize (3 int tuple): block size z,y,x compression (Compression enum): compression to be used minimal_extents: box [(z0,y0,x0), (z1,y1,x1)]. If provided, data extents will be at least this large (possibly larger). Raises: DVIDExceptions are not caught in this function and will be transferred to the caller. """ conn = DVIDConnection(dvid_server) typename = "uint8blk" logger.info( "Creating {typename} instance: {uuid}/{name}".format(**locals())) endpoint = "/repo/" + uuid + "/instance" blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0]) data = {"typename": typename, "dataname": name, "BlockSize": blockstr} if compression != Compression.DEFAULT: data["Compression"] = compression.value conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8'))
def test_sycns(self): """Test sync check and setting a sync. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") create_label_instance(dvidserver, uuid, "labels") # check if labels is listening to labels2 self.assertFalse(has_sync(dvidserver, uuid, "labels", "bodies")) # create labelvol and sync to it conn = DVIDConnection(dvidserver) endpoint = "/repo/" + uuid + "/instance" data = {"typename": "labelvol", "dataname": "bodies"} conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode()) set_sync(dvidserver, uuid, "labels", "bodies") self.assertTrue(has_sync(dvidserver, uuid, "labels", "bodies"))
def list_repos(self): """List all repositories in the store. Returns: A list of (name, uuid) tuples """ data = None try: conn = DVIDConnection(self._server) code, data, errmsg = conn.make_request("/repos/info", ConnectionMethod.GET) except DVIDException as err: raise DicedException("Failed to access /repos/info on DVID") jdata = json.loads(data) res = [] for _key, val in jdata.items(): res.append((str(val["Alias"]), str(val["Root"]))) return res
def __init__(self, server, uuid, dicedstore): self.server = server # create connection object to DVID self.rawconn = DVIDConnection(server) # hold reference self.dicedstore = dicedstore self.uuid = None # DVID version node connection self.nodeconn = None # all instances to ignore for ls self.hidden_instances = None # fetch all repo information self.repoinfo = None # whether current version is read only self.locked = None # meta for current node self.current_node = None # instances available at version self.allinstances = None self.activeinstances = None # initialize version specific data self._init_version(uuid) # create meta types if not currently available if self.MetaLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.MetaLocation) if self.FilesLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.FilesLocation)
def test_default_user(self): # For backwards compatibility, make sure we can create a # DVIDNodeService without supplying a user name connection = DVIDConnection(TEST_DVID_SERVER)
def create_label_instance(dvid_server, uuid, name, levels=0, blocksize=(64, 64, 64), compression=Compression.DEFAULT, enable_index=True, typename='labelarray', tags=[]): """ Create 64 bit labels data structure. Note: Does not check whether the type already exists. DVIDExceptions are uncaught. libdvid-cpp can be used directly but this also supports setting the compression type. Args: dvid_server (str): location of dvid server uuid (str): version id name (str): data instance name blocksize (3 int tuple): block size z,y,x compression (Compression enum): compression to be used enable_index: Whether or not to support indexing on this label instance Should usually be True, except for benchmarking purposes. typename: What instance type to create ('labelarray' or 'labelmap') tags: A list of arbitrary strings to include in the 'Tags' field of the instance. Returns: True if the labels instance didn't already exist on the server False if it already existed. (In which case this function has no effect.) Raises: DVIDExceptions are not caught in this function and will be transferred to the caller, except for the 'already exists' exception. """ conn = DVIDConnection(dvid_server) logger.info( "Creating {typename} instance: {uuid}/{name}".format(**locals())) endpoint = "/repo/" + uuid + "/instance" blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0]) data = { "typename": typename, "dataname": name, "BlockSize": blockstr, "IndexedLabels": str(enable_index).lower(), "CountLabels": str(enable_index).lower(), "MaxDownresLevel": str(levels) } if tags: tagstr = ','.join(tags) data["Tags"] = tagstr if compression != Compression.DEFAULT: data["Compression"] = compression.value try: conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8')) except DVIDException as ex: if 'already exists' in ex.message: pass else: raise return True