def tbars_push_dvid(tbars_json, dvid_server, dvid_uuid, dvid_annot): dvid_node = DVIDNodeService(dvid_server, dvid_uuid, 'fpl', 'fpl') dvid_conn = DVIDConnection(dvid_server, 'fpl', 'fpl') # create annotation if necessary try: dvid_node.custom_request('%s/info' % dvid_annot, None, ConnectionMethod.GET) except DVIDException as e: post_json = json.dumps({ 'typename': 'annotation', 'dataname': dvid_annot }) status, body, error_message = dvid_conn.make_request( '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST, post_json.encode('utf-8')) num_at_once = 100000 n_tot = len(tbars_json) for ii in range(0, n_tot, num_at_once): jj = np.minimum(ii + num_at_once, n_tot) data = json.dumps(tbars_json[ii:jj]) oo = dvid_node.custom_request('%s/elements' % dvid_annot, data.encode('utf-8'), ConnectionMethod.POST)
def create_label_instance(dvid_server, uuid, name, levels=0, blocksize=(64,64,64), compression=Compression.DEFAULT, enable_index=True, typename='labelarray', tags=[] ): """ Create 64 bit labels data structure. Note: Does not check whether the type already exists. DVIDExceptions are uncaught. libdvid-cpp can be used directly but this also supports setting the compression type. Args: dvid_server (str): location of dvid server uuid (str): version id name (str): data instance name blocksize (3 int tuple): block size z,y,x compression (Compression enum): compression to be used enable_index: Whether or not to support indexing on this label instance Should usually be True, except for benchmarking purposes. typename: What instance type to create ('labelarray' or 'labelmap') tags: A list of arbitrary strings to include in the 'Tags' field of the instance. Returns: True if the labels instance didn't already exist on the server False if it already existed. (In which case this function has no effect.) Raises: DVIDExceptions are not caught in this function and will be transferred to the caller, except for the 'already exists' exception. """ conn = DVIDConnection(dvid_server) logger.info("Creating {typename} instance: {uuid}/{name}".format( **locals() )) endpoint = "/repo/" + uuid + "/instance" blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0]) data = { "typename": typename, "dataname": name, "BlockSize": blockstr, "IndexedLabels": str(enable_index).lower(), "CountLabels": str(enable_index).lower(), "MaxDownresLevel": str(levels) } if tags: tagstr = ','.join(tags) data["Tags"] = tagstr if compression != Compression.DEFAULT: data["Compression"] = compression.value try: conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8')) except DVIDException as ex: if 'already exists' in ex.message: pass else: raise return True
def test_get(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") status, body, error_message = connection.make_request( "/server/info", ConnectionMethod.GET); self.assertEqual(status, 200) self.assertEqual(error_message, "") # This shouldn't raise an exception json_data = json.loads(body)
def test_get(self): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/server/info", ConnectionMethod.GET) self.assertEqual(status, 200) self.assertEqual(error_message, "") # This shouldn't raise an exception json_data = json.loads(body)
def delete_all_data_instances(uuid): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid ) status, body, _error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET) repo_info = json.loads(body) for instance_name in repo_info["DataInstances"].keys(): status, body, _error_message = connection.make_request( "/repo/{uuid}/{dataname}?imsure=true" .format( uuid=uuid, dataname=str(instance_name) ), ConnectionMethod.DELETE, checkHttpErrors=False )
def test_garbage_request(self): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET); # No connection errors, just missing data. self.assertEqual(error_message, "") self.assertEqual(status, 404) self.assertNotEqual(body, "")
def test_garbage_request(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2") status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET, checkHttpErrors=False); # No connection errors, just missing data. self.assertEqual(error_message, "") self.assertEqual(status, 404) self.assertNotEqual(body, "")
def test_garbage_request_throws(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2") try: status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET); except DVIDException as ex: assert ex.status == 404 else: assert False, "Bad requests are supposed to throw exceptions!"
def test_garbage_request_throws(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2") try: status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET) except DVIDException as ex: assert ex.status == 404 else: assert False, "Bad requests are supposed to throw exceptions!"
def test_garbage_request(self): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp2") status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET, checkHttpErrors=False) # No connection errors, just missing data. self.assertEqual(error_message, "") self.assertEqual(status, 404) self.assertNotEqual(body, "")
def test_garbage_request(self): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/does/not/exist", ConnectionMethod.GET) # No connection errors, just missing data. self.assertEqual(error_message, "") self.assertEqual(status, 404) self.assertNotEqual(body, "")
def delete_all_data_instances(uuid): connection = DVIDConnection(TEST_DVID_SERVER) repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid ) status, body, error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET) assert status == httplib.OK, "Request for {} returned status {}".format(repo_info_uri, status) assert error_message == "" repo_info = json.loads(body) for instance_name in repo_info["DataInstances"].keys(): status, body, error_message = connection.make_request( "/api/repo/{uuid}/{dataname}?imsure=true" .format( uuid=uuid, dataname=str(instance_name) ), ConnectionMethod.DELETE )
def delete_all_data_instances(uuid): connection = DVIDConnection(TEST_DVID_SERVER) repo_info_uri = "/repo/{uuid}/info".format( uuid=uuid ) status, body, error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET) assert status == http.client.OK, "Request for {} returned status {}".format(repo_info_uri, status) assert error_message == "" repo_info = json.loads(body) for instance_name in list(repo_info["DataInstances"].keys()): status, body, error_message = connection.make_request( "/api/repo/{uuid}/{dataname}?imsure=true" .format( uuid=uuid, dataname=str(instance_name) ), ConnectionMethod.DELETE )
def delete_all_data_instances(uuid): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") repo_info_uri = "/repo/{uuid}/info".format(uuid=uuid) status, body, _error_message = connection.make_request( repo_info_uri, ConnectionMethod.GET) repo_info = json.loads(body) for instance_name in repo_info["DataInstances"].keys(): status, body, _error_message = connection.make_request( "/repo/{uuid}/{dataname}?imsure=true".format( uuid=uuid, dataname=str(instance_name)), ConnectionMethod.DELETE, checkHttpErrors=False)
def get_metadata( hostname, uuid, data_name ): """ Query the voxels metadata for the given node/data_name. """ connection = DVIDConnection(hostname) rest_query = "/node/{uuid}/{data_name}/metadata".format( uuid=uuid, data_name=data_name ) status, response_body, _err_msg = connection.make_request( rest_query, ConnectionMethod.GET ) try: json_data = json.loads(response_body) except ValueError: raise RuntimeError("Response body could not be parsed as valid json:\n" "GET " + rest_query + "\n" + response_body) return VoxelsMetadata( json_data )
def set_syncs(dvid_server, dvid_uuid, dvid_annot, dvid_segm=None, dvid_labelsz=None): dvid_node = DVIDNodeService(dvid_server, dvid_uuid, os.getenv('USER'), 'fpl') dvid_conn = DVIDConnection(dvid_server, os.getenv('USER'), 'fpl') # create annotation if necessary try: dvid_node.custom_request('%s/info' % dvid_annot, None, ConnectionMethod.GET) except DVIDException as e: post_json = json.dumps({ 'typename': 'annotation', 'dataname': dvid_annot }) status, body, error_message = dvid_conn.make_request( '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST, post_json.encode('utf-8')) if dvid_labelsz is not None: # create labelsz if necessary try: dvid_node.custom_request('%s/info' % dvid_labelsz, None, ConnectionMethod.GET) except DVIDException as e: post_json = json.dumps({ 'typename': 'labelsz', 'dataname': dvid_labelsz }) status, body, error_message = dvid_conn.make_request( '/repo/%s/instance' % dvid_uuid, ConnectionMethod.POST, post_json.encode('utf-8')) # sync synapses to segmentation if dvid_segm is not None: syn_sync_json = json.dumps({'sync': dvid_segm}) dvid_node.custom_request('%s/sync' % dvid_annot, syn_sync_json.encode('utf-8'), ConnectionMethod.POST) # sync labelsz to synapses if dvid_labelsz is not None: lsz_sync_json = json.dumps({'sync': dvid_annot}) dvid_node.custom_request('%s/sync' % dvid_labelsz, lsz_sync_json.encode('utf-8'), ConnectionMethod.POST)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) assert status == http.client.OK, "Request for /repos/info returned status {}".format( status ) assert error_message == "" repos_info = json.loads(body) test_repos = [uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo'] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse."); return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) assert status == httplib.OK, "Request for /repos/info returned status {}".format( status ) assert error_message == "" repos_info = json.loads(body) test_repos = filter( lambda (uuid, repo_info): repo_info and repo_info['Alias'] == 'testrepo', repos_info.items() ) if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse."); return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) repos_info = json.loads(body) test_repos = [ uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo' ] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo( "testrepo", "This repo is for unit tests to use and abuse.") return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request("/repos/info", ConnectionMethod.GET) assert status == http.client.OK, "Request for /repos/info returned status {}".format(status) assert error_message == "" repos_info = json.loads(body) test_repos = [ uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]["Alias"] == "testrepo" ] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse.") return str(uuid)
def __init__(self, server, uuid, dicedstore, readonly=False): self.server = server # create connection object to DVID self.rawconn = DVIDConnection(server) # hold reference self.dicedstore = dicedstore self.uuid = None # DVID version node connection self.nodeconn = None # all instances to ignore for ls self.hidden_instances = None # fetch all repo information self.repoinfo = None # whether current version is read only self.locked = None # meta for current node self.current_node = None # instances available at version self.allinstances = None self.activeinstances = None # initialize version specific data self._init_version(uuid) # create meta types if not currently available # This operation is not available if the DVID server is running in # readonly mode, but most of this library will still work if it is. # Supplying readonly=True in the constructor will bypass this initialisation. if not readonly: if self.MetaLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.MetaLocation) if self.FilesLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.FilesLocation)
def test_zz_quickstart_usage(self): import json import numpy from libdvid import DVIDConnection, ConnectionMethod from libdvid.voxels import VoxelsAccessor, VoxelsMetadata # Open a connection to DVID connection = DVIDConnection("127.0.0.1:8000") # Get detailed dataset info: /api/repos/info (note: /api is prepended automatically) status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) dataset_details = json.loads(body) # print(json.dumps( dataset_details, indent=4 )) # Create a new remote volume (assuming you already know the uuid of the node) uuid = UUID voxels_metadata = VoxelsMetadata.create_default_metadata( (0, 0, 0, 1), numpy.uint8, 'zyxc', 1.0, "") VoxelsAccessor.create_new("127.0.0.1:8000", uuid, "my_volume", voxels_metadata) # Use the VoxelsAccessor convenience class to manipulate a particular data volume accessor = VoxelsAccessor("127.0.0.1:8000", uuid, "my_volume") # print(dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape) # Add some data (must be block-aligned) # Must include all channels. updated_data = numpy.ones((256, 192, 128, 1), dtype=numpy.uint8) accessor[256:512, 32:224, 0:128, 0] = updated_data # OR: #accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data ) # Read from it (First axis is channel.) cutout_array = accessor[300:330, 40:120, 10:110, 0] # OR: cutout_array = accessor.get_ndarray((300, 40, 10, 0), (330, 120, 110, 1)) assert isinstance(cutout_array, numpy.ndarray) assert cutout_array.shape == (30, 80, 100, 1)
def create_rawarray8(dvid_server, uuid, name, blocksize=(64, 64, 64), compression=Compression.DEFAULT): """Create 8 bit labels data structure. Note: Currenly using uint8blk only. Does not check whether the type already exists. DVIDExceptions are uncaught. libdvid-cpp can be used directly but this also supports setting the compression type. Args: dvid_server (str): location of dvid server uuid (str): version id name (str): data instance name blocksize (3 int tuple): block size z,y,x compression (Compression enum): compression to be used minimal_extents: box [(z0,y0,x0), (z1,y1,x1)]. If provided, data extents will be at least this large (possibly larger). Raises: DVIDExceptions are not caught in this function and will be transferred to the caller. """ conn = DVIDConnection(dvid_server) typename = "uint8blk" logger.info( "Creating {typename} instance: {uuid}/{name}".format(**locals())) endpoint = "/repo/" + uuid + "/instance" blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0]) data = {"typename": typename, "dataname": name, "BlockSize": blockstr} if compression != Compression.DEFAULT: data["Compression"] = compression.value conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8'))
def test_sycns(self): """Test sync check and setting a sync. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") create_label_instance(dvidserver, uuid, "labels") # check if labels is listening to labels2 self.assertFalse(has_sync(dvidserver, uuid, "labels", "bodies")) # create labelvol and sync to it conn = DVIDConnection(dvidserver) endpoint = "/repo/" + uuid + "/instance" data = {"typename": "labelvol", "dataname": "bodies"} conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode()) set_sync(dvidserver, uuid, "labels", "bodies") self.assertTrue(has_sync(dvidserver, uuid, "labels", "bodies"))
def list_repos(self): """List all repositories in the store. Returns: A list of (name, uuid) tuples """ data = None try: conn = DVIDConnection(self._server) code, data, errmsg = conn.make_request("/repos/info", ConnectionMethod.GET) except DVIDException as err: raise DicedException("Failed to access /repos/info on DVID") jdata = json.loads(data) res = [] for _key, val in jdata.items(): res.append((str(val["Alias"]), str(val["Root"]))) return res
def test_zz_quickstart_usage(self): import json import numpy from libdvid import DVIDConnection, ConnectionMethod from libdvid.voxels import VoxelsAccessor, VoxelsMetadata # Open a connection to DVID connection = DVIDConnection( "localhost:8000" ) # Get detailed dataset info: /api/repos/info (note: /api is prepended automatically) status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) dataset_details = json.loads(body) # print json.dumps( dataset_details, indent=4 ) # Create a new remote volume (assuming you already know the uuid of the node) uuid = UUID voxels_metadata = VoxelsMetadata.create_default_metadata( (1,0,0,0), numpy.uint8, 'cxyz', 1.0, "" ) VoxelsAccessor.create_new( "localhost:8000", uuid, "my_volume", voxels_metadata ) # Use the VoxelsAccessor convenience class to manipulate a particular data volume accessor = VoxelsAccessor( "localhost:8000", uuid, "my_volume" ) # print dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape # Add some data (must be block-aligned) # Must include all channels. # Must be FORTRAN array, using FORTRAN indexing order conventions # (Use order='F', and make sure you're indexing it as cxyz) updated_data = numpy.ones( (1,128,192,256), dtype=numpy.uint8, order='F' ) updated_data = numpy.asfortranarray(updated_data) accessor[:, 0:128, 32:224, 256:512] = updated_data # OR: #accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data ) # Read from it (First axis is channel.) cutout_array = accessor[:, 10:110, 40:120, 300:330] # OR: cutout_array = accessor.get_ndarray( (0,10,40,300), (1,110,120,330) ) assert isinstance(cutout_array, numpy.ndarray) assert cutout_array.shape == (1,100,80,30)
def test_zz_quickstart_usage(self): import json import numpy from libdvid import DVIDConnection, ConnectionMethod from libdvid.voxels import VoxelsAccessor, VoxelsMetadata # Open a connection to DVID connection = DVIDConnection( "127.0.0.1:8000" ) # Get detailed dataset info: /api/repos/info (note: /api is prepended automatically) status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) dataset_details = json.loads(body) # print(json.dumps( dataset_details, indent=4 )) # Create a new remote volume (assuming you already know the uuid of the node) uuid = UUID voxels_metadata = VoxelsMetadata.create_default_metadata( (0,0,0,1), numpy.uint8, 'zyxc', 1.0, "" ) VoxelsAccessor.create_new( "127.0.0.1:8000", uuid, "my_volume", voxels_metadata ) # Use the VoxelsAccessor convenience class to manipulate a particular data volume accessor = VoxelsAccessor( "127.0.0.1:8000", uuid, "my_volume" ) # print(dvid_volume.axiskeys, dvid_volume.dtype, dvid_volume.minindex, dvid_volume.shape) # Add some data (must be block-aligned) # Must include all channels. updated_data = numpy.ones( (256,192,128,1), dtype=numpy.uint8) accessor[256:512, 32:224, 0:128, 0] = updated_data # OR: #accessor.post_ndarray( (0,10,20,30), (1,110,120,130), updated_data ) # Read from it (First axis is channel.) cutout_array = accessor[300:330, 40:120, 10:110, 0] # OR: cutout_array = accessor.get_ndarray( (300,40,10,0), (330,120,110,1) ) assert isinstance(cutout_array, numpy.ndarray) assert cutout_array.shape == (30,80,100,1)
def __init__(self, server, uuid, dicedstore): self.server = server # create connection object to DVID self.rawconn = DVIDConnection(server) # hold reference self.dicedstore = dicedstore self.uuid = None # DVID version node connection self.nodeconn = None # all instances to ignore for ls self.hidden_instances = None # fetch all repo information self.repoinfo = None # whether current version is read only self.locked = None # meta for current node self.current_node = None # instances available at version self.allinstances = None self.activeinstances = None # initialize version specific data self._init_version(uuid) # create meta types if not currently available if self.MetaLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.MetaLocation) if self.FilesLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.FilesLocation)
def create_rawarray8(dvid_server, uuid, name, blocksize=(64,64,64), compression=Compression.DEFAULT ): """Create 8 bit labels data structure. Note: Currenly using uint8blk only. Does not check whether the type already exists. DVIDExceptions are uncaught. libdvid-cpp can be used directly but this also supports setting the compression type. Args: dvid_server (str): location of dvid server uuid (str): version id name (str): data instance name blocksize (3 int tuple): block size z,y,x compression (Compression enum): compression to be used minimal_extents: box [(z0,y0,x0), (z1,y1,x1)]. If provided, data extents will be at least this large (possibly larger). Raises: DVIDExceptions are not caught in this function and will be transferred to the caller. """ conn = DVIDConnection(dvid_server) typename = "uint8blk" logger.info("Creating {typename} instance: {uuid}/{name}".format( **locals() )) endpoint = "/repo/" + uuid + "/instance" blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0]) data = {"typename": typename, "dataname": name, "BlockSize": blockstr} if compression != Compression.DEFAULT: data["Compression"] = compression.value conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8'))
def _handle_new_hostname(self): """ Called by 'Connect' button. Connect to the server, download the server info and repo info, and populate the GUI widgets with the data. """ new_hostname = str(self._hostname_combobox.currentText()) if '://' in new_hostname: new_hostname = new_hostname.split('://')[1] error_msg = None self._server_info = None self._repos_info = None self._current_repo = None self._hostname = None try: # Query the server connection = DVIDConnection(new_hostname) self._server_info = ContentsBrowser._get_server_info(connection) self._repos_info = ContentsBrowser._get_repos_info(connection) self._hostname = new_hostname self._connection = connection except DVIDException as ex: error_msg = "libdvid.DVIDException: {}".format(ex.message) except ErrMsg as ex: error_msg = "libdvid.ErrMsg: {}".format(ex.message) if error_msg: QMessageBox.critical(self, "Connection Error", error_msg) self._populate_node_list(None) else: self._connect_button.setEnabled(False) self._buttonbox.button(QDialogButtonBox.Ok).setEnabled(True) enable_contents = self._repos_info is not None self._data_groupbox.setEnabled(enable_contents) self._node_groupbox.setEnabled(enable_contents) self._new_data_groupbox.setEnabled(enable_contents) self._populate_hostinfo_table() self._populate_repo_tree()
class DicedRepo(object): """Provides access to a version of the specified repo. Note: If the version of the repo has been locked (i.e., is not open), creating datainstance and writing data is disabled. If one wants to write to the repo, a new node should be branched. There are several several metadata conventions that are used by diced. * files are always located in '.files' * .meta contains various information about repo * .meta/restrictions: is a list of instance names to be ignored by default (set outside of diced) * .meta/instance:{instancename:datauuid}: contains '{"numdims": <num>}' * .meta/neuroglancer: contains a list of instances compatible with neuroglancer """ # block size constants BLKSIZE3D = 64 BLKSIZE2D = 512 BLKSIZE1D = 262144 # keep track of all internal DVID datatypes supported SupportedTypes = {"uint8blk" : ArrayDtype.uint8, "uint16blk": ArrayDtype.uint16, "uint32blk": ArrayDtype.uint32, "uint64blk": ArrayDtype.uint64, "labelblk": ArrayDtype.uint64} RawTypeMappings = {ArrayDtype.uint8: "uint8blk", ArrayDtype.uint16: "uint16blk", ArrayDtype.uint32: "uint32blk", ArrayDtype.uint64: "uint64blk"} LabelTypeMappings = {ArrayDtype.uint8: "labelblk", ArrayDtype.uint16: "labelblk", ArrayDtype.uint32: "labelblk", ArrayDtype.uint64: "labelblk"} LabelTypes = set(["labelblk"]) MetaLocation = ".meta" FilesLocation = ".files" RestrictionName = "restrictions" InstanceMetaPrefix = "instance:" def __init__(self, server, uuid, dicedstore): self.server = server # create connection object to DVID self.rawconn = DVIDConnection(server) # hold reference self.dicedstore = dicedstore self.uuid = None # DVID version node connection self.nodeconn = None # all instances to ignore for ls self.hidden_instances = None # fetch all repo information self.repoinfo = None # whether current version is read only self.locked = None # meta for current node self.current_node = None # instances available at version self.allinstances = None self.activeinstances = None # initialize version specific data self._init_version(uuid) # create meta types if not currently available if self.MetaLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.MetaLocation) if self.FilesLocation not in self.activeinstances: self.nodeconn.create_keyvalue(self.FilesLocation) def change_version(self, uuid): """Change the current node version. Args: uuid (str): version node to use Raises: DicedException if UUID not found """ for tuuid in self.alluuids: if tuuid.startswith(uuid): self._init_version(uuid) return raise DicedException("UUID not found") def get_current_version(self): """Retrieve the current version ID. Returns: String for UUID """ return self.uuid def get_array(self, name): """Retrive a DicedArray object. Returns: DicedArray Raises: DicedException if the array does not exist """ if name in self.activeinstances: # check if accepted type typename = self.activeinstances[name] if typename in self.SupportedTypes: # only support arrays islabel3D = typename in self.LabelTypes numdims = 3 # check if num dims specified try: datauuid = self.repoinfo["DataInstances"][name]["Base"]["DataUUID"] data = self.nodeconn.get_json(self.MetaLocation, str(self.InstanceMetaPrefix+name+":"+datauuid)) numdims = data["numdims"] except: pass return DicedArray(name, self.dicedstore, self.locked, self.nodeconn, numdims, self.SupportedTypes[typename], islabel3D) else: raise DicedException("Instance name: " + name + " has an unsupported type " + typename) raise DicedException("Instance name: " + name + " not found in version " + self.uuid) def create_array(self, name, dtype, dims=3, islabel3D=False, lossycompression=False, versioned=True): """Create a new array in the repo at this version. Args: name (str): unique name for this array dtype (ArrayDtype): datatype (not relevant if labels) dims (int): number of dimensions (support 1,2,3) islabel3D (bool): treat as label data (usually highly compressible) (always 64bit) versioned (bool): allow array to be versioned lossycompresion (bool): use lossy compression (only if not islabel3D) Returns: new DicedArray object Raises: DicedException if node is locked or name is already used. """ if self.locked: raise DicedException("Cannot create instance on locked node") for (tname, uuid) in self.allinstances: if tname == name: raise DicedException("Name already exists in repo") if islabel3D and dims != 3: raise DicedException("islabel3D only supported for 3D data") if islabel3D and dtype != ArrayDtype.uint64: raise DicedException("islabel3D only works with 64 bit data") endpoint = "/repo/" + self.uuid + "/instance" typename = self.RawTypeMappings[dtype] if islabel3D: typename = self.LabelTypeMappings[dtype] # handle blocksize blockstr = str(self.BLKSIZE3D) + "," + str(self.BLKSIZE3D) + "," + str(self.BLKSIZE3D) if dims == 2: blockstr = str(self.BLKSIZE2D) + "," + str(self.BLKSIZE2D) + "," + str(1) if dims == 1: blockstr = str(self.BLKSIZE1D) + "," + str(1) + "," + str(1) data = {"typename": typename, "dataname": name, "BlockSize": blockstr} if not islabel3D and lossycompression: data["Compression"] = "jpeg" self.rawconn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8')) # update current node meta self._init_version(self.uuid) # use '.meta' keyvalue to store array size (since not internal to DVID yet) self.nodeconn.put(self.MetaLocation, self.InstanceMetaPrefix+name+":"+str(self.repoinfo["DataInstances"][name]["Base"]["DataUUID"]), json.dumps({"numdims": dims}).encode('utf-8')) return DicedArray(name, self.dicedstore, False, self.nodeconn, dims, dtype, islabel3D) def get_commit_log(self): """Retrieve a list of commit log messages. Returns: [(str, str] array of uuid, log messages """ return self.loghistory def upload_filedata(self, dataname, data): """Upload file data to this repo version. Args: dataname (str): name of file data (str): data to store Raises: DicedException if locked """ if self.locked: raise DicedException("Node already locked") if isinstance(data, unicode): data = data.encode('utf-8') self.nodeconn.put(self.FilesLocation, dataname, data) def download_filedata(self, dataname): """Download file data. Args: dataname (str): name of file Returns: Data for file as a string Raises: DicedException if file does not exist """ data = None try: data = self.nodeconn.get(self.FilesLocation, dataname) except DVIDException: raise DicedException("file does not exist") return data def list_instances(self, showhidden=False): """Lists data store in this version of the repo. This will show all the array data. The user can choose to show everything shown in DVID. Args: showhidden (boolean): show all instances (even non-array) Return: [ (instance name, ArrayDtype) ] if showhidden is True the function returns [ (instance name, type name string) ]. """ res = [] for instance, typename in self.activeinstances.items(): if showhidden: res.append((instance, typename)) elif typename in self.SupportedTypes and instance not in self.hidden_instances: res.append((instance, self.SupportedTypes[typename])) return res def list_files(self): """List all the files for this version node. Returns: List of strings for file names """ json_text = self.nodeconn.custom_request("/" + self.FilesLocation + "/keys/0/z", b"", ConnectionMethod.GET) return json.loads(json_text) def create_branch(self, message): """Create a new branch from this locked node. Args: message (str): commit message Returns: string for new version UUID Raises: DicedExcpetion if node is not locked. """ if not self.locked: raise DicedException("Must lock node before branching") res = self.nodeconn.custom_request("/branch", json.dumps({"note": message}).encode('utf-8'), ConnectionMethod.POST) res = json.loads(res) # add new uuid (no need to reinit everything) self.alluuids.add(str(res["child"])) return str(res["child"]) def lock_node(self, message): """Lock node. Args: message (str): commit message Returns: string for new version UUID Raises: DicedExcpetion if node is already locked. """ if self.locked: raise DicedException("Node already locked") self.nodeconn.custom_request("/commit", json.dumps({"note": message}).encode('utf-8'), ConnectionMethod.POST) # no need to reinit everything self.locked = True def delete_file(self, filename): """Delete file from this version. This will only delete the file from the current version. Args: filename (str): name of file Raises: DicedExcpetion if node is already locked. """ if self.locked: raise DicedException("Node already locked") self.nodeconn.custom_request("/" + self.FilesLocation + "/key/" + filename, b"", ConnectionMethod.DELETE) def delete_array(self, dataname): """Delete array from repo (not just version!) -- this cannot be undone! Note: For large arrays this could be very time-consuming. While this is non-blocking, currently, DVID will not resume the deletion on restart, so it is possible for data to still be stored even if it is superficially removed. For now, the user should ensure DicedStore is open for a some time after issue the command. TODO: Implement a restartable background delete. """ addr = self.dicedstore._server.split(':')[0] rpcaddress = addr + ":" + str(self.dicedstore.rpcport) deletecall = subprocess.Popen(['dvid', '-rpc='+rpcaddress, 'repo', self.uuid, 'delete', dataname], stdout=None) deletecall.communicate() self._init_version(self.uuid) def _init_version(self, uuid): # create connection to repo self.nodeconn = DVIDNodeService(str(self.server), str(uuid)) # fetch all repo information status, data, errmsg = self.rawconn.make_request("/repo/" + uuid + "/info", ConnectionMethod.GET) self.repoinfo = json.loads(data) self.allinstances = {} # load all versions in repo self.alluuids = set() dag = self.repoinfo["DAG"]["Nodes"] for uuidt, nodedata in dag.items(): self.alluuids.add(str(nodedata["UUID"])) for instancename, val in self.repoinfo["DataInstances"].items(): # name is not necessarily unique to a repo self.allinstances[(str(instancename), str(val["Base"]["DataUUID"]))] = str(val["Base"]["TypeName"]) # datainstances that should be hidden (array of names) try: self.hidden_instances = set(self.nodeconn.get_json(self.MetaLocation, self.RestrictionName)) except: self.hidden_instances = set() nodeids = {} # check if locked note dag = self.repoinfo["DAG"]["Nodes"] for tuuid, nodedata in dag.items(): nodeids[str(nodedata["VersionID"])] = nodedata if tuuid.startswith(uuid): self.uuid = str(tuuid) self.current_node = nodedata self.locked = nodedata["Locked"] # load all ancestors ancestors = set() # commit history uuid, commit note in order from oldest to newest self.loghistory = [] currnode = self.current_node while True: ancestors.add(str(currnode["UUID"])) self.loghistory.append((str(currnode["UUID"]), currnode["Note"])) if len(currnode["Parents"]) > 0: currnode = nodeids[str(currnode["Parents"][0])] else: break self.loghistory.reverse() # load all instances self.activeinstances = {} if not self.locked: tempuuid = self.loghistory[-1][0] self.loghistory[-1] = (tempuuid, "(open node)") for instancename, val in self.repoinfo["DataInstances"].items(): if str(val["Base"]["RepoUUID"]) in ancestors: self.activeinstances[str(instancename)] = str(val["Base"]["TypeName"])
def test_default_user(self): # For backwards compatibility, make sure we can create a # DVIDNodeService without supplying a user name connection = DVIDConnection(TEST_DVID_SERVER)
def create_label_instance(dvid_server, uuid, name, levels=0, blocksize=(64, 64, 64), compression=Compression.DEFAULT, enable_index=True, typename='labelarray', tags=[]): """ Create 64 bit labels data structure. Note: Does not check whether the type already exists. DVIDExceptions are uncaught. libdvid-cpp can be used directly but this also supports setting the compression type. Args: dvid_server (str): location of dvid server uuid (str): version id name (str): data instance name blocksize (3 int tuple): block size z,y,x compression (Compression enum): compression to be used enable_index: Whether or not to support indexing on this label instance Should usually be True, except for benchmarking purposes. typename: What instance type to create ('labelarray' or 'labelmap') tags: A list of arbitrary strings to include in the 'Tags' field of the instance. Returns: True if the labels instance didn't already exist on the server False if it already existed. (In which case this function has no effect.) Raises: DVIDExceptions are not caught in this function and will be transferred to the caller, except for the 'already exists' exception. """ conn = DVIDConnection(dvid_server) logger.info( "Creating {typename} instance: {uuid}/{name}".format(**locals())) endpoint = "/repo/" + uuid + "/instance" blockstr = "%d,%d,%d" % (blocksize[2], blocksize[1], blocksize[0]) data = { "typename": typename, "dataname": name, "BlockSize": blockstr, "IndexedLabels": str(enable_index).lower(), "CountLabels": str(enable_index).lower(), "MaxDownresLevel": str(levels) } if tags: tagstr = ','.join(tags) data["Tags"] = tagstr if compression != Compression.DEFAULT: data["Compression"] = compression.value try: conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode('utf-8')) except DVIDException as ex: if 'already exists' in ex.message: pass else: raise return True