def test_isdvidversion(self): """Tests is_dvidversion function. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") self.assertTrue(is_dvidversion(dvidserver, uuid)) self.assertFalse(is_dvidversion(dvidserver, uuid + "JUNK"))
def test_create_label_instance(self): """Test creation of labelarray and block size fetch """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") create_label_instance(dvidserver, uuid, "labels", typename='labelmap') blocksize = get_blocksize(dvidserver, uuid, "labels") self.assertEqual(blocksize, (64, 64, 64))
def createrepo(): """Creates a repo which results in a new gbucket. """ try: ds = DVIDServerService(DVIDLOCATION) uuid = ds.create_new_repo("test", "test description") except Exception as e: return False, str(e) return True, uuid
def createrepo(): """Creates a repo which results in a new gbucket. """ try: ds = DVIDServerService(DVIDLOCATION) uuid = ds.create_new_repo("test", "test description") except Exception as e: return False, str(e) return True, uuid
def test_create_rawarray8(self): """Test creation of rawarray and block size fetch. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") create_rawarray8(dvidserver, uuid, "gray", (32, 16, 14), Compression.JPEG) blocksize = get_blocksize(dvidserver, uuid, "gray") self.assertEqual(blocksize, (32, 16, 14))
def test_isdatainstance(self): """Tests is_datainstance function. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") ns = DVIDNodeService(dvidserver, uuid) ns.create_labelblk("labels") self.assertTrue(is_datainstance(dvidserver, uuid, "labels")) self.assertFalse(is_datainstance(dvidserver, uuid, "labels2"))
def setUpClass(cls): server_service = DVIDServerService(TEST_DVID_SERVER) cls.uuid = server_service.create_new_repo("foo", "bar") cls.instance = 'labels_test_mask_utils' cls.labels = np.zeros((256, 256, 256), np.uint64) cls.labels[50, 50, 40:140] = 1 cls.labels[50, 60:200, 30] = 2 ns = DVIDNodeService(TEST_DVID_SERVER, cls.uuid) ns.create_label_instance(cls.instance, 64) ns.put_labels3D(cls.instance, cls.labels, (0, 0, 0))
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) assert status == http.client.OK, "Request for /repos/info returned status {}".format( status ) assert error_message == "" repos_info = json.loads(body) test_repos = [uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo'] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse."); return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) assert status == httplib.OK, "Request for /repos/info returned status {}".format( status ) assert error_message == "" repos_info = json.loads(body) test_repos = filter( lambda (uuid, repo_info): repo_info and repo_info['Alias'] == 'testrepo', repos_info.items() ) if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse."); return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER, "*****@*****.**", "myapp") status, body, _error_message = connection.make_request( "/repos/info", ConnectionMethod.GET) repos_info = json.loads(body) test_repos = [ uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]['Alias'] == 'testrepo' ] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo( "testrepo", "This repo is for unit tests to use and abuse.") return str(uuid)
def get_testrepo_root_uuid(): connection = DVIDConnection(TEST_DVID_SERVER) status, body, error_message = connection.make_request("/repos/info", ConnectionMethod.GET) assert status == http.client.OK, "Request for /repos/info returned status {}".format(status) assert error_message == "" repos_info = json.loads(body) test_repos = [ uuid_repo_info for uuid_repo_info in list(repos_info.items()) if uuid_repo_info[1] and uuid_repo_info[1]["Alias"] == "testrepo" ] if test_repos: uuid = test_repos[0][0] return str(uuid) else: from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse.") return str(uuid)
def test_dvidpadlabels(self): """Check padding data with DVID labels. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") ns = DVIDNodeService(dvidserver, uuid) ns.create_labelblk("labels") arr = np.random.randint(12442, size=(58, 58, 58)).astype(np.uint64) arr2 = np.zeros((64, 64, 64), np.uint64) arr2[0:58, 0:58, 0:58] = arr # load gray data ns.put_labels3D("labels", arr2, (0, 0, 0)) # load shifted data for comparison arr2[6:64, 6:64, 6:64] = arr # read and pad data schema = partitionSchema(PartitionDims(32, 64, 64), enablemask=True, padding=8, blank_delimiter=99999) volpart = volumePartition(0, VolumeOffset(6, 6, 6)) partitions = schema.partition_data([(volpart, arr)]) # fetch with mask dvidreader = dvidSrc(dvidserver, uuid, "labels", partitions) newparts = dvidreader.extract_volume() self.assertEqual(len(newparts), 2) for (part, vol) in newparts: if part.get_offset().z == 0: match = np.array_equal(arr2[0:32, :, :], vol) self.assertTrue(match) else: match = np.array_equal(arr2[32:64, :, :], vol) self.assertTrue(match)
def test_sycns(self): """Test sync check and setting a sync. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") create_label_instance(dvidserver, uuid, "labels") # check if labels is listening to labels2 self.assertFalse(has_sync(dvidserver, uuid, "labels", "bodies")) # create labelvol and sync to it conn = DVIDConnection(dvidserver) endpoint = "/repo/" + uuid + "/instance" data = {"typename": "labelvol", "dataname": "bodies"} conn.make_request(endpoint, ConnectionMethod.POST, json.dumps(data).encode()) set_sync(dvidserver, uuid, "labels", "bodies") self.assertTrue(has_sync(dvidserver, uuid, "labels", "bodies"))
def create_repo(self, name, description=""): """Create repo. Note: DVID does not require unique names but unique names will be enforced through this interface. This will simplify access for most common use cases. In general, users should use the web console and specific version ids to ensure access to the desired data. Args: name (str): name of DVID respository (must be unique) description (str): description of repository """ try: curr_repos = self.list_repos() for (reponame, uuid) in curr_repos: if reponame == name: raise DicedException("Repo name already exists") service = DVIDServerService(self._server) uuid = service.create_new_repo(name, description) except DVIDException as err: raise DicedException("Failed to create repo")
def test_dvidfetchgray(self): """Check reading grayscale from DVID from partitions. This also checks basic iteration and overwrite of previous data. """ service = DVIDServerService(dvidserver) uuid = service.create_new_repo("foo", "bar") ns = DVIDNodeService(dvidserver, uuid) ns.create_grayscale8("gray") arr = np.random.randint(255, size=(64, 64, 64)).astype(np.uint8) # load gray data ns.put_gray3D("gray", arr, (0, 0, 0)) # read data schema = partitionSchema(PartitionDims(32, 64, 64)) volpart = volumePartition(0, VolumeOffset(0, 0, 0)) overwrite = np.random.randint(255, size=(64, 64, 64)).astype(np.uint8) partitions = schema.partition_data([(volpart, overwrite)]) dvidreader = dvidSrc(dvidserver, uuid, "gray", partitions, maskonly=False) newparts = dvidreader.extract_volume() self.assertEqual(len(newparts), 2) for (part, vol) in newparts: if part.get_offset().z == 0: match = np.array_equal(arr[0:32, :, :], vol) self.assertTrue(match) else: match = np.array_equal(arr[32:64, :, :], vol) self.assertTrue(match) # test iteration dvidreader2 = dvidSrc(dvidserver, uuid, "gray", partitions, maskonly=False) newparts2 = [] for newpart in dvidreader2: self.assertEqual(len(newpart), 1) newparts2.extend(newpart) self.assertEqual(len(newparts2), 2) for (part, vol) in newparts2: if part.get_offset().z == 0: match = np.array_equal(arr[0:32, :, :], vol) self.assertTrue(match) else: match = np.array_equal(arr[32:64, :, :], vol) self.assertTrue(match)
help='verbose logs') args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) # initialize plugin manager plugin_manager = TrackingPluginManager(verbose=False) image_provider = plugin_manager.getImageProvider() # create dataset on server and get uuid server_address = args.dvidAddress server_service = DVIDServerService(server_address) uuid = server_service.create_new_repo(args.datasetName, "description") logging.info('UUID:\n{}'.format(uuid)) # get node service node_service = DVIDNodeService(server_address, uuid) # get dataset size and store in dvid shape = image_provider.getImageShape(args.ilpFilename, args.labelImagePath) time_range = image_provider.getTimeRange(args.ilpFilename, args.labelImagePath) if args.timeRange is not None: time_range = (max(time_range[0], args.timeRange[0]), min(time_range[1], args.timeRange[1])) logging.info('Uploading time range {} to {}'.format(time_range, server_address)) keyvalue_store = "config" node_service.create_keyvalue(keyvalue_store) settings = { "shape": shape, "time_range": time_range } node_service.put(keyvalue_store, "imageInfo", json.dumps(settings))
parser = argparse.ArgumentParser(description="Batch mode nrrd file to dvid migration script") existingNode = parser.add_argument_group('existing node', 'for working with a node that already exists on the dvid server') parser.add_argument('address', metavar='address', help='address to a valid dvid server in the form x.x.x.x:yyyy') parser.add_argument('file', metavar='nrrdfile', help='filepath for uploading to dvid') existingNode.add_argument('--uuid', '-u', metavar='uuid', help='minimal uid of the node to access on the dvid server') newNode = parser.add_argument_group('new node', 'for creating a new node before migrating the nrrd files') newNode.add_argument('--alias', '-a', metavar='alias', help='alias for a new node to create') newNode.add_argument('--description', '-d', metavar='description', help='description for new node') newNode.add_argument('--segmentation', '-s', action='store_true', help='flags data as a segmentation block in the case that there was no indication in the header') args = parser.parse_args() addr = args.address if args.alias: service = DVIDServerService(addr) uid = service.create_new_repo(args.alias, args.description) else: uid = args.uuid def push_to_dvid(method, handle, data, preoffset=(0, 0, 0), throttle=False, compress=True, chunkDepth=512): """Function for pushing to DVID.""" zsize = data.shape[0] numsplits = zsize / 512 offset = 0 pdb.set_trace() for i in xrange(numsplits): seg = data[i * chunkDepth:(i + 1) * chunkDepth, :, :] offsetTuple = (preoffset[0] + i * chunkDepth, preoffset[1], preoffset[2]) method(handle, seg, offsetTuple, throttle, compress)
def main(): # Read cmd-line args parser = argparse.ArgumentParser() parser.add_argument("--hostname", default="localhost:8000") parser.add_argument( "--uuid", required=False, help= "The node to upload to. If not provided, a new repo will be created (see --new-repo-alias)." ) parser.add_argument( "--data-name", required=False, help= "The name of the data instance to modify. If it doesn't exist, it will be created first." ) parser.add_argument( "--new-repo-alias", required=False, help="If no uuid is provided, a new repo is created, with this name.") parser.add_argument("input_file", help="For example: /tmp/myfile.h5/dataset") args = parser.parse_args() if '.h5' not in args.input_file: sys.stderr.write("File name does not indicate hdf5.\n") sys.exit(1) filepath, dset_name = args.input_file.split('.h5') filepath += '.h5' if not dset_name: sys.stderr.write( "You must provide a dataset name, e.g. myfile.h5/mydataset\n") sys.exit(1) if not os.path.exists(filepath): sys.stderr.write("File doesn't exist: {}\n".format(filepath)) sys.exit(1) # If no uuid given, create a new repo on the server uuid = args.uuid if uuid is None: alias = args.new_repo_alias or "testrepo" server = DVIDServerService(args.hostname) uuid = server.create_new_repo( alias, "This is a test repo loaded with data from ".format( args.input_file)) uuid = str(uuid) # Read the input data from the file print("Reading {}{}".format(filepath, dset_name)) with h5py.File(filepath) as f_in: data = f_in[dset_name][:] # We assume data is 3D or 4D, in C-order # We adjust it to 4D, fortran-order if data.ndim == 3: data = data[..., None] data = data.transpose() assert data.flags['F_CONTIGUOUS'], "Data is not contiguous!" assert data.ndim == 4, "Data must be 3D with axes zyx or 4D with axes zyxc (C-order)" assert data.shape[ 0] == 1, "Data must have exactly 1 channel, not {}".format( data.shape[0]) # Choose a default data instance name if necessary if data.dtype == numpy.uint8: data_name = args.data_name or "grayscale" elif data.dtype == numpy.uint64: data_name = args.data_name or "segmentation" else: sys.stderr.write("Unsupported dtype: {}\n".format(data.dtype)) sys.exit(1) # Create the new data instance if it doesn't exist already try: metadata = VoxelsAccessor.get_metadata(args.hostname, uuid, data_name) print("Data instance '{}' already exists. Will update.".format( data_name)) except DVIDException: print("Creating new data instance: {}".format(data_name)) metadata = VoxelsMetadata.create_default_metadata( data.shape, data.dtype, 'cxyz', 1.0, 'nanometers') VoxelsAccessor.create_new(args.hostname, uuid, data_name, metadata) # Finally, push the data to the server print("Pushing data to {}".format('{}/api/node/{}/{}'.format( args.hostname, uuid, data_name))) accessor = VoxelsAccessor(args.hostname, uuid, data_name) accessor.post_ndarray((0, 0, 0, 0), data.shape, data) print("DONE.")
def get_testrepo_root_uuid(): from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo( "testrepo", "This repo is for unit tests to use and abuse.") return str(uuid)
def get_testrepo_root_uuid(): from libdvid import DVIDServerService server = DVIDServerService(TEST_DVID_SERVER) uuid = server.create_new_repo("testrepo", "This repo is for unit tests to use and abuse."); return str(uuid)
def main(): # Read cmd-line args parser = argparse.ArgumentParser() parser.add_argument("--hostname", default="localhost:8000") parser.add_argument("--uuid", required=False, help="The node to upload to. If not provided, a new repo will be created (see --new-repo-alias).") parser.add_argument("--data-name", required=False, help="The name of the data instance to modify. If it doesn't exist, it will be created first.") parser.add_argument("--new-repo-alias", required=False, help="If no uuid is provided, a new repo is created, with this name.") parser.add_argument("input_file", help="For example: /tmp/myfile.h5/dataset") args = parser.parse_args() if '.h5' not in args.input_file: sys.stderr.write("File name does not indicate hdf5.\n") sys.exit(1) filepath, dset_name = args.input_file.split('.h5') filepath += '.h5' if not dset_name: sys.stderr.write("You must provide a dataset name, e.g. myfile.h5/mydataset\n") sys.exit(1) if not os.path.exists(filepath): sys.stderr.write("File doesn't exist: {}\n".format(filepath)) sys.exit(1) # If no uuid given, create a new repo on the server uuid = args.uuid if uuid is None: alias = args.new_repo_alias or "testrepo" server = DVIDServerService(args.hostname) uuid = server.create_new_repo(alias, "This is a test repo loaded with data from ".format(args.input_file)) uuid = str(uuid) # Read the input data from the file print("Reading {}{}".format( filepath, dset_name )) with h5py.File(filepath) as f_in: data = f_in[dset_name][:] # We assume data is 3D or 4D, in C-order # We adjust it to 4D, fortran-order if data.ndim == 3: data = data[...,None] data = data.transpose() assert data.flags['F_CONTIGUOUS'], "Data is not contiguous!" assert data.ndim == 4, "Data must be 3D with axes zyx or 4D with axes zyxc (C-order)" assert data.shape[0] == 1, "Data must have exactly 1 channel, not {}".format( data.shape[0] ) # Choose a default data instance name if necessary if data.dtype == numpy.uint8: data_name = args.data_name or "grayscale" elif data.dtype == numpy.uint64: data_name = args.data_name or "segmentation" else: sys.stderr.write("Unsupported dtype: {}\n".format(data.dtype)) sys.exit(1) # Create the new data instance if it doesn't exist already try: metadata = VoxelsAccessor.get_metadata(args.hostname, uuid, data_name) print("Data instance '{}' already exists. Will update.".format( data_name )) except DVIDException: print("Creating new data instance: {}".format( data_name )) metadata = VoxelsMetadata.create_default_metadata(data.shape, data.dtype, 'cxyz', 1.0, 'nanometers') VoxelsAccessor.create_new(args.hostname, uuid, data_name, metadata) # Finally, push the data to the server print("Pushing data to {}".format( '{}/api/node/{}/{}'.format( args.hostname, uuid, data_name ) )) accessor = VoxelsAccessor(args.hostname, uuid, data_name) accessor.post_ndarray((0,0,0,0), data.shape, data) print("DONE.")