def GetBlockList(self, vault, marker=None, limit=None): """Retrieve the list of blocks in the vault :param vault: vault to get the block list for :param marker: marker denoting the start of the list :param limit: integer denoting the maximum entries to retrieve :stores: The block information in the blocks property of the Vault :returns: True on success :raises: TypeError if vault is not a Vault object :raises: RunTimeError on failure """ url = api_v1.get_blocks_path(vault.vault_id) query_args = {} if marker and limit: query_args = { 'marker': marker, 'limit': limit, } elif marker: query_args = { 'marker': marker } elif limit: query_args = { 'limit': limit } else: pass ret_url = set_qs_on_url(url, query_args) self.ReInit(self.sslenabled, ret_url) self.__update_headers() self.__log_request_data(fn='Get Block List') res = requests.get(self.Uri, headers=self.Headers) self.__log_response_data(res, jsondata=True, fn='Get Block List') if res.status_code == 200: block_ids = [] for block_entry in res.json(): vault.blocks[block_entry] = api_block.Block(vault.project_id, vault.vault_id, block_entry) block_ids.append(block_entry) if 'x-next-batch' in res.headers: parsed_url = urlparse(res.headers['x-next-batch']) qs = parse_qs(parsed_url[4]) vault.blocks.marker = qs['marker'][0] else: vault.blocks.marker = None return block_ids else: raise RuntimeError( 'Failed to get Block list for Vault . ' 'Error ({0:}): {1:}'.format(res.status_code, res.text))
def ListVaults(self, project, marker=None, limit=None): """List vaults for the user :param marker: vaultid within the list to start at :param limit: the maximum number of entries to retrieve :returns: deuceclient.api.Projects instance containing the vaults :raises: RuntimeError on failure """ url = api_v1.get_vault_base_path() query_args = {} if marker and limit: query_args = { 'marker': marker, 'limit': limit, } elif marker: query_args = { 'marker': marker } elif limit: query_args = { 'limit': limit } else: pass ret_url = set_qs_on_url(url, query_args) self.ReInit(self.sslenabled, ret_url) self.__update_headers() self.__log_request_data(fn='List Vaults') res = requests.get(self.Uri, headers=self.Headers) self.__log_response_data(res, jsondata=True, fn='List Vaults') if res.status_code == 200: for vault_name, vault_data in res.json().items(): if vault_name not in project: project[vault_name] = api_vault.Vault( project_id=project.project_id, vault_id=vault_name) project[vault_name].status = 'valid' if 'x-next-batch' in res.headers: parsed_url = urlparse(res.headers['x-next-batch']) qs = parse_qs(parsed_url[4]) project.marker = qs['marker'][0] else: project.marker = None return True else: raise RuntimeError( 'Failed to List Vaults. ' 'Error ({0:}): {1:}'.format(res.status_code, res.text))
def UploadBlocks(self, vault, block_ids, request_mapping=True): """Upload a series of blocks at the same time :param vault: vault to upload the blocks into :param block_ids: block ids in the vault to upload, must be an iterable object :returns: True on success """ url = api_v1.get_blocks_path(vault.vault_id) query_args = {} if request_mapping: query_args = { 'mapping': request_mapping } else: pass ret_url = set_qs_on_url(url, query_args) self.ReInit(self.sslenabled, ret_url) self.__update_headers() headers = {} headers.update(self.Headers) headers['Content-Type'] = 'application/msgpack' block_data = [] for block_id in block_ids: block = vault.blocks[block_id] block_data.append((block_id, block.data)) contents = dict(block_data) body = msgpack.packb(contents) self.__log_request_data(fn='Upload Multiple Blocks - msgpack') res = requests.post(self.Uri, headers=headers, data=body) self.__log_response_data(res, jsondata=False, fn='Upload Multiple Blocks - msgpack') if res.status_code == 201: return True elif res.status_code == 200: for block_id, storage_block_id in res.json().items(): self.log.info('Vault {0}: Block {1} maps to storage block {2}' .format(vault.vault_id, block_id, storage_block_id)) return True else: raise RuntimeError( 'Failed to upload blocks to Vault. ' 'Error ({0:}): {1:}'.format(res.status_code, res.text))
def test_set_qs_on_url(self): url = 'http://whatever:8080/hello/world' params = { 'param1': 'value1', 'param2': 'value2' } ret_url = set_qs_on_url(url, params) parts = parse.urlparse(ret_url) qs = parse.parse_qs(parts.query) self.assertEqual(qs['param1'][0], 'value1') self.assertEqual(qs['param2'][0], 'value2')
def GetBlockStorageList(self, vault, marker=None, limit=None): """List blocks directly from block storage :param vault: instance of deuce.api.vault.Vault :param marker: string :param limit: string :return: True if expected status code is returned, Runtime Error raised if that's not the case. """ url = api_v1.get_storage_blocks_path(vault.vault_id) query_args = {} if marker and limit: query_args = { 'marker': marker, 'limit': limit, } elif marker: query_args = { 'marker': marker } elif limit: query_args = { 'limit': limit } else: pass ret_url = set_qs_on_url(url, query_args) self.ReInit(self.sslenabled, ret_url) self.__update_headers() self.__log_request_data(fn='Get Block Storage List') res = requests.get(self.Uri, headers=self.Headers) self.__log_response_data(res, jsondata=True, fn='Get Block Storage List') if res.status_code == 200: block_list = api_storageblocks.StorageBlocks( project_id=self.project_id, vault_id=vault.vault_id) blocks = { storageblockid: api_block.Block(project_id=self.project_id, vault_id=vault.vault_id, storage_id=storageblockid, block_type='storage') for storageblockid in res.json()} block_list.update(blocks) vault.storageblocks.update(block_list) if 'x-next-batch' in res.headers: parsed_url = urlparse(res.headers['x-next-batch']) qs = parse_qs(parsed_url[4]) vault.storageblocks.marker = qs['marker'][0] else: vault.storageblocks.marker = None return [storageblockid for storageblockid in res.json()] else: raise RuntimeError( 'Failed to get Block Storage list for Vault . ' 'Error ({0:}): {1:}'.format(res.status_code, res.text))
def GetFileBlockList(self, vault, file_id, marker=None, limit=None): """Retrieve the list of blocks assigned to the file :param vault: vault to the file belongs to :param fileid: fileid of the file in the Vault to list the blocks for :param marker: blockid within the list to start at :param limit: the maximum number of entries to retrieve :stores: The resulting block list in the file data for the vault. :returns: True on success """ if file_id not in vault.files: raise KeyError( 'file_id must specify a file in the provided Vault.') url = api_v1.get_fileblocks_path(vault.vault_id, file_id) query_args = {} if marker and limit: query_args = { 'marker': marker, 'limit': limit, } elif marker: query_args = { 'marker': marker } elif limit: query_args = { 'limit': limit } else: pass ret_url = set_qs_on_url(url, query_args) self.ReInit(self.sslenabled, ret_url) self.__update_headers() self.__log_request_data(fn='Get File Block List') res = requests.get(self.Uri, headers=self.Headers) self.__log_response_data(res, jsondata=True, fn='Get File Block List') if res.status_code == 200: block_ids = [] for block_id, offset in res.json(): vault.files[file_id].offsets[offset] = block_id block_ids.append(block_id) next_marker = None if 'x-next-batch' in res.headers: parsed_url = urlparse(res.headers['x-next-batch']) qs = parse_qs(parsed_url[4]) next_marker = qs['marker'][0] return (block_ids, next_marker) else: raise RuntimeError( 'Failed to get Block list for File . ' 'Error ({0:}): {1:}'.format(res.status_code, res.text))
def ListFiles(self, vault, marker=None, limit=None): """List files in the Vault :param vault: vault to list the files from :param marker: fileid within the list to start at :param limit: the maximum number of entries to retrieve :returns: a list of file ids in the vault """ url = api_v1.get_files_path(vault.vault_id) query_args = {} if marker and limit: query_args = { 'marker': marker, 'limit': limit, } elif marker: query_args = { 'marker': marker } elif limit: query_args = { 'limit': limit } else: pass ret_url = set_qs_on_url(url, query_args) self.ReInit(self.sslenabled, ret_url) self.__update_headers() self.__log_request_data(fn='List Files') res = requests.get(self.Uri, headers=self.Headers) self.__log_response_data(res, jsondata=True, fn='List Files') if res.status_code == 200: if 'x-next-batch' in res.headers: parsed_url = urlparse(res.headers['x-next-batch']) qs = parse_qs(parsed_url[4]) vault.files.marker = qs['marker'][0] else: vault.files.marker = None return_list = [] for file_id in res.json(): return_list.append(file_id) file_uri = api_v1.get_file_path(vault.vault_id, file_id) self.ReInit(self.sslenabled, url) file_url = self.Uri kw = { 'project_id': self.project_id, 'vault_id': vault.vault_id, 'file_id': file_id, 'url': file_url } vault.files[file_id] = api_file.File(**kw) return return_list else: raise RuntimeError( 'Failed to List Files in the Vault. ' 'Error ({0:}): {1:}'.format(res.status_code, res.text))