def test_vault_crud(self): vault_id = self.create_vault_id() v = Vault.get(vault_id) assert v is None v = Vault.create(vault_id) assert v is not None v.delete() v = Vault.get(vault_id) assert v is None
def post(self, vault_id, file_id=None): """Initializes a new file. The location of the new file is returned in the Location header """ vault = Vault.get(vault_id) # caller tried to post to a vault that # does not exist if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) abort(400, headers={"Transaction-ID": deuce.context.transaction.request_id}) # overload to use the same end-point for creating a new file # and assigning blocks to a file that is in progress if file_id is not None: return self._assign(vault, vault_id, file_id) file = vault.create_file() response.headers["Location"] = "files/%s" % file.file_id response.status_code = 201 # Created logger.info('File [{0}] created'. format(response.headers["Location"]))
def get_one(self, vault_id, file_id): """Fetches, re-assembles and streams a single file out of Deuce""" vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) abort(404, headers={"Transaction-ID": deuce.context.transaction.request_id}) f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) abort(404, headers={"Transaction-ID": deuce.context.transaction.request_id}) if not f.finalized: abort(412, headers={"Transaction-ID": deuce.context.transaction.request_id}) block_gen = deuce.metadata_driver.create_file_block_generator( vault_id, file_id) block_ids = [block[0] for block in sorted(block_gen, key=lambda block: block[1])] objs = vault.get_blocks_generator(block_ids) response.content_length = vault.get_file_length(file_id) response.body_file = FileCat(objs) response.status_code = 200
def get_all(self, vault_id): vault = Vault.get(request.project_id, vault_id) if not vault: abort(404) inmarker = request.params.get('marker') limit = int(request.params.get('limit', conf.api_configuration.max_returned_num)) # The +1 is to fetch one past the user's # requested limit so that we can determine # if the list was truncated or not files = vault.get_files(inmarker, limit + 1) resp = list(files) # Note: the list may not actually be truncated truncated = len(resp) == limit + 1 outmarker = resp.pop().file_id if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs(request.url, query_args) response.headers["X-Next-Batch"] = returl return resp
def on_get(self, req, resp, vault_id): """List the blocks in the vault from storage-alone """ vault = Vault.get(vault_id) if vault is None: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound inmarker = req.get_param('marker') if req.get_param('marker') else None limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \ conf.api_configuration.default_returned_num # We actually fetch the user's requested # limit +1 to detect if the list is being # truncated or not. storage = BlockStorage.get(vault_id) storage_blocks = storage.get_blocks_generator(inmarker, limit + 1) responses = list(storage_blocks) # Was the list truncated? See note above about +1 truncated = len(responses) > 0 and len(responses) == limit + 1 outmarker = responses.pop().storage_block_id if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header("X-Next-Batch", returl) resp.body = json.dumps( [response.storage_block_id for response in responses])
def on_post(self, req, resp, vault_id, file_id): """This endpoint Assigns blocks to files """ vault = Vault.get(vault_id) # caller tried to post to a vault that # does not exist if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPBadRequestAPI('Vault does not exist') f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) raise errors.HTTPNotFound if f.finalized: logger.error('Finalized file [{0}] ' 'cannot be modified'.format(file_id)) raise errors.HTTPConflict('Finalized file cannot be modified') body = req.stream.read(req.content_length) # TODO (TheSriram): Validate payload payload = json.loads(body.decode()) block_ids, offsets = zip(*payload) missing_blocks = deuce.metadata_driver.has_blocks(vault_id, block_ids) deuce.metadata_driver.assign_blocks(vault_id, file_id, block_ids, offsets) resp.body = json.dumps(missing_blocks)
def on_post(self, req, resp, vault_id, file_id): """This endpoint Assigns blocks to files """ vault = Vault.get(vault_id) # caller tried to post to a vault that # does not exist if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPBadRequestAPI('Vault does not exist') f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) raise errors.HTTPNotFound if f.finalized: logger.error('Finalized file [{0}] ' 'cannot be modified'.format(file_id)) raise errors.HTTPConflict('Finalized file cannot be modified') body = req.stream.read(req.content_length) # TODO (TheSriram): Validate payload payload = json.loads(body.decode()) block_ids, offsets = zip(*payload) missing_blocks = deuce.metadata_driver.has_blocks(vault_id, block_ids, check_status=True) deuce.metadata_driver.assign_blocks(vault_id, file_id, block_ids, offsets) resp.body = json.dumps(missing_blocks)
def get_all(self, vault_id, file_id): vault = Vault.get(request.project_id, vault_id) assert vault is not None f = vault.get_file(file_id) if not f: abort(404) inmarker = int(request.params.get('marker', 0)) limit = int(request.params.get('limit', conf.api_configuration.max_returned_num)) # Get the block generator from the metadata driver. # Note: +1 on limit is to fetch one past the limt # for the purpose of determining if the # list was truncated retblks = deuce.metadata_driver.create_file_block_generator( request.project_id, vault_id, file_id, inmarker, limit + 1) resp = list(retblks) truncated = len(resp) > 0 and len(resp) == limit + 1 outmarker = resp.pop()[1] if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs(request.url, query_args) response.headers["X-Next-Batch"] = returl return resp
def on_delete(self, req, resp, vault_id, block_id): """Unregisters a block_id from a given vault_id in the storage and metadata """ vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) resp.status = falcon.HTTP_404 return try: response = vault.delete_block(vault_id, block_id) except ConstraintError as ex: logger.error(json.dumps(ex.args)) raise errors.HTTPConflict(json.dumps(ex.args)) except Exception as ex: # pragma: no cover logger.error(ex) raise errors.HTTPServiceUnavailable else: if response: logger.info('block [{0}] deleted from vault {1}'.format( block_id, vault_id)) resp.status = falcon.HTTP_204 else: logger.error('block [{0}] does not exist'.format(block_id)) raise errors.HTTPNotFound
def on_put(self, req, resp, vault_id, block_id): """Uploads a block into Deuce. The URL of the block is returned in the Location header """ vault = Vault.get(vault_id) try: retval, storage_id = vault.put_block( block_id, req.stream.read(), req.content_length) resp.set_header('X-Storage-ID', str(storage_id)) resp.set_header('X-Block-ID', str(block_id)) block = vault.get_block(block_id) ref_cnt = 0 ref_mod = 0 if retval: ref_cnt = block.get_ref_count() ref_mod = block.get_ref_modified() resp.set_header('X-Block-Reference-Count', str(ref_cnt)) resp.set_header('X-Ref-Modified', str(ref_mod)) resp.status = ( falcon.HTTP_201 if retval is True else falcon.HTTP_500) logger.info('block [{0}] added [{1}]'.format(block_id, storage_id)) except ValueError as e: raise errors.HTTPPreconditionFailed('hash error') except BufferError as e: raise errors.HTTPPreconditionFailed( 'content length did not match data length')
def on_get(self, req, resp, vault_id): vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound # NOTE(TheSriram): get_param(param) automatically returns None # if param is not present inmarker = req.get_param('marker') limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \ conf.api_configuration.default_returned_num # We actually fetch the user's requested # limit +1 to detect if the list is being # truncated or not. blocks = vault.get_blocks(inmarker, limit + 1) # List the blocks into JSON and return. # TODO: figure out a way to stream this back(??) responses = list(blocks) # Was the list truncated? See note above about +1 truncated = len(responses) > 0 and len(responses) == limit + 1 outmarker = responses.pop().metadata_block_id if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header("X-Next-Batch", returl) resp.body = json.dumps([response.metadata_block_id for response in responses])
def on_delete(self, req, resp, vault_id, block_id): """Unregisters a block_id from a given vault_id in the storage and metadata """ vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) resp.status = falcon.HTTP_404 return try: response = vault.delete_block(vault_id, block_id) except ConstraintError as ex: logger.error(json.dumps(ex.args)) raise errors.HTTPConflict(json.dumps(ex.args)) except Exception as ex: # pragma: no cover logger.error(ex) raise errors.HTTPServiceUnavailable else: if response: logger.info('block [{0}] deleted from vault {1}' .format(block_id, vault_id)) resp.status = falcon.HTTP_204 else: logger.error('block [{0}] does not exist'.format(block_id)) raise errors.HTTPNotFound
def on_put(self, req, resp, vault_id, block_id): """Uploads a block into Deuce. The URL of the block is returned in the Location header """ vault = Vault.get(vault_id) try: retval, storage_id = vault.put_block(block_id, req.stream.read(), req.content_length) resp.set_header('X-Storage-ID', str(storage_id)) resp.set_header('X-Block-ID', str(block_id)) block = vault.get_block(block_id) ref_cnt = 0 ref_mod = 0 if retval: ref_cnt = block.get_ref_count() ref_mod = block.get_ref_modified() resp.set_header('X-Block-Reference-Count', str(ref_cnt)) resp.set_header('X-Ref-Modified', str(ref_mod)) resp.status = (falcon.HTTP_201 if retval is True else falcon.HTTP_500) logger.info('block [{0}] added [{1}]'.format(block_id, storage_id)) except ValueError as e: raise errors.HTTPPreconditionFailed('hash error') except BufferError as e: raise errors.HTTPPreconditionFailed( 'content length did not match data length')
def on_get(self, req, resp, vault_id): vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound # NOTE(TheSriram): get_param(param) automatically returns None # if param is not present inmarker = req.get_param('marker') limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \ conf.api_configuration.default_returned_num # We actually fetch the user's requested # limit +1 to detect if the list is being # truncated or not. blocks = vault.get_blocks(inmarker, limit + 1) # List the blocks into JSON and return. # TODO: figure out a way to stream this back(??) responses = list(blocks) # Was the list truncated? See note above about +1 truncated = len(responses) > 0 and len(responses) == limit + 1 outmarker = responses.pop().metadata_block_id if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header("X-Next-Batch", returl) resp.body = json.dumps( [response.metadata_block_id for response in responses])
def on_post(self, req, resp, vault_id): vault = Vault.get(vault_id) try: unpacked = msgpack.unpackb(req.stream.read()) if not isinstance(unpacked, dict): raise TypeError else: block_ids = list(unpacked.keys()) block_datas = list(unpacked.values()) try: retval = vault.put_async_block(block_ids, block_datas) if retval: resp.status = falcon.HTTP_201 else: raise errors.HTTPInternalServerError('Block ' 'Post Failed') logger.info('blocks [{0}] added'.format(block_ids)) except ValueError: raise errors.HTTPPreconditionFailed('hash error') except (TypeError, ValueError): logger.error('Request Body not well formed ' 'for posting multiple blocks to {0}'.format(vault_id)) raise errors.HTTPBadRequestBody("Request Body not well formed")
def on_get(self, req, resp): # NOTE(TheSriram): get_param(param) automatically returns None # if param is not present inmarker = req.get_param('marker') limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \ conf.api_configuration.default_returned_num vaultlist = Vault.get_vaults_generator( inmarker, limit + 1) response = list(vaultlist) if not response: resp.body = json.dumps([]) # Note: the list may not actually be truncated truncated = len(response) == limit + 1 outmarker = response.pop() if truncated else None # Set x-next-batch resp header. if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header(name="X-Next-Batch", value=returl) # Set return json for vault URLs. p = urlparse(req.url) resp.body = json.dumps(dict(six.moves.map(lambda vaultname: (vaultname, {"url": p.scheme + '://' + p.netloc + p.path + '/' + vaultname}), response)))
def on_get(self, req, resp, vault_id): vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound # NOTE(TheSriram): get_param(param) automatically returns None # if param is not present inmarker = req.get_param('marker') limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') \ else conf.api_configuration.default_returned_num # The +1 is to fetch one past the user's # requested limit so that we can determine # if the list was truncated or not files = vault.get_files(inmarker, limit + 1) responses = list(files) # Note: the list may not actually be truncated truncated = len(responses) == limit + 1 outmarker = responses.pop().file_id if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header("X-Next-Batch", returl) resp.body = json.dumps([response.file_id for response in responses])
def on_get(self, req, resp, vault_id): """List the blocks in the vault from storage-alone """ vault = Vault.get(vault_id) if vault is None: logger.error("Vault [{0}] does not exist".format(vault_id)) raise errors.HTTPNotFound inmarker = req.get_param("marker") if req.get_param("marker") else None limit = ( req.get_param_as_int("limit") if req.get_param_as_int("limit") else conf.api_configuration.default_returned_num ) # We actually fetch the user's requested # limit +1 to detect if the list is being # truncated or not. storage = BlockStorage.get(vault_id) storage_blocks = storage.get_blocks_generator(inmarker, limit + 1) responses = list(storage_blocks) # Was the list truncated? See note above about +1 truncated = len(responses) > 0 and len(responses) == limit + 1 outmarker = responses.pop().storage_block_id if truncated else None if outmarker: query_args = {"marker": outmarker} query_args["limit"] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header("X-Next-Batch", returl) resp.body = json.dumps([response.storage_block_id for response in responses])
def on_get(self, req, resp, vault_id, file_id): """Fetches, re-assembles and streams a single file out of Deuce""" vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) raise errors.HTTPNotFound if not f.finalized: raise errors.HTTPConflict('File not Finalized') block_gen = deuce.metadata_driver.create_file_block_generator( vault_id, file_id) block_ids = [block[0] for block in sorted(block_gen, key=lambda block: block[1])] objs = vault.get_blocks_generator(block_ids) # NOTE(TheSriram): falcon 0.2.0 might fix this problem, # we should be able to set resp.stream to any file like # object instead of an iterator. resp.stream = (obj.read() for obj in objs) resp.status = falcon.HTTP_200 resp.set_header('Content-Length', str(vault.get_file_length(file_id))) resp.content_type = 'application/octet-stream'
def get_all(self, vault_id): vault = Vault.get(request.project_id, vault_id) if not vault: response.status_code = 404 return inmarker = request.params.get('marker') limit = int(request.params.get('limit', conf.api_configuration.max_returned_num)) # We actually fetch the user's requested # limit +1 to detect if the list is being # truncated or not. blocks = vault.get_blocks(inmarker, limit + 1) # List the blocks into JSON and return. # TODO: figure out a way to stream this back(??) resp = list(blocks) # Was the list truncated? See note above about +1 truncated = len(resp) > 0 and len(resp) == limit + 1 outmarker = resp.pop().block_id if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs(request.url, query_args) response.headers["X-Next-Batch"] = returl return resp
def on_post(self, req, resp, vault_id): vault = Vault.get(vault_id) try: unpacked = msgpack.unpackb(req.stream.read()) if not isinstance(unpacked, dict): raise TypeError else: block_ids = list(unpacked.keys()) block_datas = list(unpacked.values()) try: retval, retblocks = vault.put_async_block( block_ids, block_datas) if retval: resp.status = falcon.HTTP_200 resp.body = json.dumps({block_id: storage_id for block_id, storage_id in retblocks}) else: raise errors.HTTPInternalServerError('Block ' 'Post Failed') logger.info('blocks [{0}] added'.format(block_ids)) except ValueError: raise errors.HTTPPreconditionFailed('hash error') except (TypeError, ValueError): logger.error('Request Body not well formed ' 'for posting multiple blocks to {0}'.format(vault_id)) raise errors.HTTPBadRequestBody("Request Body not well formed")
def put(self, vault_id, block_id=None): """Uploads a block into Deuce. The URL of the block is returned in the Location header """ vault = Vault.get(request.project_id, vault_id) vault.put_block( block_id, request.body, request.headers['content-length'])
def on_head(self, req, resp, vault_id): """Returns the vault controller object""" if Vault.get(vault_id): resp.status = falcon.HTTP_204 else: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound
def on_put(self, req, resp, vault_id): vault = Vault.create(vault_id) # TODO: Need check and monitor failed vault. logger.info('Vault [{0}] created'.format(vault_id)) if vault: resp.status = falcon.HTTP_201 else: raise errors.HTTPInternalServerError('Vault Creation Failed')
def delete(self, vault_id): vault = Vault.get(request.project_id, vault_id) if vault: vault.delete() response.status_code = 200 else: response.status_code = 404
def get_one(self, vault_id): """Returns the vault controller object""" if Vault.get(request.project_id, vault_id): response.status_code = 200 else: response.status_code = 404 return None
def on_head(self, req, resp, vault_id, block_id): """Checks for the existence of the block in the metadata storage and if successful check for it in the storage driver if it fails we return a 502, otherwise we return all other headers returned on GET /v1.0/vaults/{vault_id}/blocks/{block_id} """ vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound # This is safe to construct before we check the status of the block # b/c the constructor does not do anything other than save values # to itself, no lookups, etc block = Block(vault_id, block_id) try: if not vault.has_block(block_id, check_storage=True): logger.error('block [{0}] does not exist'.format(block_id)) raise errors.HTTPNotFound ref_cnt = block.get_ref_count() resp.set_header('X-Block-Reference-Count', str(ref_cnt)) ref_mod = block.get_ref_modified() resp.set_header('X-Ref-Modified', str(ref_mod)) storage_id = block.get_storage_id() resp.set_header('X-Storage-ID', str(storage_id)) resp.set_header('X-Block-ID', str(block_id)) resp.set_header('X-Block-Size', str(block.get_block_length())) resp.status = falcon.HTTP_204 except ConsistencyError as ex: # We have the block in metadata... # so we can get anything that only touches metadata ref_cnt = block.get_ref_count() resp.set_header('X-Block-Reference-Count', str(ref_cnt)) ref_mod = block.get_ref_modified() resp.set_header('X-Ref-Modified', str(ref_mod)) storage_id = block.get_storage_id() resp.set_header('X-Storage-ID', str(storage_id)) resp.set_header('X-Block-ID', str(block_id)) # Block-size is retrieved from storage... logger.error(ex) raise errors.HTTPGone(str(ex))
def test_block_crud(self): vault_id = self.create_vault_id() v = Vault.create(vault_id) # Check for blocks, should be none blocks_gen = v.get_blocks(0, 0) blocks_list = list(blocks_gen) assert len(blocks_list) == 0
def on_patch(self, req, resp, vault_id): vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound vault.reset_block_status() resp.status = falcon.HTTP_204
def on_get(self, req, resp, vault_id): """Returns the statistics on vault controller object""" vault = Vault.get(vault_id) if vault: resp.body = json.dumps(vault.get_vault_statistics()) resp.status = falcon.HTTP_200 else: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound
def delete(self, vault_id, file_id): vault = Vault.get(vault_id) if not vault: abort(404) f = vault.get_file(file_id) if not f: abort(404) vault.delete_file(file_id)
def get_one(self, vault_id): """Returns the statistics on vault controller object""" vault = Vault.get(vault_id) if vault: response.status_code = 200 return vault.get_vault_statistics() else: logger.error('Vault [{0}] does not exist'.format(vault_id)) response.status_code = 404 return None
def on_get(self, req, resp, vault_id): """Returns the statistics on vault controller object""" vault = Vault.get(vault_id) if vault: vault_stats = vault.get_vault_statistics() resp.body = json.dumps(vault_stats) resp.status = falcon.HTTP_200 else: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound
def on_delete(self, req, resp, vault_id, file_id): vault = Vault.get(vault_id) if not vault: raise errors.HTTPNotFound f = vault.get_file(file_id) if not f: raise errors.HTTPNotFound vault.delete_file(file_id) resp.status = falcon.HTTP_204
def head(self, vault_id): """Returns the vault controller object""" if Vault.get(vault_id): # weblint complains about the content-type header being # present as pecan doesn't intelligently add it or remove # it. del response.headers["Content-Type"] response.status_code = 204 return response else: logger.error('Vault [{0}] does not exist'.format(vault_id)) response.status_code = 404
def put(self, vault_id, block_id=None): """Uploads a block into Deuce. The URL of the block is returned in the Location header """ vault = Vault.get(vault_id) try: retval = vault.put_block( block_id, request.body, request.headers['content-length']) response.status_code = (201 if retval is True else 500) logger.info('block [{0}] added'.format(block_id)) except ValueError as e: response.status_code = 412
def on_get(self, req, resp, vault_id): """Returns the statistics on vault controller object""" vault = Vault.get(vault_id) if vault: bad_blocks, bad_files = vault.get_vault_health() response = { 'Vault': vault_id, 'Bad Blocks': bad_blocks, 'Bad Files': bad_files } resp.body = json.dumps(response) else: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound
def on_delete(self, req, resp, vault_id): vault = Vault.get(vault_id) if vault: if vault.delete(): logger.info('Vault [{0}] deleted'.format(vault_id)) resp.status = falcon.HTTP_204 else: logger.info('Vault [{0}] cannot be deleted'.format(vault_id)) raise errors.HTTPConflict('Vault cannot be deleted') else: logger.error('Vault [{0}] deletion failed; ' 'Vault does not exist'.format(vault_id)) raise errors.HTTPNotFound
def on_get(self, req, resp, vault_id, file_id): """Fetches, re-assembles and streams a single file out of Deuce""" vault = Vault.get(vault_id) if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) raise errors.HTTPNotFound if not f.finalized: raise errors.HTTPConflict('File not Finalized') block_gen = deuce.metadata_driver.create_file_block_generator( vault_id, file_id) block_ids = [block[0] for block in sorted(block_gen, key=lambda block: block[1])] objs = vault.get_blocks_generator(block_ids) # NOTE(TheSriram): falcon 0.2.0 might fix this problem, # we should be able to set resp.stream to any file like # object instead of an iterator. def premature_close(storage_id): msg = '[{0}/{1}/{2}] is missing data' \ 'for storage block {3}'.format(deuce.context.project_id, vault_id, file_id, storage_id) logger.error(msg) raise StopIteration(msg) resp.stream = (obj.read() if obj else premature_close(storageid) for (storageid, obj) in objs) resp.status = falcon.HTTP_200 resp.set_header('Content-Length', str(vault.get_file_length(file_id))) resp.content_type = 'application/octet-stream'
def on_post(self, req, resp, vault_id): """Initializes a new file. The location of the new file is returned in the Location header """ vault = Vault.get(vault_id) # caller tried to post to a vault that # does not exist if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPNotFound file = vault.create_file() resp.set_header("Location", "{0}/{1}".format(req.url, file.file_id)) resp.set_header("X-File-ID", file.file_id) resp.status = falcon.HTTP_201 # Created logger.info('File [{0}] created'.format(file.file_id))
def test_file_crud(self): vault_id = self.create_vault_id() v = Vault.create(vault_id) f = v.create_file() assert isinstance(f, File) assert f.vault_id == vault_id file_id = f.file_id assert (len(file_id) > 0) file2 = v.get_file(file_id) file2_length = v.get_file_length(file_id) assert isinstance(file2, File) assert file2.file_id == file_id assert file2_length == 0
def on_get(self, req, resp, vault_id, file_id): vault = Vault.get(vault_id) assert vault is not None f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) raise errors.HTTPNotFound # NOTE(TheSriram): get_param(param) automatically returns None # if param is not present inmarker = req.get_param_as_int('marker') limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') \ else conf.api_configuration.default_returned_num # Get the block generator from the metadata driver. # Note: +1 on limit is to fetch one past the limt # for the purpose of determining if the # list was truncated retblks = deuce.metadata_driver.create_file_block_generator( vault_id, file_id, inmarker, limit + 1) responses = list(retblks) truncated = len(responses) > 0 and len(responses) == limit + 1 outmarker = responses.pop()[1] if truncated else None if outmarker: query_args = {'marker': outmarker} query_args['limit'] = limit returl = set_qs_on_url(req.url, query_args) resp.set_header("X-Next-Batch", returl) resp.body = json.dumps(responses)
def on_post(self, req, resp, vault_id, file_id): """This endpoint finalizes a file """ vault = Vault.get(vault_id) # caller tried to post to a vault that # does not exist if not vault: logger.error('Vault [{0}] does not exist'.format(vault_id)) raise errors.HTTPBadRequestAPI('Vault does not exist') f = vault.get_file(file_id) if not f: logger.error('File [{0}] does not exist'.format(file_id)) raise errors.HTTPNotFound if f.finalized: logger.error('Finalized file [{0}] ' 'cannot be modified'.format(file_id)) raise errors.HTTPConflict('Finalized file cannot be modified') try: filesize = int(req.get_header('x-file-length', required=True)) res = deuce.metadata_driver.finalize_file(vault_id, file_id, filesize) except Exception as e: # There are gaps or overlaps in blocks of the file # The list of errors returns details = str(e) logger.error('File [{0}] finalization ' 'failed; [{1}]'.format(file_id, details)) raise errors.HTTPConflict(json.dumps(details)) else: resp.status = falcon.HTTP_200 return
def test_get_nonexistent_block(self): v = Vault.get('should_not_exist') assert v is None
def on_get(self, req, resp, vault_id, block_id): """Returns a specific block""" # Step 1: Is the block in our vault store? If not, return 404 # Step 2: Stream the block back to the user vault = Vault.get(vault_id) # Existence of the vault should have been confirmed # in the vault controller assert vault is not None try: block = vault.get_block(block_id) if block is None: logger.error('block [{0}] does not exist'.format(block_id)) # We have to do the has_block() check in order to # differentiate between a 404 and 410 error. # 404 should be returned if even metadata doesn't know # about the block; while 410 should be returned if # metadata knows about the block but it is not found # in storage. Since we already know the block doesn't # exist in storage, we can skip the storage check if vault.has_block(block_id, check_storage=False): logger.error( 'block [{0}] does not exist (vault check)'.format( block_id)) raise ConsistencyError(deuce.context.project_id, vault_id, block_id, msg='Block does not exist' ' in Block Storage') raise errors.HTTPNotFound ref_cnt = block.get_ref_count() resp.set_header('X-Block-Reference-Count', str(ref_cnt)) ref_mod = block.get_ref_modified() resp.set_header('X-Ref-Modified', str(ref_mod)) storage_id = block.get_storage_id() resp.set_header('X-Storage-ID', str(storage_id)) resp.set_header('X-Block-ID', str(block_id)) resp.stream = block.get_obj() resp.stream_len = block.get_block_length() resp.status = falcon.HTTP_200 resp.content_type = 'application/octet-stream' except ConsistencyError as ex: # We have the block in metadata... # so we can get anything that only touches metadata block = Block(vault_id, block_id) ref_cnt = block.get_ref_count() resp.set_header('X-Block-Reference-Count', str(ref_cnt)) ref_mod = block.get_ref_modified() resp.set_header('X-Ref-Modified', str(ref_mod)) storage_id = block.get_storage_id() resp.set_header('X-Storage-ID', str(storage_id)) resp.set_header('X-Block-ID', str(block_id)) # Block-size is retrieved from storage... logger.error(ex) raise errors.HTTPGone(str(ex))
def get_blocks_generator(self, marker, limit): return (BlockStorage(Vault.get(self.vault_id), storage_block_id) for storage_block_id in deuce.storage_driver. get_vault_block_list(self.vault_id, limit, marker))
def get(vault_id): vault = Vault.get(vault_id) return BlockStorage(vault) if vault else None