def test_vault_crud(self):
vault_id = self.create_vault_id()
v = Vault.get(vault_id)
assert v is None
v = Vault.create(vault_id)
assert v is not None
v.delete()
v = Vault.get(vault_id)
assert v is None
def test_vault_crud(self):
vault_id = self.create_vault_id()
v = Vault.get(vault_id)
assert v is None
v = Vault.create(vault_id)
assert v is not None
v.delete()
v = Vault.get(vault_id)
assert v is None
def post(self, vault_id, file_id=None):
"""Initializes a new file. The location of
the new file is returned in the Location
header
"""
vault = Vault.get(vault_id)
# caller tried to post to a vault that
# does not exist
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
abort(400, headers={"Transaction-ID":
deuce.context.transaction.request_id})
# overload to use the same end-point for creating a new file
# and assigning blocks to a file that is in progress
if file_id is not None:
return self._assign(vault, vault_id, file_id)
file = vault.create_file()
response.headers["Location"] = "files/%s" % file.file_id
response.status_code = 201 # Created
logger.info('File [{0}] created'.
format(response.headers["Location"]))
def get_one(self, vault_id, file_id):
"""Fetches, re-assembles and streams a single
file out of Deuce"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
abort(404, headers={"Transaction-ID":
deuce.context.transaction.request_id})
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
abort(404, headers={"Transaction-ID":
deuce.context.transaction.request_id})
if not f.finalized:
abort(412, headers={"Transaction-ID":
deuce.context.transaction.request_id})
block_gen = deuce.metadata_driver.create_file_block_generator(
vault_id, file_id)
block_ids = [block[0] for block in sorted(block_gen,
key=lambda block: block[1])]
objs = vault.get_blocks_generator(block_ids)
response.content_length = vault.get_file_length(file_id)
response.body_file = FileCat(objs)
response.status_code = 200
def get_all(self, vault_id):
vault = Vault.get(request.project_id, vault_id)
if not vault:
abort(404)
inmarker = request.params.get('marker')
limit = int(request.params.get('limit',
conf.api_configuration.max_returned_num))
# The +1 is to fetch one past the user's
# requested limit so that we can determine
# if the list was truncated or not
files = vault.get_files(inmarker, limit + 1)
resp = list(files)
# Note: the list may not actually be truncated
truncated = len(resp) == limit + 1
outmarker = resp.pop().file_id if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs(request.url, query_args)
response.headers["X-Next-Batch"] = returl
return resp
def on_get(self, req, resp, vault_id):
"""List the blocks in the vault from storage-alone
"""
vault = Vault.get(vault_id)
if vault is None:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
inmarker = req.get_param('marker') if req.get_param('marker') else None
limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \
conf.api_configuration.default_returned_num
# We actually fetch the user's requested
# limit +1 to detect if the list is being
# truncated or not.
storage = BlockStorage.get(vault_id)
storage_blocks = storage.get_blocks_generator(inmarker, limit + 1)
responses = list(storage_blocks)
# Was the list truncated? See note above about +1
truncated = len(responses) > 0 and len(responses) == limit + 1
outmarker = responses.pop().storage_block_id if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header("X-Next-Batch", returl)
resp.body = json.dumps(
[response.storage_block_id for response in responses])
def on_post(self, req, resp, vault_id, file_id):
"""This endpoint Assigns blocks to files
"""
vault = Vault.get(vault_id)
# caller tried to post to a vault that
# does not exist
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPBadRequestAPI('Vault does not exist')
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
raise errors.HTTPNotFound
if f.finalized:
logger.error('Finalized file [{0}] '
'cannot be modified'.format(file_id))
raise errors.HTTPConflict('Finalized file cannot be modified')
body = req.stream.read(req.content_length)
# TODO (TheSriram): Validate payload
payload = json.loads(body.decode())
block_ids, offsets = zip(*payload)
missing_blocks = deuce.metadata_driver.has_blocks(vault_id, block_ids)
deuce.metadata_driver.assign_blocks(vault_id, file_id, block_ids,
offsets)
resp.body = json.dumps(missing_blocks)
def on_post(self, req, resp, vault_id, file_id):
"""This endpoint Assigns blocks to files
"""
vault = Vault.get(vault_id)
# caller tried to post to a vault that
# does not exist
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPBadRequestAPI('Vault does not exist')
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
raise errors.HTTPNotFound
if f.finalized:
logger.error('Finalized file [{0}] '
'cannot be modified'.format(file_id))
raise errors.HTTPConflict('Finalized file cannot be modified')
body = req.stream.read(req.content_length)
# TODO (TheSriram): Validate payload
payload = json.loads(body.decode())
block_ids, offsets = zip(*payload)
missing_blocks = deuce.metadata_driver.has_blocks(vault_id, block_ids,
check_status=True)
deuce.metadata_driver.assign_blocks(vault_id, file_id, block_ids,
offsets)
resp.body = json.dumps(missing_blocks)
def get_all(self, vault_id, file_id):
vault = Vault.get(request.project_id, vault_id)
assert vault is not None
f = vault.get_file(file_id)
if not f:
abort(404)
inmarker = int(request.params.get('marker', 0))
limit = int(request.params.get('limit',
conf.api_configuration.max_returned_num))
# Get the block generator from the metadata driver.
# Note: +1 on limit is to fetch one past the limt
# for the purpose of determining if the
# list was truncated
retblks = deuce.metadata_driver.create_file_block_generator(
request.project_id, vault_id, file_id, inmarker, limit + 1)
resp = list(retblks)
truncated = len(resp) > 0 and len(resp) == limit + 1
outmarker = resp.pop()[1] if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs(request.url, query_args)
response.headers["X-Next-Batch"] = returl
return resp
def on_delete(self, req, resp, vault_id, block_id):
"""Unregisters a block_id from a given vault_id in
the storage and metadata
"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
resp.status = falcon.HTTP_404
return
try:
response = vault.delete_block(vault_id, block_id)
except ConstraintError as ex:
logger.error(json.dumps(ex.args))
raise errors.HTTPConflict(json.dumps(ex.args))
except Exception as ex: # pragma: no cover
logger.error(ex)
raise errors.HTTPServiceUnavailable
else:
if response:
logger.info('block [{0}] deleted from vault {1}'.format(
block_id, vault_id))
resp.status = falcon.HTTP_204
else:
logger.error('block [{0}] does not exist'.format(block_id))
raise errors.HTTPNotFound
def on_put(self, req, resp, vault_id, block_id):
"""Uploads a block into Deuce. The URL of the block
is returned in the Location header
"""
vault = Vault.get(vault_id)
try:
retval, storage_id = vault.put_block(
block_id, req.stream.read(), req.content_length)
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
block = vault.get_block(block_id)
ref_cnt = 0
ref_mod = 0
if retval:
ref_cnt = block.get_ref_count()
ref_mod = block.get_ref_modified()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
resp.set_header('X-Ref-Modified', str(ref_mod))
resp.status = (
falcon.HTTP_201 if retval is True else falcon.HTTP_500)
logger.info('block [{0}] added [{1}]'.format(block_id, storage_id))
except ValueError as e:
raise errors.HTTPPreconditionFailed('hash error')
except BufferError as e:
raise errors.HTTPPreconditionFailed(
'content length did not match data length')
def on_get(self, req, resp, vault_id):
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
# NOTE(TheSriram): get_param(param) automatically returns None
# if param is not present
inmarker = req.get_param('marker')
limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \
conf.api_configuration.default_returned_num
# We actually fetch the user's requested
# limit +1 to detect if the list is being
# truncated or not.
blocks = vault.get_blocks(inmarker, limit + 1)
# List the blocks into JSON and return.
# TODO: figure out a way to stream this back(??)
responses = list(blocks)
# Was the list truncated? See note above about +1
truncated = len(responses) > 0 and len(responses) == limit + 1
outmarker = responses.pop().metadata_block_id if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header("X-Next-Batch", returl)
resp.body = json.dumps([response.metadata_block_id
for response in responses])
def on_delete(self, req, resp, vault_id, block_id):
"""Unregisters a block_id from a given vault_id in
the storage and metadata
"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
resp.status = falcon.HTTP_404
return
try:
response = vault.delete_block(vault_id, block_id)
except ConstraintError as ex:
logger.error(json.dumps(ex.args))
raise errors.HTTPConflict(json.dumps(ex.args))
except Exception as ex: # pragma: no cover
logger.error(ex)
raise errors.HTTPServiceUnavailable
else:
if response:
logger.info('block [{0}] deleted from vault {1}'
.format(block_id, vault_id))
resp.status = falcon.HTTP_204
else:
logger.error('block [{0}] does not exist'.format(block_id))
raise errors.HTTPNotFound
def on_put(self, req, resp, vault_id, block_id):
"""Uploads a block into Deuce. The URL of the block
is returned in the Location header
"""
vault = Vault.get(vault_id)
try:
retval, storage_id = vault.put_block(block_id, req.stream.read(),
req.content_length)
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
block = vault.get_block(block_id)
ref_cnt = 0
ref_mod = 0
if retval:
ref_cnt = block.get_ref_count()
ref_mod = block.get_ref_modified()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
resp.set_header('X-Ref-Modified', str(ref_mod))
resp.status = (falcon.HTTP_201
if retval is True else falcon.HTTP_500)
logger.info('block [{0}] added [{1}]'.format(block_id, storage_id))
except ValueError as e:
raise errors.HTTPPreconditionFailed('hash error')
except BufferError as e:
raise errors.HTTPPreconditionFailed(
'content length did not match data length')
def on_get(self, req, resp, vault_id):
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
# NOTE(TheSriram): get_param(param) automatically returns None
# if param is not present
inmarker = req.get_param('marker')
limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \
conf.api_configuration.default_returned_num
# We actually fetch the user's requested
# limit +1 to detect if the list is being
# truncated or not.
blocks = vault.get_blocks(inmarker, limit + 1)
# List the blocks into JSON and return.
# TODO: figure out a way to stream this back(??)
responses = list(blocks)
# Was the list truncated? See note above about +1
truncated = len(responses) > 0 and len(responses) == limit + 1
outmarker = responses.pop().metadata_block_id if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header("X-Next-Batch", returl)
resp.body = json.dumps(
[response.metadata_block_id for response in responses])
def on_post(self, req, resp, vault_id):
vault = Vault.get(vault_id)
try:
unpacked = msgpack.unpackb(req.stream.read())
if not isinstance(unpacked, dict):
raise TypeError
else:
block_ids = list(unpacked.keys())
block_datas = list(unpacked.values())
try:
retval = vault.put_async_block(block_ids, block_datas)
if retval:
resp.status = falcon.HTTP_201
else:
raise errors.HTTPInternalServerError('Block '
'Post Failed')
logger.info('blocks [{0}] added'.format(block_ids))
except ValueError:
raise errors.HTTPPreconditionFailed('hash error')
except (TypeError, ValueError):
logger.error('Request Body not well formed '
'for posting multiple blocks to {0}'.format(vault_id))
raise errors.HTTPBadRequestBody("Request Body not well formed")
def on_get(self, req, resp):
# NOTE(TheSriram): get_param(param) automatically returns None
# if param is not present
inmarker = req.get_param('marker')
limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \
conf.api_configuration.default_returned_num
vaultlist = Vault.get_vaults_generator(
inmarker, limit + 1)
response = list(vaultlist)
if not response:
resp.body = json.dumps([])
# Note: the list may not actually be truncated
truncated = len(response) == limit + 1
outmarker = response.pop() if truncated else None
# Set x-next-batch resp header.
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header(name="X-Next-Batch", value=returl)
# Set return json for vault URLs.
p = urlparse(req.url)
resp.body = json.dumps(dict(six.moves.map(lambda vaultname:
(vaultname, {"url": p.scheme +
'://' + p.netloc + p.path + '/' + vaultname}), response)))
def on_get(self, req, resp, vault_id):
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
# NOTE(TheSriram): get_param(param) automatically returns None
# if param is not present
inmarker = req.get_param('marker')
limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') \
else conf.api_configuration.default_returned_num
# The +1 is to fetch one past the user's
# requested limit so that we can determine
# if the list was truncated or not
files = vault.get_files(inmarker, limit + 1)
responses = list(files)
# Note: the list may not actually be truncated
truncated = len(responses) == limit + 1
outmarker = responses.pop().file_id if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header("X-Next-Batch", returl)
resp.body = json.dumps([response.file_id for response in responses])
def on_get(self, req, resp, vault_id):
"""List the blocks in the vault from storage-alone
"""
vault = Vault.get(vault_id)
if vault is None:
logger.error("Vault [{0}] does not exist".format(vault_id))
raise errors.HTTPNotFound
inmarker = req.get_param("marker") if req.get_param("marker") else None
limit = (
req.get_param_as_int("limit")
if req.get_param_as_int("limit")
else conf.api_configuration.default_returned_num
)
# We actually fetch the user's requested
# limit +1 to detect if the list is being
# truncated or not.
storage = BlockStorage.get(vault_id)
storage_blocks = storage.get_blocks_generator(inmarker, limit + 1)
responses = list(storage_blocks)
# Was the list truncated? See note above about +1
truncated = len(responses) > 0 and len(responses) == limit + 1
outmarker = responses.pop().storage_block_id if truncated else None
if outmarker:
query_args = {"marker": outmarker}
query_args["limit"] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header("X-Next-Batch", returl)
resp.body = json.dumps([response.storage_block_id for response in responses])
def on_get(self, req, resp, vault_id, file_id):
"""Fetches, re-assembles and streams a single
file out of Deuce"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
raise errors.HTTPNotFound
if not f.finalized:
raise errors.HTTPConflict('File not Finalized')
block_gen = deuce.metadata_driver.create_file_block_generator(
vault_id, file_id)
block_ids = [block[0] for block in sorted(block_gen,
key=lambda block: block[1])]
objs = vault.get_blocks_generator(block_ids)
# NOTE(TheSriram): falcon 0.2.0 might fix this problem,
# we should be able to set resp.stream to any file like
# object instead of an iterator.
resp.stream = (obj.read() for obj in objs)
resp.status = falcon.HTTP_200
resp.set_header('Content-Length', str(vault.get_file_length(file_id)))
resp.content_type = 'application/octet-stream'
def get_all(self, vault_id):
vault = Vault.get(request.project_id, vault_id)
if not vault:
response.status_code = 404
return
inmarker = request.params.get('marker')
limit = int(request.params.get('limit',
conf.api_configuration.max_returned_num))
# We actually fetch the user's requested
# limit +1 to detect if the list is being
# truncated or not.
blocks = vault.get_blocks(inmarker, limit + 1)
# List the blocks into JSON and return.
# TODO: figure out a way to stream this back(??)
resp = list(blocks)
# Was the list truncated? See note above about +1
truncated = len(resp) > 0 and len(resp) == limit + 1
outmarker = resp.pop().block_id if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs(request.url, query_args)
response.headers["X-Next-Batch"] = returl
return resp
def on_post(self, req, resp, vault_id):
vault = Vault.get(vault_id)
try:
unpacked = msgpack.unpackb(req.stream.read())
if not isinstance(unpacked, dict):
raise TypeError
else:
block_ids = list(unpacked.keys())
block_datas = list(unpacked.values())
try:
retval, retblocks = vault.put_async_block(
block_ids,
block_datas)
if retval:
resp.status = falcon.HTTP_200
resp.body = json.dumps({block_id: storage_id
for block_id, storage_id
in retblocks})
else:
raise errors.HTTPInternalServerError('Block '
'Post Failed')
logger.info('blocks [{0}] added'.format(block_ids))
except ValueError:
raise errors.HTTPPreconditionFailed('hash error')
except (TypeError, ValueError):
logger.error('Request Body not well formed '
'for posting multiple blocks to {0}'.format(vault_id))
raise errors.HTTPBadRequestBody("Request Body not well formed")
def put(self, vault_id, block_id=None):
"""Uploads a block into Deuce. The URL of the block
is returned in the Location header
"""
vault = Vault.get(request.project_id, vault_id)
vault.put_block(
block_id, request.body, request.headers['content-length'])
def on_head(self, req, resp, vault_id):
"""Returns the vault controller object"""
if Vault.get(vault_id):
resp.status = falcon.HTTP_204
else:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
def on_put(self, req, resp, vault_id):
vault = Vault.create(vault_id)
# TODO: Need check and monitor failed vault.
logger.info('Vault [{0}] created'.format(vault_id))
if vault:
resp.status = falcon.HTTP_201
else:
raise errors.HTTPInternalServerError('Vault Creation Failed')
def delete(self, vault_id):
vault = Vault.get(request.project_id, vault_id)
if vault:
vault.delete()
response.status_code = 200
else:
response.status_code = 404
def get_one(self, vault_id):
"""Returns the vault controller object"""
if Vault.get(request.project_id, vault_id):
response.status_code = 200
else:
response.status_code = 404
return None
def on_head(self, req, resp, vault_id, block_id):
"""Checks for the existence of the block in the
metadata storage and if successful check for it in
the storage driver
if it fails we return a 502, otherwise we return
all other headers returned on
GET /v1.0/vaults/{vault_id}/blocks/{block_id}
"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
# This is safe to construct before we check the status of the block
# b/c the constructor does not do anything other than save values
# to itself, no lookups, etc
block = Block(vault_id, block_id)
try:
if not vault.has_block(block_id, check_storage=True):
logger.error('block [{0}] does not exist'.format(block_id))
raise errors.HTTPNotFound
ref_cnt = block.get_ref_count()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
ref_mod = block.get_ref_modified()
resp.set_header('X-Ref-Modified', str(ref_mod))
storage_id = block.get_storage_id()
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
resp.set_header('X-Block-Size',
str(block.get_block_length()))
resp.status = falcon.HTTP_204
except ConsistencyError as ex:
# We have the block in metadata...
# so we can get anything that only touches metadata
ref_cnt = block.get_ref_count()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
ref_mod = block.get_ref_modified()
resp.set_header('X-Ref-Modified', str(ref_mod))
storage_id = block.get_storage_id()
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
# Block-size is retrieved from storage...
logger.error(ex)
raise errors.HTTPGone(str(ex))
def on_head(self, req, resp, vault_id, block_id):
"""Checks for the existence of the block in the
metadata storage and if successful check for it in
the storage driver
if it fails we return a 502, otherwise we return
all other headers returned on
GET /v1.0/vaults/{vault_id}/blocks/{block_id}
"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
# This is safe to construct before we check the status of the block
# b/c the constructor does not do anything other than save values
# to itself, no lookups, etc
block = Block(vault_id, block_id)
try:
if not vault.has_block(block_id, check_storage=True):
logger.error('block [{0}] does not exist'.format(block_id))
raise errors.HTTPNotFound
ref_cnt = block.get_ref_count()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
ref_mod = block.get_ref_modified()
resp.set_header('X-Ref-Modified', str(ref_mod))
storage_id = block.get_storage_id()
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
resp.set_header('X-Block-Size',
str(block.get_block_length()))
resp.status = falcon.HTTP_204
except ConsistencyError as ex:
# We have the block in metadata...
# so we can get anything that only touches metadata
ref_cnt = block.get_ref_count()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
ref_mod = block.get_ref_modified()
resp.set_header('X-Ref-Modified', str(ref_mod))
storage_id = block.get_storage_id()
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
# Block-size is retrieved from storage...
logger.error(ex)
raise errors.HTTPGone(str(ex))
def test_block_crud(self):
vault_id = self.create_vault_id()
v = Vault.create(vault_id)
# Check for blocks, should be none
blocks_gen = v.get_blocks(0, 0)
blocks_list = list(blocks_gen)
assert len(blocks_list) == 0
def on_patch(self, req, resp, vault_id):
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
vault.reset_block_status()
resp.status = falcon.HTTP_204
def test_block_crud(self):
vault_id = self.create_vault_id()
v = Vault.create(vault_id)
# Check for blocks, should be none
blocks_gen = v.get_blocks(0, 0)
blocks_list = list(blocks_gen)
assert len(blocks_list) == 0
def on_get(self, req, resp, vault_id):
"""Returns the statistics on vault controller object"""
vault = Vault.get(vault_id)
if vault:
resp.body = json.dumps(vault.get_vault_statistics())
resp.status = falcon.HTTP_200
else:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
def on_patch(self, req, resp, vault_id):
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
vault.reset_block_status()
resp.status = falcon.HTTP_204
def delete(self, vault_id, file_id):
vault = Vault.get(vault_id)
if not vault:
abort(404)
f = vault.get_file(file_id)
if not f:
abort(404)
vault.delete_file(file_id)
def get_one(self, vault_id):
"""Returns the statistics on vault controller object"""
vault = Vault.get(vault_id)
if vault:
response.status_code = 200
return vault.get_vault_statistics()
else:
logger.error('Vault [{0}] does not exist'.format(vault_id))
response.status_code = 404
return None
def on_get(self, req, resp, vault_id):
"""Returns the statistics on vault controller object"""
vault = Vault.get(vault_id)
if vault:
vault_stats = vault.get_vault_statistics()
resp.body = json.dumps(vault_stats)
resp.status = falcon.HTTP_200
else:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
def on_delete(self, req, resp, vault_id, file_id):
vault = Vault.get(vault_id)
if not vault:
raise errors.HTTPNotFound
f = vault.get_file(file_id)
if not f:
raise errors.HTTPNotFound
vault.delete_file(file_id)
resp.status = falcon.HTTP_204
def head(self, vault_id):
"""Returns the vault controller object"""
if Vault.get(vault_id):
# weblint complains about the content-type header being
# present as pecan doesn't intelligently add it or remove
# it.
del response.headers["Content-Type"]
response.status_code = 204
return response
else:
logger.error('Vault [{0}] does not exist'.format(vault_id))
response.status_code = 404
def put(self, vault_id, block_id=None):
"""Uploads a block into Deuce. The URL of the block
is returned in the Location header
"""
vault = Vault.get(vault_id)
try:
retval = vault.put_block(
block_id, request.body, request.headers['content-length'])
response.status_code = (201 if retval is True else 500)
logger.info('block [{0}] added'.format(block_id))
except ValueError as e:
response.status_code = 412
def on_get(self, req, resp, vault_id):
"""Returns the statistics on vault controller object"""
vault = Vault.get(vault_id)
if vault:
bad_blocks, bad_files = vault.get_vault_health()
response = {
'Vault': vault_id,
'Bad Blocks': bad_blocks,
'Bad Files': bad_files
}
resp.body = json.dumps(response)
else:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
def on_delete(self, req, resp, vault_id):
vault = Vault.get(vault_id)
if vault:
if vault.delete():
logger.info('Vault [{0}] deleted'.format(vault_id))
resp.status = falcon.HTTP_204
else:
logger.info('Vault [{0}] cannot be deleted'.format(vault_id))
raise errors.HTTPConflict('Vault cannot be deleted')
else:
logger.error('Vault [{0}] deletion failed; '
'Vault does not exist'.format(vault_id))
raise errors.HTTPNotFound
def on_get(self, req, resp, vault_id, file_id):
"""Fetches, re-assembles and streams a single
file out of Deuce"""
vault = Vault.get(vault_id)
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
raise errors.HTTPNotFound
if not f.finalized:
raise errors.HTTPConflict('File not Finalized')
block_gen = deuce.metadata_driver.create_file_block_generator(
vault_id, file_id)
block_ids = [block[0] for block in sorted(block_gen,
key=lambda block: block[1])]
objs = vault.get_blocks_generator(block_ids)
# NOTE(TheSriram): falcon 0.2.0 might fix this problem,
# we should be able to set resp.stream to any file like
# object instead of an iterator.
def premature_close(storage_id):
msg = '[{0}/{1}/{2}] is missing data' \
'for storage block {3}'.format(deuce.context.project_id,
vault_id,
file_id,
storage_id)
logger.error(msg)
raise StopIteration(msg)
resp.stream = (obj.read() if obj else premature_close(storageid)
for (storageid, obj) in objs)
resp.status = falcon.HTTP_200
resp.set_header('Content-Length', str(vault.get_file_length(file_id)))
resp.content_type = 'application/octet-stream'
def on_post(self, req, resp, vault_id):
"""Initializes a new file. The location of
the new file is returned in the Location
header
"""
vault = Vault.get(vault_id)
# caller tried to post to a vault that
# does not exist
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPNotFound
file = vault.create_file()
resp.set_header("Location", "{0}/{1}".format(req.url, file.file_id))
resp.set_header("X-File-ID", file.file_id)
resp.status = falcon.HTTP_201 # Created
logger.info('File [{0}] created'.format(file.file_id))
def test_file_crud(self):
vault_id = self.create_vault_id()
v = Vault.create(vault_id)
f = v.create_file()
assert isinstance(f, File)
assert f.vault_id == vault_id
file_id = f.file_id
assert (len(file_id) > 0)
file2 = v.get_file(file_id)
file2_length = v.get_file_length(file_id)
assert isinstance(file2, File)
assert file2.file_id == file_id
assert file2_length == 0
def on_get(self, req, resp, vault_id, file_id):
vault = Vault.get(vault_id)
assert vault is not None
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
raise errors.HTTPNotFound
# NOTE(TheSriram): get_param(param) automatically returns None
# if param is not present
inmarker = req.get_param_as_int('marker')
limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') \
else conf.api_configuration.default_returned_num
# Get the block generator from the metadata driver.
# Note: +1 on limit is to fetch one past the limt
# for the purpose of determining if the
# list was truncated
retblks = deuce.metadata_driver.create_file_block_generator(
vault_id, file_id, inmarker, limit + 1)
responses = list(retblks)
truncated = len(responses) > 0 and len(responses) == limit + 1
outmarker = responses.pop()[1] if truncated else None
if outmarker:
query_args = {'marker': outmarker}
query_args['limit'] = limit
returl = set_qs_on_url(req.url, query_args)
resp.set_header("X-Next-Batch", returl)
resp.body = json.dumps(responses)
def on_post(self, req, resp, vault_id, file_id):
"""This endpoint finalizes a file
"""
vault = Vault.get(vault_id)
# caller tried to post to a vault that
# does not exist
if not vault:
logger.error('Vault [{0}] does not exist'.format(vault_id))
raise errors.HTTPBadRequestAPI('Vault does not exist')
f = vault.get_file(file_id)
if not f:
logger.error('File [{0}] does not exist'.format(file_id))
raise errors.HTTPNotFound
if f.finalized:
logger.error('Finalized file [{0}] '
'cannot be modified'.format(file_id))
raise errors.HTTPConflict('Finalized file cannot be modified')
try:
filesize = int(req.get_header('x-file-length', required=True))
res = deuce.metadata_driver.finalize_file(vault_id, file_id,
filesize)
except Exception as e:
# There are gaps or overlaps in blocks of the file
# The list of errors returns
details = str(e)
logger.error('File [{0}] finalization '
'failed; [{1}]'.format(file_id, details))
raise errors.HTTPConflict(json.dumps(details))
else:
resp.status = falcon.HTTP_200
return
def test_get_nonexistent_block(self):
v = Vault.get('should_not_exist')
assert v is None
def on_get(self, req, resp, vault_id, block_id):
"""Returns a specific block"""
# Step 1: Is the block in our vault store? If not, return 404
# Step 2: Stream the block back to the user
vault = Vault.get(vault_id)
# Existence of the vault should have been confirmed
# in the vault controller
assert vault is not None
try:
block = vault.get_block(block_id)
if block is None:
logger.error('block [{0}] does not exist'.format(block_id))
# We have to do the has_block() check in order to
# differentiate between a 404 and 410 error.
# 404 should be returned if even metadata doesn't know
# about the block; while 410 should be returned if
# metadata knows about the block but it is not found
# in storage. Since we already know the block doesn't
# exist in storage, we can skip the storage check
if vault.has_block(block_id, check_storage=False):
logger.error(
'block [{0}] does not exist (vault check)'.format(
block_id))
raise ConsistencyError(deuce.context.project_id,
vault_id,
block_id,
msg='Block does not exist'
' in Block Storage')
raise errors.HTTPNotFound
ref_cnt = block.get_ref_count()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
ref_mod = block.get_ref_modified()
resp.set_header('X-Ref-Modified', str(ref_mod))
storage_id = block.get_storage_id()
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
resp.stream = block.get_obj()
resp.stream_len = block.get_block_length()
resp.status = falcon.HTTP_200
resp.content_type = 'application/octet-stream'
except ConsistencyError as ex:
# We have the block in metadata...
# so we can get anything that only touches metadata
block = Block(vault_id, block_id)
ref_cnt = block.get_ref_count()
resp.set_header('X-Block-Reference-Count', str(ref_cnt))
ref_mod = block.get_ref_modified()
resp.set_header('X-Ref-Modified', str(ref_mod))
storage_id = block.get_storage_id()
resp.set_header('X-Storage-ID', str(storage_id))
resp.set_header('X-Block-ID', str(block_id))
# Block-size is retrieved from storage...
logger.error(ex)
raise errors.HTTPGone(str(ex))
def get_blocks_generator(self, marker, limit):
return (BlockStorage(Vault.get(self.vault_id), storage_block_id)
for storage_block_id in deuce.storage_driver.
get_vault_block_list(self.vault_id, limit, marker))
def get(vault_id):
vault = Vault.get(vault_id)
return BlockStorage(vault) if vault else None
