class BucketsResource(api_base.BaseResource):
"""API resource for realizing CRUD operations for buckets."""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:create_cleartext_documents')
def on_put(self, req, resp, bucket_name=None):
documents = self.from_yaml(req, expect_list=True, allow_empty=True)
# NOTE: Must validate documents before doing policy enforcement,
# because we expect certain formatting of the documents while doing
# policy enforcement. If any documents fail basic schema validaiton
# raise an exception immediately.
data_schemas = db_api.revision_documents_get(
schema=types.DATA_SCHEMA_SCHEMA, deleted=False)
try:
doc_validator = document_validation.DocumentValidation(
documents, data_schemas, pre_validate=True)
validations = doc_validator.validate_all()
except deckhand_errors.InvalidDocumentFormat as e:
with excutils.save_and_reraise_exception():
LOG.exception(e.format_message())
for document in documents:
if secrets_manager.SecretsManager.requires_encryption(document):
policy.conditional_authorize(
'deckhand:create_encrypted_documents', req.context)
break
try:
documents = self._prepare_secret_documents(documents)
except deckhand_errors.BarbicanException:
with excutils.save_and_reraise_exception():
LOG.error('An unknown exception occurred while trying to store'
' a secret in Barbican.')
created_documents = self._create_revision_documents(
bucket_name, documents, validations)
resp.body = self.view_builder.list(created_documents)
resp.status = falcon.HTTP_200
def _prepare_secret_documents(self, documents):
# Encrypt data for secret documents, if any.
for document in documents:
if secrets_manager.SecretsManager.requires_encryption(document):
secret_ref = secrets_manager.SecretsManager.create(document)
document['data'] = secret_ref
return documents
def _create_revision_documents(self, bucket_name, documents, validations):
try:
created_documents = db_api.documents_create(
bucket_name, documents, validations=validations)
except (deckhand_errors.DuplicateDocumentExists,
deckhand_errors.SingletonDocumentConflict) as e:
with excutils.save_and_reraise_exception():
LOG.exception(e.format_message())
return created_documents
class RevisionDocumentsResource(api_base.BaseResource):
"""API resource for realizing revision documents endpoint."""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
'metadata.layeringDefinition.layer', 'metadata.label',
'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets'])
def on_get(self, req, resp, revision_id):
"""Returns all documents for a `revision_id`.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will be as originally posted with no substitutions or
layering applied.
"""
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
order_by = req.params.pop('order', None)
sort_by = req.params.pop('sort', None)
limit = req.params.pop('limit', None)
cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
if cleartext_secrets is None:
cleartext_secrets = True
req.params.pop('cleartext-secrets', None)
filters = req.params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
filters['deleted'] = False # Never return deleted documents to user.
try:
documents = db_api.revision_documents_get(
revision_id, **filters)
except errors.RevisionNotFound as e:
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
if not cleartext_secrets:
documents = utils.redact_documents(documents)
# Sorts by creation date by default.
documents = utils.multisort(documents, sort_by, order_by)
if limit is not None:
documents = documents[:limit]
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(documents)
class RenderedDocumentsResource(api_base.BaseResource):
"""API resource for realizing rendered documents endpoint.
Rendered documents are also revision documents, but unlike revision
documents, they are finalized documents, having undergone secret
substitution and document layering.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will have secrets substituted into them and be layered with
other documents in the revision, in accordance with the ``LayeringPolicy``
that currently exists in the system.
"""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params(['schema', 'metadata.name', 'metadata.label'])
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
try:
documents = db_api.revision_get_documents(revision_id, **filters)
except errors.RevisionNotFound as e:
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
# TODO(fmontei): Currently the only phase of rendering that is
# performed is secret substitution, which can be done in any randomized
# order. However, secret substitution logic will have to be moved into
# a separate module that handles layering alongside substitution once
# layering has been fully integrated into this endpoint.
secrets_substitution = secrets_manager.SecretsSubstitution(documents)
try:
rendered_documents = secrets_substitution.substitute_all()
except errors.DocumentNotFound as e:
LOG.error('Failed to render the documents because a secret '
'document could not be found.')
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
class RevisionDocumentsResource(api_base.BaseResource):
"""API resource for realizing revision documents endpoint."""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
'metadata.layeringDefinition.layer', 'metadata.label', 'status.bucket'
])
def on_get(self, req, resp, sanitized_params, revision_id):
"""Returns all documents for a `revision_id`.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will be as originally posted with no substitutions or
layering applied.
"""
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
# Never return deleted documents to user.
filters['deleted'] = False
try:
documents = db_api.revision_get_documents(revision_id, **filters)
except errors.RevisionNotFound as e:
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(documents)
class RenderedDocumentsResource(api_base.BaseResource):
"""API resource for realizing rendered documents endpoint.
Rendered documents are also revision documents, but unlike revision
documents, they are finalized documents, having undergone secret
substitution and document layering.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will have secrets substituted into them and be layered with
other documents in the revision, in accordance with the ``LayeringPolicy``
that currently exists in the system.
"""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.label', 'status.bucket', 'order',
'sort'
])
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {'metadata.storagePolicy': ['cleartext'], 'deleted': False}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
documents = self._retrieve_documents_for_rendering(
revision_id, **filters)
substitution_sources = self._retrieve_substitution_sources()
try:
# NOTE(fmontei): `validate` is False because documents have already
# been pre-validated during ingestion. Documents are post-validated
# below, regardless.
document_layering = layering.DocumentLayering(documents,
substitution_sources,
validate=False)
rendered_documents = document_layering.render()
except (errors.InvalidDocumentLayer, errors.InvalidDocumentParent,
errors.IndeterminateDocumentParent, errors.MissingDocumentKey,
errors.UnsupportedActionMethod) as e:
raise falcon.HTTPBadRequest(description=e.format_message())
except (errors.LayeringPolicyNotFound,
errors.SubstitutionSourceNotFound) as e:
raise falcon.HTTPConflict(description=e.format_message())
except errors.errors.UnknownSubstitutionError as e:
raise falcon.HTTPInternalServerError(
description=e.format_message())
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
user_filters = sanitized_params.copy()
rendered_documents = [
d for d in rendered_documents
if utils.deepfilter(d, **user_filters)
]
if sort_by:
rendered_documents = utils.multisort(rendered_documents, sort_by,
order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def _retrieve_documents_for_rendering(self, revision_id, **filters):
"""Retrieve all necessary documents needed for rendering. If a layering
policy isn't found in the current revision, retrieve it in a subsequent
call and add it to the list of documents.
"""
try:
documents = db_api.revision_documents_get(revision_id, **filters)
except errors.RevisionNotFound as e:
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
if not any([
d['schema'].startswith(types.LAYERING_POLICY_SCHEMA)
for d in documents
]):
try:
layering_policy_filters = {
'deleted': False,
'schema': types.LAYERING_POLICY_SCHEMA
}
layering_policy = db_api.document_get(
**layering_policy_filters)
except errors.DocumentNotFound as e:
LOG.exception(e.format_message())
else:
documents.append(layering_policy)
return documents
def _retrieve_substitution_sources(self):
# Return all concrete documents as potential substitution sources.
return db_api.document_get_all(
**{'metadata.layeringDefinition.abstract': False})
def _post_validate(self, rendered_documents):
# Perform schema validation post-rendering to ensure that rendering
# and substitution didn't break anything.
data_schemas = db_api.revision_documents_get(
schema=types.DATA_SCHEMA_SCHEMA, deleted=False)
doc_validator = document_validation.DocumentValidation(
rendered_documents, data_schemas)
try:
validations = doc_validator.validate_all()
except errors.InvalidDocumentFormat as e:
LOG.error('Failed to post-validate rendered documents.')
LOG.exception(e.format_message())
raise falcon.HTTPInternalServerError(
description=e.format_message())
else:
failed_validations = [
v for v in validations if v['status'] == 'failure'
]
if failed_validations:
raise falcon.HTTPBadRequest(description=failed_validations)
class BucketsResource(api_base.BaseResource):
"""API resource for realizing CRUD operations for buckets."""
view_builder = document_view.ViewBuilder()
secrets_mgr = secrets_manager.SecretsManager()
@policy.authorize('deckhand:create_cleartext_documents')
def on_put(self, req, resp, bucket_name=None):
document_data = req.stream.read(req.content_length or 0)
try:
documents = list(yaml.safe_load_all(document_data))
except yaml.YAMLError as e:
error_msg = ("Could not parse the document into YAML data. "
"Details: %s." % e)
LOG.error(error_msg)
raise falcon.HTTPBadRequest(description=six.text_type(e))
# NOTE: Must validate documents before doing policy enforcement,
# because we expect certain formatting of the documents while doing
# policy enforcement. If any documents fail basic schema validaiton
# raise an exception immediately.
doc_validator = document_validation.DocumentValidation(documents)
try:
validations = doc_validator.validate_all()
except (deckhand_errors.InvalidDocumentFormat,
deckhand_errors.InvalidDocumentSchema) as e:
LOG.error(e.format_message())
raise falcon.HTTPBadRequest(description=e.format_message())
for document in documents:
if document['metadata'].get('storagePolicy') == 'encrypted':
policy.conditional_authorize(
'deckhand:create_encrypted_documents', req.context)
break
self._prepare_secret_documents(documents)
created_documents = self._create_revision_documents(
bucket_name, documents, validations)
if created_documents:
resp.body = self.view_builder.list(created_documents)
resp.status = falcon.HTTP_200
def _prepare_secret_documents(self, secret_documents):
# Encrypt data for secret documents, if any.
for document in secret_documents:
# TODO(fmontei): Move all of this to document validation directly.
if document['metadata'].get('storagePolicy') == 'encrypted':
secret_data = self.secrets_mgr.create(document)
document['data'] = secret_data
elif any([
document['schema'].startswith(t)
for t in types.DOCUMENT_SECRET_TYPES
]):
document['data'] = {'secret': document['data']}
def _create_revision_documents(self, bucket_name, documents, validations):
try:
created_documents = db_api.documents_create(
bucket_name, documents, validations=validations)
except (deckhand_errors.DocumentExists,
deckhand_errors.SingletonDocumentConflict) as e:
raise falcon.HTTPConflict(description=e.format_message())
except Exception as e:
raise falcon.HTTPInternalServerError(description=six.text_type(e))
return created_documents
def setUp(self):
super(TestDocumentViews, self).setUp()
self.view_builder = document.ViewBuilder()
def setUp(self):
super(TestDocumentViews, self).setUp()
self.view_builder = document.ViewBuilder()
self.factory = factories.ValidationPolicyFactory()
class RenderedDocumentsResource(api_base.BaseResource):
"""API resource for realizing rendered documents endpoint.
Rendered documents are also revision documents, but unlike revision
documents, they are finalized documents, having undergone secret
substitution and document layering.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will have secrets substituted into them and be layered with
other documents in the revision, in accordance with the ``LayeringPolicy``
that currently exists in the system.
"""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.layer',
'metadata.label', 'status.bucket', 'order', 'sort', 'limit',
'cleartext-secrets'])
def on_get(self, req, resp, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {
'metadata.storagePolicy': ['cleartext'],
'deleted': False
}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
if cleartext_secrets is None:
cleartext_secrets = True
req.params.pop('cleartext-secrets', None)
rendered_documents, cache_hit = common.get_rendered_docs(
revision_id, cleartext_secrets, **filters)
# If the rendered documents result set is cached, then post-validation
# for that result set has already been performed successfully, so it
# can be safely skipped over as an optimization.
if not cache_hit:
data_schemas = db_api.revision_documents_get(
schema=types.DATA_SCHEMA_SCHEMA, deleted=False)
validator = document_validation.DocumentValidation(
rendered_documents, data_schemas, pre_validate=False)
engine.validate_render(revision_id, rendered_documents, validator)
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = req.params.pop('order', None)
sort_by = req.params.pop('sort', None)
limit = req.params.pop('limit', None)
user_filters = req.params.copy()
if not cleartext_secrets:
rendered_documents = utils.redact_documents(rendered_documents)
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
if limit is not None:
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
class RenderedDocumentsResource(api_base.BaseResource):
"""API resource for realizing rendered documents endpoint.
Rendered documents are also revision documents, but unlike revision
documents, they are finalized documents, having undergone secret
substitution and document layering.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will have secrets substituted into them and be layered with
other documents in the revision, in accordance with the ``LayeringPolicy``
that currently exists in the system.
"""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.layer',
'metadata.label', 'status.bucket', 'order', 'sort', 'limit'])
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {
'metadata.storagePolicy': ['cleartext'],
'deleted': False
}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
rendered_documents = common.get_rendered_docs(revision_id, **filters)
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
limit = sanitized_params.pop('limit', None)
user_filters = sanitized_params.copy()
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
if limit is not None:
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
self._post_validate(rendered_documents)
resp.body = self.view_builder.list(rendered_documents)
def _post_validate(self, rendered_documents):
# Perform schema validation post-rendering to ensure that rendering
# and substitution didn't break anything.
data_schemas = db_api.revision_documents_get(
schema=types.DATA_SCHEMA_SCHEMA, deleted=False)
doc_validator = document_validation.DocumentValidation(
rendered_documents, data_schemas, pre_validate=False)
try:
validations = doc_validator.validate_all()
except errors.InvalidDocumentFormat as e:
# Post-rendering validation errors likely indicate an internal
# rendering bug, so override the default code to 500.
e.code = 500
LOG.error('Failed to post-validate rendered documents.')
LOG.exception(e.format_message())
raise e
else:
error_list = []
for validation in validations:
if validation['status'] == 'failure':
error_list.extend([
vm.ValidationMessage(
message=error['message'],
name=vm.DOCUMENT_POST_RENDERING_FAILURE,
doc_schema=error['schema'],
doc_name=error['name'],
doc_layer=error['layer'],
diagnostic={
k: v for k, v in error.items() if k in (
'schema_path',
'validation_schema',
'error_section'
)
}
)
for error in validation['errors']
])
if error_list:
raise errors.InvalidDocumentFormat(
error_list=error_list,
reason='Validation'
)