class BucketsResource(api_base.BaseResource): """API resource for realizing CRUD operations for buckets.""" view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:create_cleartext_documents') def on_put(self, req, resp, bucket_name=None): documents = self.from_yaml(req, expect_list=True, allow_empty=True) # NOTE: Must validate documents before doing policy enforcement, # because we expect certain formatting of the documents while doing # policy enforcement. If any documents fail basic schema validaiton # raise an exception immediately. data_schemas = db_api.revision_documents_get( schema=types.DATA_SCHEMA_SCHEMA, deleted=False) try: doc_validator = document_validation.DocumentValidation( documents, data_schemas, pre_validate=True) validations = doc_validator.validate_all() except deckhand_errors.InvalidDocumentFormat as e: with excutils.save_and_reraise_exception(): LOG.exception(e.format_message()) for document in documents: if secrets_manager.SecretsManager.requires_encryption(document): policy.conditional_authorize( 'deckhand:create_encrypted_documents', req.context) break try: documents = self._prepare_secret_documents(documents) except deckhand_errors.BarbicanException: with excutils.save_and_reraise_exception(): LOG.error('An unknown exception occurred while trying to store' ' a secret in Barbican.') created_documents = self._create_revision_documents( bucket_name, documents, validations) resp.body = self.view_builder.list(created_documents) resp.status = falcon.HTTP_200 def _prepare_secret_documents(self, documents): # Encrypt data for secret documents, if any. for document in documents: if secrets_manager.SecretsManager.requires_encryption(document): secret_ref = secrets_manager.SecretsManager.create(document) document['data'] = secret_ref return documents def _create_revision_documents(self, bucket_name, documents, validations): try: created_documents = db_api.documents_create( bucket_name, documents, validations=validations) except (deckhand_errors.DuplicateDocumentExists, deckhand_errors.SingletonDocumentConflict) as e: with excutils.save_and_reraise_exception(): LOG.exception(e.format_message()) return created_documents
class RevisionDocumentsResource(api_base.BaseResource): """API resource for realizing revision documents endpoint.""" view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.abstract', 'metadata.layeringDefinition.layer', 'metadata.label', 'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets']) def on_get(self, req, resp, revision_id): """Returns all documents for a `revision_id`. Returns a multi-document YAML response containing all the documents matching the filters specified via query string parameters. Returned documents will be as originally posted with no substitutions or layering applied. """ include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) order_by = req.params.pop('order', None) sort_by = req.params.pop('sort', None) limit = req.params.pop('limit', None) cleartext_secrets = req.get_param_as_bool('cleartext-secrets') if cleartext_secrets is None: cleartext_secrets = True req.params.pop('cleartext-secrets', None) filters = req.params.copy() filters['metadata.storagePolicy'] = ['cleartext'] if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') filters['deleted'] = False # Never return deleted documents to user. try: documents = db_api.revision_documents_get( revision_id, **filters) except errors.RevisionNotFound as e: LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) if not cleartext_secrets: documents = utils.redact_documents(documents) # Sorts by creation date by default. documents = utils.multisort(documents, sort_by, order_by) if limit is not None: documents = documents[:limit] resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(documents)
class RenderedDocumentsResource(api_base.BaseResource): """API resource for realizing rendered documents endpoint. Rendered documents are also revision documents, but unlike revision documents, they are finalized documents, having undergone secret substitution and document layering. Returns a multi-document YAML response containing all the documents matching the filters specified via query string parameters. Returned documents will have secrets substituted into them and be layered with other documents in the revision, in accordance with the ``LayeringPolicy`` that currently exists in the system. """ view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params(['schema', 'metadata.name', 'metadata.label']) def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = sanitized_params.copy() filters['metadata.storagePolicy'] = ['cleartext'] if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') try: documents = db_api.revision_get_documents(revision_id, **filters) except errors.RevisionNotFound as e: LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) # TODO(fmontei): Currently the only phase of rendering that is # performed is secret substitution, which can be done in any randomized # order. However, secret substitution logic will have to be moved into # a separate module that handles layering alongside substitution once # layering has been fully integrated into this endpoint. secrets_substitution = secrets_manager.SecretsSubstitution(documents) try: rendered_documents = secrets_substitution.substitute_all() except errors.DocumentNotFound as e: LOG.error('Failed to render the documents because a secret ' 'document could not be found.') LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents)
class RevisionDocumentsResource(api_base.BaseResource): """API resource for realizing revision documents endpoint.""" view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.abstract', 'metadata.layeringDefinition.layer', 'metadata.label', 'status.bucket' ]) def on_get(self, req, resp, sanitized_params, revision_id): """Returns all documents for a `revision_id`. Returns a multi-document YAML response containing all the documents matching the filters specified via query string parameters. Returned documents will be as originally posted with no substitutions or layering applied. """ include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = sanitized_params.copy() filters['metadata.storagePolicy'] = ['cleartext'] if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') # Never return deleted documents to user. filters['deleted'] = False try: documents = db_api.revision_get_documents(revision_id, **filters) except errors.RevisionNotFound as e: LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(documents)
class RenderedDocumentsResource(api_base.BaseResource): """API resource for realizing rendered documents endpoint. Rendered documents are also revision documents, but unlike revision documents, they are finalized documents, having undergone secret substitution and document layering. Returns a multi-document YAML response containing all the documents matching the filters specified via query string parameters. Returned documents will have secrets substituted into them and be layered with other documents in the revision, in accordance with the ``LayeringPolicy`` that currently exists in the system. """ view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.label', 'status.bucket', 'order', 'sort' ]) def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = {'metadata.storagePolicy': ['cleartext'], 'deleted': False} if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') documents = self._retrieve_documents_for_rendering( revision_id, **filters) substitution_sources = self._retrieve_substitution_sources() try: # NOTE(fmontei): `validate` is False because documents have already # been pre-validated during ingestion. Documents are post-validated # below, regardless. document_layering = layering.DocumentLayering(documents, substitution_sources, validate=False) rendered_documents = document_layering.render() except (errors.InvalidDocumentLayer, errors.InvalidDocumentParent, errors.IndeterminateDocumentParent, errors.MissingDocumentKey, errors.UnsupportedActionMethod) as e: raise falcon.HTTPBadRequest(description=e.format_message()) except (errors.LayeringPolicyNotFound, errors.SubstitutionSourceNotFound) as e: raise falcon.HTTPConflict(description=e.format_message()) except errors.errors.UnknownSubstitutionError as e: raise falcon.HTTPInternalServerError( description=e.format_message()) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) user_filters = sanitized_params.copy() rendered_documents = [ d for d in rendered_documents if utils.deepfilter(d, **user_filters) ] if sort_by: rendered_documents = utils.multisort(rendered_documents, sort_by, order_by) resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents) self._post_validate(rendered_documents) def _retrieve_documents_for_rendering(self, revision_id, **filters): """Retrieve all necessary documents needed for rendering. If a layering policy isn't found in the current revision, retrieve it in a subsequent call and add it to the list of documents. """ try: documents = db_api.revision_documents_get(revision_id, **filters) except errors.RevisionNotFound as e: LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) if not any([ d['schema'].startswith(types.LAYERING_POLICY_SCHEMA) for d in documents ]): try: layering_policy_filters = { 'deleted': False, 'schema': types.LAYERING_POLICY_SCHEMA } layering_policy = db_api.document_get( **layering_policy_filters) except errors.DocumentNotFound as e: LOG.exception(e.format_message()) else: documents.append(layering_policy) return documents def _retrieve_substitution_sources(self): # Return all concrete documents as potential substitution sources. return db_api.document_get_all( **{'metadata.layeringDefinition.abstract': False}) def _post_validate(self, rendered_documents): # Perform schema validation post-rendering to ensure that rendering # and substitution didn't break anything. data_schemas = db_api.revision_documents_get( schema=types.DATA_SCHEMA_SCHEMA, deleted=False) doc_validator = document_validation.DocumentValidation( rendered_documents, data_schemas) try: validations = doc_validator.validate_all() except errors.InvalidDocumentFormat as e: LOG.error('Failed to post-validate rendered documents.') LOG.exception(e.format_message()) raise falcon.HTTPInternalServerError( description=e.format_message()) else: failed_validations = [ v for v in validations if v['status'] == 'failure' ] if failed_validations: raise falcon.HTTPBadRequest(description=failed_validations)
class BucketsResource(api_base.BaseResource): """API resource for realizing CRUD operations for buckets.""" view_builder = document_view.ViewBuilder() secrets_mgr = secrets_manager.SecretsManager() @policy.authorize('deckhand:create_cleartext_documents') def on_put(self, req, resp, bucket_name=None): document_data = req.stream.read(req.content_length or 0) try: documents = list(yaml.safe_load_all(document_data)) except yaml.YAMLError as e: error_msg = ("Could not parse the document into YAML data. " "Details: %s." % e) LOG.error(error_msg) raise falcon.HTTPBadRequest(description=six.text_type(e)) # NOTE: Must validate documents before doing policy enforcement, # because we expect certain formatting of the documents while doing # policy enforcement. If any documents fail basic schema validaiton # raise an exception immediately. doc_validator = document_validation.DocumentValidation(documents) try: validations = doc_validator.validate_all() except (deckhand_errors.InvalidDocumentFormat, deckhand_errors.InvalidDocumentSchema) as e: LOG.error(e.format_message()) raise falcon.HTTPBadRequest(description=e.format_message()) for document in documents: if document['metadata'].get('storagePolicy') == 'encrypted': policy.conditional_authorize( 'deckhand:create_encrypted_documents', req.context) break self._prepare_secret_documents(documents) created_documents = self._create_revision_documents( bucket_name, documents, validations) if created_documents: resp.body = self.view_builder.list(created_documents) resp.status = falcon.HTTP_200 def _prepare_secret_documents(self, secret_documents): # Encrypt data for secret documents, if any. for document in secret_documents: # TODO(fmontei): Move all of this to document validation directly. if document['metadata'].get('storagePolicy') == 'encrypted': secret_data = self.secrets_mgr.create(document) document['data'] = secret_data elif any([ document['schema'].startswith(t) for t in types.DOCUMENT_SECRET_TYPES ]): document['data'] = {'secret': document['data']} def _create_revision_documents(self, bucket_name, documents, validations): try: created_documents = db_api.documents_create( bucket_name, documents, validations=validations) except (deckhand_errors.DocumentExists, deckhand_errors.SingletonDocumentConflict) as e: raise falcon.HTTPConflict(description=e.format_message()) except Exception as e: raise falcon.HTTPInternalServerError(description=six.text_type(e)) return created_documents
def setUp(self): super(TestDocumentViews, self).setUp() self.view_builder = document.ViewBuilder()
def setUp(self): super(TestDocumentViews, self).setUp() self.view_builder = document.ViewBuilder() self.factory = factories.ValidationPolicyFactory()
class RenderedDocumentsResource(api_base.BaseResource): """API resource for realizing rendered documents endpoint. Rendered documents are also revision documents, but unlike revision documents, they are finalized documents, having undergone secret substitution and document layering. Returns a multi-document YAML response containing all the documents matching the filters specified via query string parameters. Returned documents will have secrets substituted into them and be layered with other documents in the revision, in accordance with the ``LayeringPolicy`` that currently exists in the system. """ view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.layer', 'metadata.label', 'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets']) def on_get(self, req, resp, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = { 'metadata.storagePolicy': ['cleartext'], 'deleted': False } if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') cleartext_secrets = req.get_param_as_bool('cleartext-secrets') if cleartext_secrets is None: cleartext_secrets = True req.params.pop('cleartext-secrets', None) rendered_documents, cache_hit = common.get_rendered_docs( revision_id, cleartext_secrets, **filters) # If the rendered documents result set is cached, then post-validation # for that result set has already been performed successfully, so it # can be safely skipped over as an optimization. if not cache_hit: data_schemas = db_api.revision_documents_get( schema=types.DATA_SCHEMA_SCHEMA, deleted=False) validator = document_validation.DocumentValidation( rendered_documents, data_schemas, pre_validate=False) engine.validate_render(revision_id, rendered_documents, validator) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = req.params.pop('order', None) sort_by = req.params.pop('sort', None) limit = req.params.pop('limit', None) user_filters = req.params.copy() if not cleartext_secrets: rendered_documents = utils.redact_documents(rendered_documents) rendered_documents = [ d for d in rendered_documents if utils.deepfilter( d, **user_filters)] if sort_by: rendered_documents = utils.multisort( rendered_documents, sort_by, order_by) if limit is not None: rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents)
class RenderedDocumentsResource(api_base.BaseResource): """API resource for realizing rendered documents endpoint. Rendered documents are also revision documents, but unlike revision documents, they are finalized documents, having undergone secret substitution and document layering. Returns a multi-document YAML response containing all the documents matching the filters specified via query string parameters. Returned documents will have secrets substituted into them and be layered with other documents in the revision, in accordance with the ``LayeringPolicy`` that currently exists in the system. """ view_builder = document_view.ViewBuilder() @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.layer', 'metadata.label', 'status.bucket', 'order', 'sort', 'limit']) def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = { 'metadata.storagePolicy': ['cleartext'], 'deleted': False } if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') rendered_documents = common.get_rendered_docs(revision_id, **filters) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) limit = sanitized_params.pop('limit', None) user_filters = sanitized_params.copy() rendered_documents = [ d for d in rendered_documents if utils.deepfilter( d, **user_filters)] if sort_by: rendered_documents = utils.multisort( rendered_documents, sort_by, order_by) if limit is not None: rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 self._post_validate(rendered_documents) resp.body = self.view_builder.list(rendered_documents) def _post_validate(self, rendered_documents): # Perform schema validation post-rendering to ensure that rendering # and substitution didn't break anything. data_schemas = db_api.revision_documents_get( schema=types.DATA_SCHEMA_SCHEMA, deleted=False) doc_validator = document_validation.DocumentValidation( rendered_documents, data_schemas, pre_validate=False) try: validations = doc_validator.validate_all() except errors.InvalidDocumentFormat as e: # Post-rendering validation errors likely indicate an internal # rendering bug, so override the default code to 500. e.code = 500 LOG.error('Failed to post-validate rendered documents.') LOG.exception(e.format_message()) raise e else: error_list = [] for validation in validations: if validation['status'] == 'failure': error_list.extend([ vm.ValidationMessage( message=error['message'], name=vm.DOCUMENT_POST_RENDERING_FAILURE, doc_schema=error['schema'], doc_name=error['name'], doc_layer=error['layer'], diagnostic={ k: v for k, v in error.items() if k in ( 'schema_path', 'validation_schema', 'error_section' ) } ) for error in validation['errors'] ]) if error_list: raise errors.InvalidDocumentFormat( error_list=error_list, reason='Validation' )