def test_secret_substititon_missing_src_path_in_src_doc_raises_exc(self): """Validates that if a secret can't be found in a substitution source document then an exception is raised. """ certificate = self.secrets_factory.gen_test( 'Certificate', 'cleartext', data={}) certificate['metadata']['name'] = 'example-cert' document_mapping = { "_GLOBAL_SUBSTITUTIONS_1_": [{ "dest": { "path": ".chart.values.tls.certificate" }, "src": { "schema": "deckhand/Certificate/v1", "name": "example-cert", "path": ".path-to-nowhere" } }] } payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents( bucket_name, [certificate] + [payload[-1]]) secrets_substitution = secrets_manager.SecretsSubstitution(documents) with testtools.ExpectedException( errors.SubstitutionSourceDataNotFound): next(secrets_substitution.substitute_all(documents))
def test_secret_substitution_missing_encryption_sources_raises_exc(self): """Validate that when ``encryption_sources`` doesn't contain a reference that a ``EncryptionSourceNotFound`` is raised. """ secret_ref = test_utils.rand_barbican_ref() certificate = self.secrets_factory.gen_test( 'Certificate', 'encrypted', data=secret_ref) certificate['metadata']['name'] = 'example-cert' document_mapping = { "_GLOBAL_SUBSTITUTIONS_1_": [{ "dest": { "path": ".chart.values.tls.certificate" }, "src": { "schema": "deckhand/Certificate/v1", "name": "example-cert", "path": ".path-to-nowhere" } }] } payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents( bucket_name, [certificate] + [payload[-1]]) secrets_substitution = secrets_manager.SecretsSubstitution( documents, encryption_sources={'foo': 'bar'}) with testtools.ExpectedException(errors.EncryptionSourceNotFound): next(secrets_substitution.substitute_all(documents))
def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = sanitized_params.copy() filters['metadata.storagePolicy'] = ['cleartext'] if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') try: documents = db_api.revision_get_documents(revision_id, **filters) except errors.RevisionNotFound as e: LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) # TODO(fmontei): Currently the only phase of rendering that is # performed is secret substitution, which can be done in any randomized # order. However, secret substitution logic will have to be moved into # a separate module that handles layering alongside substitution once # layering has been fully integrated into this endpoint. secrets_substitution = secrets_manager.SecretsSubstitution(documents) try: rendered_documents = secrets_substitution.substitute_all() except errors.DocumentNotFound as e: LOG.error('Failed to render the documents because a secret ' 'document could not be found.') LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents)
def _test_secrets_substitution(self, secret_type, expected_exception): secret_ref = test_utils.rand_barbican_ref() certificate = self.secrets_factory.gen_test( 'Certificate', secret_type, data=secret_ref) certificate['metadata']['name'] = 'example-cert' document_mapping = { "_GLOBAL_SUBSTITUTIONS_1_": [{ "dest": { "path": ".chart.values.tls.certificate" }, "src": { "schema": "deckhand/Certificate/v1", "name": "example-cert", "path": "." } }] } payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents( bucket_name, [certificate] + [payload[-1]]) secrets_substitution = secrets_manager.SecretsSubstitution(documents) with testtools.ExpectedException(expected_exception): next(secrets_substitution.substitute_all(documents))
def _apply_substitutions(self, document): try: secrets_substitution = secrets_manager.SecretsSubstitution( document, self._substitution_sources) return secrets_substitution.substitute_all() except errors.SubstitutionDependencyNotFound: LOG.error('Failed to render the documents because a secret ' 'document could not be found.')
def _test_secret_substitution(self, document_mapping, secret_documents, expected_data): payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents(bucket_name, secret_documents + [payload[-1]]) expected_documents = copy.deepcopy([documents[-1]]) expected_documents[0]['data'] = expected_data secret_substitution = secrets_manager.SecretsSubstitution(documents) substituted_docs = secret_substitution.substitute_all() self.assertEqual(expected_documents, substituted_docs)
def _test_doc_substitution(self, document_mapping, substitution_sources, expected_data, encryption_sources=None): payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents( bucket_name, substitution_sources + [payload[-1]]) expected_document = copy.deepcopy(documents[-1]) expected_document['data'] = expected_data secret_substitution = secrets_manager.SecretsSubstitution( encryption_sources=encryption_sources, substitution_sources=substitution_sources) substituted_docs = list(secret_substitution.substitute_all(documents)) self.assertIn(expected_document, substituted_docs)
def _test_doc_substitution(self, document_mapping, secret_documents, expected_data): payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents(bucket_name, secret_documents + [payload[-1]]) expected_document = copy.deepcopy(documents[-1]) expected_document['data'] = expected_data substitution_sources = db_api.document_get_all( **{'metadata.layeringDefinition.abstract': False}) secret_substitution = secrets_manager.SecretsSubstitution( substitution_sources) substituted_docs = list(secret_substitution.substitute_all(documents)) self.assertIn(expected_document, substituted_docs)
def test_doc_substitution_multiple_pattern_substitutions(self): test_yaml = """ --- schema: deckhand/LayeringPolicy/v1 metadata: schema: metadata/Control/v1 name: layering-policy data: layerOrder: - global - site --- schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: ucp-drydock layeringDefinition: abstract: false layer: global storagePolicy: cleartext substitutions: - src: schema: twigleg/CommonAddresses/v1 name: common-addresses path: .genesis.ip dest: path: .values.conf.drydock.maasdriver.maas_api_url pattern: 'MAAS_IP' - src: schema: twigleg/CommonAddresses/v1 name: common-addresses path: .node_ports.maas_api dest: path: .values.conf.drydock.maasdriver.maas_api_url pattern: 'MAAS_PORT' data: values: conf: drydock: maasdriver: maas_api_url: http://MAAS_IP:MAAS_PORT/MAAS/api/2.0/ --- schema: twigleg/CommonAddresses/v1 metadata: schema: metadata/Document/v1 name: common-addresses layeringDefinition: abstract: false layer: site storagePolicy: cleartext data: genesis: ip: 10.24.31.31 node_ports: maas_api: 30001 ... """ documents = list(yaml.safe_load_all(test_yaml)) expected = copy.deepcopy(documents[1]) expected['data']['values']['conf']['drydock']['maasdriver'][ 'maas_api_url'] = 'http://10.24.31.31:30001/MAAS/api/2.0/' secret_substitution = secrets_manager.SecretsSubstitution(documents) substituted_docs = list(secret_substitution.substitute_all(documents)) self.assertEqual(expected, substituted_docs[0])
def __init__(self, documents, validate=True, fail_on_missing_sub_src=True, encryption_sources=None, cleartext_secrets=False): """Contructor for ``DocumentLayering``. :param layering_policy: The document with schema ``deckhand/LayeringPolicy`` needed for layering. :param documents: List of all other documents to be layered together in accordance with the ``layerOrder`` defined by the LayeringPolicy document. :type documents: List[dict] :param validate: Whether to pre-validate documents using built-in schema validation. Skips over externally registered ``DataSchema`` documents to avoid false positives. Default is True. :type validate: bool :param fail_on_missing_sub_src: Whether to fail on a missing substitution source. Default is True. :type fail_on_missing_sub_src: bool :param encryption_sources: A dictionary that maps the reference contained in the destination document's data section to the actual unecrypted data. If encrypting data with Barbican, the reference will be a Barbican secret reference. :type encryption_sources: dict :param cleartext_secrets: Whether to show unencrypted data as cleartext. :type cleartext_secrets: bool :raises LayeringPolicyNotFound: If no LayeringPolicy was found among list of ``documents``. :raises InvalidDocumentLayer: If document layer not found in layerOrder for provided LayeringPolicy. :raises InvalidDocumentParent: If child references parent but they don't have the same schema or their layers are incompatible. :raises IndeterminateDocumentParent: If more than one parent document was found for a document. """ self._documents_by_layer = {} self._documents_by_labels = {} self._layering_policy = None self._sorted_documents = {} self._documents_by_index = {} # TODO(felipemonteiro): Add a hook for post-validation too. if validate: self._pre_validate_documents(documents) layering_policies = list( filter(lambda x: x.get('schema').startswith( types.LAYERING_POLICY_SCHEMA), documents)) if layering_policies: self._layering_policy = dd(layering_policies[0]) if len(layering_policies) > 1: LOG.warning('More than one layering policy document was ' 'passed in. Using the first one found: [%s] %s.', self._layering_policy.schema, self._layering_policy.name) if self._layering_policy is None: error_msg = ( 'No layering policy found in the system so could not render ' 'documents.') LOG.error(error_msg) raise errors.LayeringPolicyNotFound() for document in documents: document = dd(document) self._documents_by_index.setdefault(document.meta, document) if document.layer: if document.layer not in self._layering_policy.layer_order: LOG.error('Document layer %s for document [%s] %s not ' 'in layerOrder: %s.', document.layer, document.schema, document.name, self._layering_policy.layer_order) raise errors.InvalidDocumentLayer( document_layer=document.layer, document_schema=document.schema, document_name=document.name, layer_order=', '.join( self._layering_policy.layer_order), layering_policy_name=self._layering_policy.name) self._documents_by_layer.setdefault(document.layer, []) self._documents_by_layer[document.layer].append(document) if document.parent_selector: for label_key, label_val in document.parent_selector.items(): self._documents_by_labels.setdefault( (label_key, label_val), []) self._documents_by_labels[ (label_key, label_val)].append(document) self._layer_order = self._get_layering_order(self._layering_policy) self._calc_all_document_children() substitution_sources = self._calc_replacements_and_substitutions( [ d for d in self._documents_by_index.values() if not d.is_abstract ]) self.secrets_substitution = secrets_manager.SecretsSubstitution( substitution_sources, encryption_sources=encryption_sources, fail_on_missing_sub_src=fail_on_missing_sub_src, cleartext_secrets=cleartext_secrets) self._sorted_documents = self._topologically_sort_documents( substitution_sources) del self._documents_by_layer del self._documents_by_labels
def __init__(self, documents, substitution_sources=None, validate=True): """Contructor for ``DocumentLayering``. :param layering_policy: The document with schema ``deckhand/LayeringPolicy`` needed for layering. :param documents: List of all other documents to be layered together in accordance with the ``layerOrder`` defined by the LayeringPolicy document. :type documents: List[dict] :param substitution_sources: List of documents that are potential sources for substitution. Should only include concrete documents. :type substitution_sources: List[dict] :param validate: Whether to pre-validate documents using built-in schema validation. Default is True. :type validate: bool :raises LayeringPolicyNotFound: If no LayeringPolicy was found among list of ``documents``. :raises InvalidDocumentLayer: If document layer not found in layerOrder for provided LayeringPolicy. :raises InvalidDocumentParent: If child references parent but they don't have the same schema or their layers are incompatible. :raises IndeterminateDocumentParent: If more than one parent document was found for a document. """ self._documents_to_layer = [] self._documents_by_layer = {} self._documents_by_labels = {} self._layering_policy = None if validate: self._validate_documents(documents) layering_policies = list( filter( lambda x: x.get('schema').startswith( types.LAYERING_POLICY_SCHEMA), documents)) if layering_policies: self._layering_policy = document_wrapper.DocumentDict( layering_policies[0]) if len(layering_policies) > 1: LOG.warning( 'More than one layering policy document was ' 'passed in. Using the first one found: [%s] %s.', self._layering_policy.schema, self._layering_policy.name) if self._layering_policy is None: error_msg = ( 'No layering policy found in the system so could not render ' 'documents.') LOG.error(error_msg) raise errors.LayeringPolicyNotFound() sorted_documents = self._topologically_sort_documents(documents) for document in sorted_documents: document = document_wrapper.DocumentDict(document) if document.layering_definition: self._documents_to_layer.append(document) if document.layer: if document.layer not in self._layering_policy.layer_order: LOG.error( 'Document layer %s for document [%s] %s not ' 'in layerOrder: %s.', document.layer, document.schema, document.name, self._layering_policy.layer_order) raise errors.InvalidDocumentLayer( document_layer=document.layer, document_schema=document.schema, document_name=document.name, layer_order=', '.join( self._layering_policy.layer_order), layering_policy_name=self._layering_policy.name) self._documents_by_layer.setdefault(document.layer, []) self._documents_by_layer[document.layer].append(document) if document.parent_selector: for label_key, label_val in document.parent_selector.items(): self._documents_by_labels.setdefault( (label_key, label_val), []) self._documents_by_labels[(label_key, label_val)].append(document) self._layer_order = self._get_layering_order(self._layering_policy) self._calc_all_document_children() self._substitution_sources = substitution_sources or [] self.secrets_substitution = secrets_manager.SecretsSubstitution( self._substitution_sources) del self._documents_by_layer del self._documents_by_labels