def test_secret_substititon_missing_src_path_in_src_doc_raises_exc(self):
"""Validates that if a secret can't be found in a substitution
source document then an exception is raised.
"""
certificate = self.secrets_factory.gen_test(
'Certificate', 'cleartext', data={})
certificate['metadata']['name'] = 'example-cert'
document_mapping = {
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.tls.certificate"
},
"src": {
"schema": "deckhand/Certificate/v1",
"name": "example-cert",
"path": ".path-to-nowhere"
}
}]
}
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(
bucket_name, [certificate] + [payload[-1]])
secrets_substitution = secrets_manager.SecretsSubstitution(documents)
with testtools.ExpectedException(
errors.SubstitutionSourceDataNotFound):
next(secrets_substitution.substitute_all(documents))
def test_secret_substitution_missing_encryption_sources_raises_exc(self):
"""Validate that when ``encryption_sources`` doesn't contain a
reference that a ``EncryptionSourceNotFound`` is raised.
"""
secret_ref = test_utils.rand_barbican_ref()
certificate = self.secrets_factory.gen_test(
'Certificate', 'encrypted', data=secret_ref)
certificate['metadata']['name'] = 'example-cert'
document_mapping = {
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.tls.certificate"
},
"src": {
"schema": "deckhand/Certificate/v1",
"name": "example-cert",
"path": ".path-to-nowhere"
}
}]
}
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(
bucket_name, [certificate] + [payload[-1]])
secrets_substitution = secrets_manager.SecretsSubstitution(
documents, encryption_sources={'foo': 'bar'})
with testtools.ExpectedException(errors.EncryptionSourceNotFound):
next(secrets_substitution.substitute_all(documents))
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
try:
documents = db_api.revision_get_documents(revision_id, **filters)
except errors.RevisionNotFound as e:
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
# TODO(fmontei): Currently the only phase of rendering that is
# performed is secret substitution, which can be done in any randomized
# order. However, secret substitution logic will have to be moved into
# a separate module that handles layering alongside substitution once
# layering has been fully integrated into this endpoint.
secrets_substitution = secrets_manager.SecretsSubstitution(documents)
try:
rendered_documents = secrets_substitution.substitute_all()
except errors.DocumentNotFound as e:
LOG.error('Failed to render the documents because a secret '
'document could not be found.')
LOG.exception(six.text_type(e))
raise falcon.HTTPNotFound(description=e.format_message())
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
def _test_secrets_substitution(self, secret_type, expected_exception):
secret_ref = test_utils.rand_barbican_ref()
certificate = self.secrets_factory.gen_test(
'Certificate', secret_type, data=secret_ref)
certificate['metadata']['name'] = 'example-cert'
document_mapping = {
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.tls.certificate"
},
"src": {
"schema": "deckhand/Certificate/v1",
"name": "example-cert",
"path": "."
}
}]
}
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(
bucket_name, [certificate] + [payload[-1]])
secrets_substitution = secrets_manager.SecretsSubstitution(documents)
with testtools.ExpectedException(expected_exception):
next(secrets_substitution.substitute_all(documents))
def _apply_substitutions(self, document):
try:
secrets_substitution = secrets_manager.SecretsSubstitution(
document, self._substitution_sources)
return secrets_substitution.substitute_all()
except errors.SubstitutionDependencyNotFound:
LOG.error('Failed to render the documents because a secret '
'document could not be found.')
def _test_secret_substitution(self, document_mapping, secret_documents,
expected_data):
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(bucket_name,
secret_documents + [payload[-1]])
expected_documents = copy.deepcopy([documents[-1]])
expected_documents[0]['data'] = expected_data
secret_substitution = secrets_manager.SecretsSubstitution(documents)
substituted_docs = secret_substitution.substitute_all()
self.assertEqual(expected_documents, substituted_docs)
def _test_doc_substitution(self, document_mapping, substitution_sources,
expected_data, encryption_sources=None):
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(
bucket_name, substitution_sources + [payload[-1]])
expected_document = copy.deepcopy(documents[-1])
expected_document['data'] = expected_data
secret_substitution = secrets_manager.SecretsSubstitution(
encryption_sources=encryption_sources,
substitution_sources=substitution_sources)
substituted_docs = list(secret_substitution.substitute_all(documents))
self.assertIn(expected_document, substituted_docs)
def _test_doc_substitution(self, document_mapping, secret_documents,
expected_data):
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(bucket_name,
secret_documents + [payload[-1]])
expected_document = copy.deepcopy(documents[-1])
expected_document['data'] = expected_data
substitution_sources = db_api.document_get_all(
**{'metadata.layeringDefinition.abstract': False})
secret_substitution = secrets_manager.SecretsSubstitution(
substitution_sources)
substituted_docs = list(secret_substitution.substitute_all(documents))
self.assertIn(expected_document, substituted_docs)
def test_doc_substitution_multiple_pattern_substitutions(self):
test_yaml = """
---
schema: deckhand/LayeringPolicy/v1
metadata:
schema: metadata/Control/v1
name: layering-policy
data:
layerOrder:
- global
- site
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-drydock
layeringDefinition:
abstract: false
layer: global
storagePolicy: cleartext
substitutions:
- src:
schema: twigleg/CommonAddresses/v1
name: common-addresses
path: .genesis.ip
dest:
path: .values.conf.drydock.maasdriver.maas_api_url
pattern: 'MAAS_IP'
- src:
schema: twigleg/CommonAddresses/v1
name: common-addresses
path: .node_ports.maas_api
dest:
path: .values.conf.drydock.maasdriver.maas_api_url
pattern: 'MAAS_PORT'
data:
values:
conf:
drydock:
maasdriver:
maas_api_url: http://MAAS_IP:MAAS_PORT/MAAS/api/2.0/
---
schema: twigleg/CommonAddresses/v1
metadata:
schema: metadata/Document/v1
name: common-addresses
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
genesis:
ip: 10.24.31.31
node_ports:
maas_api: 30001
...
"""
documents = list(yaml.safe_load_all(test_yaml))
expected = copy.deepcopy(documents[1])
expected['data']['values']['conf']['drydock']['maasdriver'][
'maas_api_url'] = 'http://10.24.31.31:30001/MAAS/api/2.0/'
secret_substitution = secrets_manager.SecretsSubstitution(documents)
substituted_docs = list(secret_substitution.substitute_all(documents))
self.assertEqual(expected, substituted_docs[0])
def __init__(self,
documents,
validate=True,
fail_on_missing_sub_src=True,
encryption_sources=None,
cleartext_secrets=False):
"""Contructor for ``DocumentLayering``.
:param layering_policy: The document with schema
``deckhand/LayeringPolicy`` needed for layering.
:param documents: List of all other documents to be layered together
in accordance with the ``layerOrder`` defined by the
LayeringPolicy document.
:type documents: List[dict]
:param validate: Whether to pre-validate documents using built-in
schema validation. Skips over externally registered ``DataSchema``
documents to avoid false positives. Default is True.
:type validate: bool
:param fail_on_missing_sub_src: Whether to fail on a missing
substitution source. Default is True.
:type fail_on_missing_sub_src: bool
:param encryption_sources: A dictionary that maps the reference
contained in the destination document's data section to the
actual unecrypted data. If encrypting data with Barbican, the
reference will be a Barbican secret reference.
:type encryption_sources: dict
:param cleartext_secrets: Whether to show unencrypted data as
cleartext.
:type cleartext_secrets: bool
:raises LayeringPolicyNotFound: If no LayeringPolicy was found among
list of ``documents``.
:raises InvalidDocumentLayer: If document layer not found in layerOrder
for provided LayeringPolicy.
:raises InvalidDocumentParent: If child references parent but they
don't have the same schema or their layers are incompatible.
:raises IndeterminateDocumentParent: If more than one parent document
was found for a document.
"""
self._documents_by_layer = {}
self._documents_by_labels = {}
self._layering_policy = None
self._sorted_documents = {}
self._documents_by_index = {}
# TODO(felipemonteiro): Add a hook for post-validation too.
if validate:
self._pre_validate_documents(documents)
layering_policies = list(
filter(lambda x: x.get('schema').startswith(
types.LAYERING_POLICY_SCHEMA), documents))
if layering_policies:
self._layering_policy = dd(layering_policies[0])
if len(layering_policies) > 1:
LOG.warning('More than one layering policy document was '
'passed in. Using the first one found: [%s] %s.',
self._layering_policy.schema,
self._layering_policy.name)
if self._layering_policy is None:
error_msg = (
'No layering policy found in the system so could not render '
'documents.')
LOG.error(error_msg)
raise errors.LayeringPolicyNotFound()
for document in documents:
document = dd(document)
self._documents_by_index.setdefault(document.meta, document)
if document.layer:
if document.layer not in self._layering_policy.layer_order:
LOG.error('Document layer %s for document [%s] %s not '
'in layerOrder: %s.', document.layer,
document.schema, document.name,
self._layering_policy.layer_order)
raise errors.InvalidDocumentLayer(
document_layer=document.layer,
document_schema=document.schema,
document_name=document.name,
layer_order=', '.join(
self._layering_policy.layer_order),
layering_policy_name=self._layering_policy.name)
self._documents_by_layer.setdefault(document.layer, [])
self._documents_by_layer[document.layer].append(document)
if document.parent_selector:
for label_key, label_val in document.parent_selector.items():
self._documents_by_labels.setdefault(
(label_key, label_val), [])
self._documents_by_labels[
(label_key, label_val)].append(document)
self._layer_order = self._get_layering_order(self._layering_policy)
self._calc_all_document_children()
substitution_sources = self._calc_replacements_and_substitutions(
[
d for d in self._documents_by_index.values()
if not d.is_abstract
])
self.secrets_substitution = secrets_manager.SecretsSubstitution(
substitution_sources,
encryption_sources=encryption_sources,
fail_on_missing_sub_src=fail_on_missing_sub_src,
cleartext_secrets=cleartext_secrets)
self._sorted_documents = self._topologically_sort_documents(
substitution_sources)
del self._documents_by_layer
del self._documents_by_labels
def __init__(self, documents, substitution_sources=None, validate=True):
"""Contructor for ``DocumentLayering``.
:param layering_policy: The document with schema
``deckhand/LayeringPolicy`` needed for layering.
:param documents: List of all other documents to be layered together
in accordance with the ``layerOrder`` defined by the
LayeringPolicy document.
:type documents: List[dict]
:param substitution_sources: List of documents that are potential
sources for substitution. Should only include concrete documents.
:type substitution_sources: List[dict]
:param validate: Whether to pre-validate documents using built-in
schema validation. Default is True.
:type validate: bool
:raises LayeringPolicyNotFound: If no LayeringPolicy was found among
list of ``documents``.
:raises InvalidDocumentLayer: If document layer not found in layerOrder
for provided LayeringPolicy.
:raises InvalidDocumentParent: If child references parent but they
don't have the same schema or their layers are incompatible.
:raises IndeterminateDocumentParent: If more than one parent document
was found for a document.
"""
self._documents_to_layer = []
self._documents_by_layer = {}
self._documents_by_labels = {}
self._layering_policy = None
if validate:
self._validate_documents(documents)
layering_policies = list(
filter(
lambda x: x.get('schema').startswith(
types.LAYERING_POLICY_SCHEMA), documents))
if layering_policies:
self._layering_policy = document_wrapper.DocumentDict(
layering_policies[0])
if len(layering_policies) > 1:
LOG.warning(
'More than one layering policy document was '
'passed in. Using the first one found: [%s] %s.',
self._layering_policy.schema, self._layering_policy.name)
if self._layering_policy is None:
error_msg = (
'No layering policy found in the system so could not render '
'documents.')
LOG.error(error_msg)
raise errors.LayeringPolicyNotFound()
sorted_documents = self._topologically_sort_documents(documents)
for document in sorted_documents:
document = document_wrapper.DocumentDict(document)
if document.layering_definition:
self._documents_to_layer.append(document)
if document.layer:
if document.layer not in self._layering_policy.layer_order:
LOG.error(
'Document layer %s for document [%s] %s not '
'in layerOrder: %s.', document.layer, document.schema,
document.name, self._layering_policy.layer_order)
raise errors.InvalidDocumentLayer(
document_layer=document.layer,
document_schema=document.schema,
document_name=document.name,
layer_order=', '.join(
self._layering_policy.layer_order),
layering_policy_name=self._layering_policy.name)
self._documents_by_layer.setdefault(document.layer, [])
self._documents_by_layer[document.layer].append(document)
if document.parent_selector:
for label_key, label_val in document.parent_selector.items():
self._documents_by_labels.setdefault(
(label_key, label_val), [])
self._documents_by_labels[(label_key,
label_val)].append(document)
self._layer_order = self._get_layering_order(self._layering_policy)
self._calc_all_document_children()
self._substitution_sources = substitution_sources or []
self.secrets_substitution = secrets_manager.SecretsSubstitution(
self._substitution_sources)
del self._documents_by_layer
del self._documents_by_labels
