def create(cls, secret_doc):
"""Securely store secrets contained in ``secret_doc``.
Documents with ``metadata.storagePolicy`` == "clearText" have their
secrets stored directly in Deckhand.
Documents with ``metadata.storagePolicy`` == "encrypted" are stored in
Barbican directly. Deckhand in turn stores the reference returned
by Barbican in its own DB.
:param secret_doc: A Deckhand document with a schema that belongs to
``types.DOCUMENT_SECRET_TYPES``.
:returns: Unecrypted data section from ``secret_doc`` if the document's
``storagePolicy`` is "cleartext" or a Barbican secret reference
if the ``storagePolicy`` is "encrypted'.
"""
# TODO(fmontei): Look into POSTing Deckhand metadata into Barbican's
# Secrets Metadata API to make it easier to track stale secrets from
# prior revisions that need to be deleted.
if not isinstance(secret_doc, dd):
secret_doc = dd(secret_doc)
if secret_doc.is_encrypted:
payload = cls.barbican_driver.create_secret(secret_doc)
else:
payload = secret_doc.data
return payload
def __init__(self,
substitution_sources=None,
fail_on_missing_sub_src=True,
encryption_sources=None,
cleartext_secrets=False):
"""SecretSubstitution constructor.
This class will automatically detect documents that require
substitution; documents need not be filtered prior to being passed to
the constructor.
:param substitution_sources: List of documents that are potential
sources for substitution. Or dict of documents keyed on tuple of
(schema, metadata.name). Should only include concrete documents.
:type substitution_sources: List[dict] or dict
:param bool fail_on_missing_sub_src: Whether to fail on a missing
substitution source. Default is True.
:param encryption_sources: A dictionary that maps the reference
contained in the destination document's data section to the
actual unecrypted data. If encrypting data with Barbican, the
reference will be a Barbican secret reference.
:type encryption_sources: dict
:param cleartext_secrets: Whether to show unencrypted data as
cleartext.
:type cleartext_secrets: bool
"""
# This maps a 2-tuple of (schema, name) to a document from which the
# document.meta can be extracted which is a 3-tuple of (schema, layer,
# name). This is necessary since the substitution format in the
# document itself only provides a 2-tuple of (schema, name).
self._substitution_sources = {}
self._encryption_sources = encryption_sources or {}
self._fail_on_missing_sub_src = fail_on_missing_sub_src
self._cleartext_secrets = cleartext_secrets
if isinstance(substitution_sources, dict):
self._substitution_sources = substitution_sources
else:
self._substitution_sources = dict()
for document in substitution_sources:
if not isinstance(document, dd):
document = dd(document)
if document.schema and document.name:
self._substitution_sources.setdefault(
(document.schema, document.name), document)
def delete(cls, document):
"""Delete a secret from Barbican.
:param dict document: Document with secret_ref in ``data`` section with
format: "https://{barbican_host}/v1/secrets/{secret_uuid}"
:returns: None
"""
if not isinstance(document, dd):
document = dd(document)
secret_ref = document.data
if document.is_encrypted and document.has_barbican_ref:
LOG.debug('Deleting Barbican secret: %s.', secret_ref)
cls.barbican_driver.delete_secret(secret_ref=secret_ref)
else:
LOG.warning('%s is not a valid Barbican secret. Could not delete.',
secret_ref)
def _calc_replacements_and_substitutions(
self, substitution_sources):
# Used to track document names and schemas for documents that are not
# replacement documents
non_replacement_documents = set()
for document in self._documents_by_index.values():
parent_meta = self._parents.get(document.meta)
parent = self._documents_by_index.get(parent_meta)
if document.is_replacement:
replacement.check_document_with_replacement_field_has_parent(
parent_meta, parent, document)
replacement.check_replacement_and_parent_same_schema_and_name(
parent, document)
parent.replaced_by = document
else:
# Handles case where parent and child have replacement: false
# as in this case both documents should not be replacement
# documents, requiring them to have different schema/name pair.
replacement.check_child_and_parent_different_metadata_name(
parent, document)
replacement.check_replacement_is_false_uniqueness(
document, non_replacement_documents)
# Since a substitution source only provides the document's
# `metadata.name` and `schema`, their tuple acts as the dictionary key.
# If a substitution source has a replacement, the replacement is used
# instead.
substitution_source_map = {}
for src in substitution_sources:
src_ref = dd(src)
if src_ref.meta in self._documents_by_index:
src_ref = self._documents_by_index[src_ref.meta]
if src_ref.has_replacement:
replacement.check_only_one_level_of_replacement(src_ref)
src_ref = src_ref.replaced_by
substitution_source_map[(src_ref.schema, src_ref.name)] = src_ref
return substitution_source_map
def substitute_all(self, documents):
"""Substitute all documents that have a `metadata.substitutions` field.
Concrete (non-abstract) documents can be used as a source of
substitution into other documents. This substitution is
layer-independent, a document in the region layer could insert data
from a document in the site layer.
:param documents: List of documents that are candidates for
substitution.
:type documents: dict or List[dict]
:returns: List of fully substituted documents.
:rtype: Generator[:class:`DocumentDict`]
:raises SubstitutionSourceNotFound: If a substitution source document
is referenced by another document but wasn't found.
:raises UnknownSubstitutionError: If an unknown error occurred during
substitution.
"""
documents_to_substitute = []
if not isinstance(documents, list):
documents = [documents]
for document in documents:
if not isinstance(document, dd):
document = dd(document)
# If the document has substitutions include it.
if document.substitutions:
documents_to_substitute.append(document)
LOG.debug(
'Performing substitution on following documents: %s', ', '.join(
['[%s, %s] %s' % d.meta for d in documents_to_substitute]))
for document in documents_to_substitute:
redact_dest = False
LOG.debug('Checking for substitutions for document [%s, %s] %s.',
*document.meta)
for sub in document.substitutions:
src_schema = sub['src']['schema']
src_name = sub['src']['name']
src_path = sub['src']['path']
if (src_schema, src_name) in self._substitution_sources:
src_doc = self._substitution_sources[(src_schema,
src_name)]
else:
message = ('Could not find substitution source document '
'[%s] %s among the provided substitution '
'sources.' % (src_schema, src_name))
if self._fail_on_missing_sub_src:
LOG.error(message)
raise errors.SubstitutionSourceNotFound(
src_schema=src_schema,
src_name=src_name,
document_schema=document.schema,
document_name=document.name)
else:
LOG.warning(message)
continue
if src_doc.is_encrypted:
redact_dest = True
# If the data is a dictionary, retrieve the nested secret
# via jsonpath_parse, else the secret is the primitive/string
# stored in the data section itself.
if isinstance(src_doc.get('data'), dict):
src_secret = utils.jsonpath_parse(src_doc.get('data', {}),
src_path)
else:
src_secret = src_doc.get('data')
self._check_src_secret_is_not_none(src_secret, src_path,
src_doc, document)
# If the document has storagePolicy == encrypted then resolve
# the Barbican reference into the actual secret.
if src_doc.is_encrypted and src_doc.has_barbican_ref:
src_secret = self.get_unencrypted_data(
src_secret, src_doc, document)
if not isinstance(sub['dest'], list):
dest_array = [sub['dest']]
dest_is_list = False
else:
dest_array = sub['dest']
dest_is_list = True
for i, each_dest_path in enumerate(dest_array):
dest_path = each_dest_path['path']
dest_pattern = each_dest_path.get('pattern', None)
dest_recurse = each_dest_path.get('recurse', {})
# If the source document is encrypted and cleartext_secrets
# is False, then redact the substitution metadata in the
# destination document to prevent reverse-engineering of
# where the sensitive data came from.
if src_doc.is_encrypted and not self._cleartext_secrets:
sub['src']['path'] = dd.redact(src_path)
if dest_is_list:
sub['dest'][i]['path'] = dd.redact(dest_path)
else:
sub['dest']['path'] = dd.redact(dest_path)
LOG.debug(
'Substituting from schema=%s layer=%s name=%s '
'src_path=%s into dest_path=%s, dest_pattern=%s',
src_schema, src_doc.layer, src_name, src_path,
dest_path, dest_pattern)
document = self._substitute_one(document,
src_doc=src_doc,
src_secret=src_secret,
dest_path=dest_path,
dest_pattern=dest_pattern,
dest_recurse=dest_recurse)
# If we just substituted from an encrypted document
# into a cleartext document, we need to redact the
# dest document as well so the secret stays hidden
if (not document.is_encrypted and redact_dest
and not self._cleartext_secrets):
document.storage_policy = 'encrypted'
yield document
def __init__(self,
documents,
validate=True,
fail_on_missing_sub_src=True,
encryption_sources=None,
cleartext_secrets=False):
"""Contructor for ``DocumentLayering``.
:param layering_policy: The document with schema
``deckhand/LayeringPolicy`` needed for layering.
:param documents: List of all other documents to be layered together
in accordance with the ``layerOrder`` defined by the
LayeringPolicy document.
:type documents: List[dict]
:param validate: Whether to pre-validate documents using built-in
schema validation. Skips over externally registered ``DataSchema``
documents to avoid false positives. Default is True.
:type validate: bool
:param fail_on_missing_sub_src: Whether to fail on a missing
substitution source. Default is True.
:type fail_on_missing_sub_src: bool
:param encryption_sources: A dictionary that maps the reference
contained in the destination document's data section to the
actual unecrypted data. If encrypting data with Barbican, the
reference will be a Barbican secret reference.
:type encryption_sources: dict
:param cleartext_secrets: Whether to show unencrypted data as
cleartext.
:type cleartext_secrets: bool
:raises LayeringPolicyNotFound: If no LayeringPolicy was found among
list of ``documents``.
:raises InvalidDocumentLayer: If document layer not found in layerOrder
for provided LayeringPolicy.
:raises InvalidDocumentParent: If child references parent but they
don't have the same schema or their layers are incompatible.
:raises IndeterminateDocumentParent: If more than one parent document
was found for a document.
"""
self._documents_by_layer = {}
self._documents_by_labels = {}
self._layering_policy = None
self._sorted_documents = {}
self._documents_by_index = {}
# TODO(felipemonteiro): Add a hook for post-validation too.
if validate:
self._pre_validate_documents(documents)
layering_policies = list(
filter(lambda x: x.get('schema').startswith(
types.LAYERING_POLICY_SCHEMA), documents))
if layering_policies:
self._layering_policy = dd(layering_policies[0])
if len(layering_policies) > 1:
LOG.warning('More than one layering policy document was '
'passed in. Using the first one found: [%s] %s.',
self._layering_policy.schema,
self._layering_policy.name)
if self._layering_policy is None:
error_msg = (
'No layering policy found in the system so could not render '
'documents.')
LOG.error(error_msg)
raise errors.LayeringPolicyNotFound()
for document in documents:
document = dd(document)
self._documents_by_index.setdefault(document.meta, document)
if document.layer:
if document.layer not in self._layering_policy.layer_order:
LOG.error('Document layer %s for document [%s] %s not '
'in layerOrder: %s.', document.layer,
document.schema, document.name,
self._layering_policy.layer_order)
raise errors.InvalidDocumentLayer(
document_layer=document.layer,
document_schema=document.schema,
document_name=document.name,
layer_order=', '.join(
self._layering_policy.layer_order),
layering_policy_name=self._layering_policy.name)
self._documents_by_layer.setdefault(document.layer, [])
self._documents_by_layer[document.layer].append(document)
if document.parent_selector:
for label_key, label_val in document.parent_selector.items():
self._documents_by_labels.setdefault(
(label_key, label_val), [])
self._documents_by_labels[
(label_key, label_val)].append(document)
self._layer_order = self._get_layering_order(self._layering_policy)
self._calc_all_document_children()
substitution_sources = self._calc_replacements_and_substitutions(
[
d for d in self._documents_by_index.values()
if not d.is_abstract
])
self.secrets_substitution = secrets_manager.SecretsSubstitution(
substitution_sources,
encryption_sources=encryption_sources,
fail_on_missing_sub_src=fail_on_missing_sub_src,
cleartext_secrets=cleartext_secrets)
self._sorted_documents = self._topologically_sort_documents(
substitution_sources)
del self._documents_by_layer
del self._documents_by_labels
def _calc_replacements_and_substitutions(self, substitution_sources):
def _check_document_with_replacement_field_has_parent(
parent_meta, parent, document):
if not parent_meta or not parent:
error_message = (
'Document replacement requires that the document with '
'`replacement: true` have a parent.')
raise errors.InvalidDocumentReplacement(schema=document.schema,
name=document.name,
layer=document.layer,
reason=error_message)
def _check_replacement_and_parent_same_schema_and_name(
parent, document):
# This checks that a document can only be a replacement for
# another document with the same `metadata.name` and `schema`.
if not (document.schema == parent.schema
and document.name == parent.name):
error_message = (
'Document replacement requires that both documents '
'have the same `schema` and `metadata.name`.')
raise errors.InvalidDocumentReplacement(schema=document.schema,
name=document.name,
layer=document.layer,
reason=error_message)
def _check_non_replacement_and_parent_different_schema_and_name(
parent, document):
if (parent and document.schema == parent.schema
and document.name == parent.name):
error_message = (
'Non-replacement documents cannot have the same `schema` '
'and `metadata.name` as their parent. Either add '
'`replacement: true` to the document or give the document '
'a different name.')
raise errors.InvalidDocumentReplacement(schema=document.schema,
name=document.name,
layer=document.layer,
reason=error_message)
def _check_replacement_not_itself_replaced_by_another(src_ref):
# If the document has a replacement, use the replacement as the
# substitution source instead.
if src_ref.is_replacement:
error_message = ('A replacement document cannot itself'
' be replaced by another document.')
raise errors.InvalidDocumentReplacement(schema=src_ref.schema,
name=src_ref.name,
layer=src_ref.layer,
reason=error_message)
for document in self._documents_by_index.values():
parent_meta = self._parents.get(document.meta)
parent = self._documents_by_index.get(parent_meta)
if document.is_replacement:
_check_document_with_replacement_field_has_parent(
parent_meta, parent, document)
_check_replacement_and_parent_same_schema_and_name(
parent, document)
parent.replaced_by = document
else:
_check_non_replacement_and_parent_different_schema_and_name(
parent, document)
# Since a substitution source only provides the document's
# `metadata.name` and `schema`, their tuple acts as the dictionary key.
# If a substitution source has a replacement, the replacement is used
# instead.
substitution_source_map = {}
for src in substitution_sources:
src_ref = dd(src)
if src_ref.meta in self._documents_by_index:
src_ref = self._documents_by_index[src_ref.meta]
if src_ref.has_replacement:
_check_replacement_not_itself_replaced_by_another(src_ref)
src_ref = src_ref.replaced_by
substitution_source_map[(src_ref.schema, src_ref.name)] = src_ref
return substitution_source_map