def sanitize_potential_secrets(document):
"""Sanitize all secret data that may have been substituted into the
document. Uses references in ``document.substitutions`` to determine
which values to sanitize. Only meaningful to call this on post-rendered
documents.
:param DocumentDict document: Document to sanitize.
"""
safe_message = 'Sanitized to avoid exposing secret.'
for sub in document.substitutions:
utils.jsonpath_replace(document['data'], safe_message,
sub['dest']['path'])
def test_abstract_document_not_validated(self, mock_log):
test_document = self._read_data('sample_passphrase')
# Set the document to abstract.
abstract_document = utils.jsonpath_replace(
test_document, True, '.metadata.layeringDefinition.abstract')
document_validation.DocumentValidation(
abstract_document).validate_all()
self.assertTrue(mock_log.info.called)
self.assertIn("Skipping schema validation for abstract document",
mock_log.info.mock_calls[0][1][0])
def substitute_all(self):
"""Substitute all documents that have a `metadata.substitutions` field.
Concrete (non-abstract) documents can be used as a source of
substitution into other documents. This substitution is
layer-independent, a document in the region layer could insert data
from a document in the site layer.
:returns: List of fully substituted documents.
"""
LOG.debug('Substituting secrets for documents: %s', self.documents)
substituted_docs = []
for doc in self.documents:
LOG.debug(
'Checking for substitutions in schema=%s, metadata.name=%s',
doc.get_name(), doc.get_schema())
for sub in doc.get_substitutions():
src_schema = sub['src']['schema']
src_name = sub['src']['name']
src_path = sub['src']['path']
if src_path == '.':
src_path = '.secret'
# TODO(fmontei): Use secrets_manager for this logic. Need to
# check Barbican for the secret if it has been encrypted.
src_doc = db_api.document_get(
schema=src_schema,
name=src_name,
is_secret=True,
**{'metadata.layeringDefinition.abstract': False})
src_secret = utils.jsonpath_parse(src_doc['data'], src_path)
dest_path = sub['dest']['path']
dest_pattern = sub['dest'].get('pattern', None)
LOG.debug(
'Substituting from schema=%s name=%s src_path=%s '
'into dest_path=%s, dest_pattern=%s', src_schema, src_name,
src_path, dest_path, dest_pattern)
substituted_data = utils.jsonpath_replace(
doc['data'], src_secret, dest_path, dest_pattern)
doc['data'].update(substituted_data)
substituted_docs.append(doc.to_dict())
return substituted_docs
def test_validation_failure_does_not_expose_secrets(self, mock_jsonschema):
m_args = mock.Mock()
mock_jsonschema.Draft4Validator(m_args).iter_errors.side_effect = [
# Return empty list of errors for base schema validator and pretend
# that 1 error is returned for next validator.
[], [mock.Mock(path=[], schema_path=[])]
]
test_document = self._read_data('sample_document')
for sub in test_document['metadata']['substitutions']:
substituted_data = utils.jsonpath_replace(
test_document['data'], 'scary-secret', sub['dest']['path'])
test_document['data'].update(substituted_data)
self.assertEqual(
'scary-secret', utils.jsonpath_parse(test_document['data'],
sub['dest']['path']))
validations = document_validation.DocumentValidation(
test_document).validate_all()
self.assertEqual(1, len(validations[0]['errors']))
self.assertIn('Sanitized to avoid exposing secret.',
str(validations[0]['errors'][-1]))
self.assertNotIn('scary-secret.', str(validations[0]['errors'][-1]))
def substitute_all(self):
"""Substitute all documents that have a `metadata.substitutions` field.
Concrete (non-abstract) documents can be used as a source of
substitution into other documents. This substitution is
layer-independent, a document in the region layer could insert data
from a document in the site layer.
:returns: List of fully substituted documents.
:rtype: List[:class:`DocumentDict`]
:raises SubstitutionDependencyNotFound: If a substitution source wasn't
found or something else went wrong during substitution.
"""
LOG.debug('Performing substitution on following documents: %s',
', '.join(['[%s] %s' % (d.schema, d.name)
for d in self._documents]))
substituted_docs = []
for document in self._documents:
LOG.debug('Checking for substitutions for document [%s] %s.',
document.schema, document.name)
for sub in document.substitutions:
src_schema = sub['src']['schema']
src_name = sub['src']['name']
src_path = sub['src']['path']
if not src_schema:
LOG.warning('Source document schema "%s" is unspecified '
'under substitutions for document [%s] %s.',
src_schema, document.schema, document.name)
if not src_name:
LOG.warning('Source document name "%s" is unspecified'
' under substitutions for document [%s] %s.',
src_name, document.schema, document.name)
if not src_path:
LOG.warning('Source document path "%s" is unspecified '
'under substitutions for document [%s] %s.',
src_path, document.schema, document.name)
if (src_schema, src_name) in self._substitution_sources:
src_doc = self._substitution_sources[
(src_schema, src_name)]
else:
src_doc = {}
LOG.warning('Could not find substitution source document '
'[%s] %s among the provided '
'`substitution_sources`.', src_schema,
src_name)
# If the data is a dictionary, retrieve the nested secret
# via jsonpath_parse, else the secret is the primitive/string
# stored in the data section itself.
if isinstance(src_doc.get('data'), dict):
src_secret = utils.jsonpath_parse(src_doc.get('data', {}),
src_path)
else:
src_secret = src_doc.get('data')
dest_path = sub['dest']['path']
dest_pattern = sub['dest'].get('pattern', None)
if not dest_path:
LOG.warning('Destination document path "%s" is unspecified'
' under substitutions for document [%s] %s.',
dest_path, document.schema, document.name)
LOG.debug('Substituting from schema=%s name=%s src_path=%s '
'into dest_path=%s, dest_pattern=%s', src_schema,
src_name, src_path, dest_path, dest_pattern)
try:
substituted_data = utils.jsonpath_replace(
document['data'], src_secret, dest_path, dest_pattern)
if isinstance(substituted_data, dict):
document['data'].update(substituted_data)
else:
document['data'] = substituted_data
except Exception as e:
LOG.error('Unexpected exception occurred while attempting '
'secret substitution. %s', six.text_type(e))
raise errors.SubstitutionDependencyNotFound(
details=six.text_type(e))
substituted_docs.append(document)
return substituted_docs
def substitutions(self, value):
return utils.jsonpath_replace(self, value, 'metadata.substitutions')
def substitute_all(self, documents):
"""Substitute all documents that have a `metadata.substitutions` field.
Concrete (non-abstract) documents can be used as a source of
substitution into other documents. This substitution is
layer-independent, a document in the region layer could insert data
from a document in the site layer.
:param documents: List of documents that are candidates for
substitution.
:type documents: dict or List[dict]
:returns: List of fully substituted documents.
:rtype: Generator[:class:`DocumentDict`]
:raises SubstitutionSourceNotFound: If a substitution source document
is referenced by another document but wasn't found.
:raises UnknownSubstitutionError: If an unknown error occurred during
substitution.
"""
documents_to_substitute = []
if not isinstance(documents, list):
documents = [documents]
for document in documents:
if not isinstance(document, document_wrapper.DocumentDict):
document = document_wrapper.DocumentDict(document)
# If the document has substitutions include it.
if document.substitutions:
documents_to_substitute.append(document)
LOG.debug(
'Performing substitution on following documents: %s', ', '.join([
'[%s] %s' % (d.schema, d.name) for d in documents_to_substitute
]))
for document in documents_to_substitute:
LOG.debug('Checking for substitutions for document [%s] %s.',
document.schema, document.name)
for sub in document.substitutions:
src_schema = sub['src']['schema']
src_name = sub['src']['name']
src_path = sub['src']['path']
if (src_schema, src_name) in self._substitution_sources:
src_doc = self._substitution_sources[(src_schema,
src_name)]
else:
message = ('Could not find substitution source document '
'[%s] %s among the provided '
'`substitution_sources`.', src_schema, src_name)
LOG.error(message)
raise errors.SubstitutionSourceNotFound(
src_schema=src_schema,
src_name=src_name,
document_schema=document.schema,
document_name=document.name)
# If the data is a dictionary, retrieve the nested secret
# via jsonpath_parse, else the secret is the primitive/string
# stored in the data section itself.
if isinstance(src_doc.get('data'), dict):
src_secret = utils.jsonpath_parse(src_doc.get('data', {}),
src_path)
else:
src_secret = src_doc.get('data')
dest_path = sub['dest']['path']
dest_pattern = sub['dest'].get('pattern', None)
LOG.debug(
'Substituting from schema=%s name=%s src_path=%s '
'into dest_path=%s, dest_pattern=%s', src_schema, src_name,
src_path, dest_path, dest_pattern)
try:
substituted_data = utils.jsonpath_replace(
document['data'], src_secret, dest_path, dest_pattern)
sub_source = self._substitution_sources.get(
(document.schema, document.name))
if (isinstance(document['data'], dict)
and isinstance(substituted_data, dict)):
document['data'].update(substituted_data)
if sub_source:
sub_source['data'].update(substituted_data)
elif substituted_data:
document['data'] = substituted_data
if sub_source:
sub_source['data'] = substituted_data
else:
message = ('Failed to create JSON path "%s" in the '
'destination document [%s] %s. No data was '
'substituted.', dest_path, document.schema,
document.name)
LOG.error(message)
raise errors.UnknownSubstitutionError(details=message)
except Exception as e:
LOG.error(
'Unexpected exception occurred while attempting '
'secret substitution. %s', six.text_type(e))
raise errors.UnknownSubstitutionError(
details=six.text_type(e))
yield document