def deckhand_render(documents=None,
fail_on_missing_sub_src=False,
validate=True):
documents = documents or []
errors = []
rendered_documents = []
schemas, schema_errors = load_schemas_from_docs(documents)
errors.extend(schema_errors)
try:
deckhand_eng = layering.DocumentLayering(
documents,
fail_on_missing_sub_src=fail_on_missing_sub_src,
validate=validate)
rendered_documents = [dict(d) for d in deckhand_eng.render()]
if validate:
validator = document_validation.DocumentValidation(
rendered_documents)
results = validator.validate_all()
for result in results:
if result['errors']:
errors.append(
(DECKHAND_RENDER_EXCEPTION,
'During rendering Deckhand was unable to validate '
'the following document, details: %s.' %
(result['errors'])))
except dh_errors.DeckhandException as e:
errors.append((DECKHAND_RENDER_EXCEPTION,
'An unknown Deckhand exception occurred while trying'
' to render documents: %s. Details: %s.' %
(str(e), e.error_list)))
return rendered_documents, errors
def __init__(self,
*,
documents,
debug=False,
substitute=True,
allow_missing_substitutions=True,
leave_kubectl=False,
validate=True):
LOG.info("Parsing document schemas.")
schema_set = validation.load_schemas_from_docs(documents)
LOG.info("Parsed %d document schemas." % len(schema_set))
LOG.info("Building config from %d documents." % len(documents))
if substitute:
LOG.info("Rendering documents via Deckhand engine.")
try:
deckhand_eng = layering.DocumentLayering(
documents,
fail_on_missing_sub_src=not allow_missing_substitutions)
documents = [dict(d) for d in deckhand_eng.render()]
except dh_errors.DeckhandException as e:
LOG.exception(
'An unknown Deckhand exception occurred while trying'
' to render documents.')
raise exceptions.DeckhandException(str(e))
LOG.info("Deckhand engine returned %d documents." % len(documents))
if validate:
validation.check_schemas(documents, schemas=schema_set)
self.debug = debug
self.documents = documents
self.leave_kubectl = leave_kubectl
def _test_layering(self, documents, site_expected=None,
region_expected=None, global_expected=None,
exception_expected=None, substitution_sources=None):
document_layering = layering.DocumentLayering(
documents, substitution_sources)
if all([site_expected, region_expected, global_expected,
exception_expected]):
raise ValueError(
'(site_expected|region_expected|global_expected) and '
'(exception_expected) are mutually exclusive.')
if exception_expected:
self.assertRaises(exception_expected, document_layering.render)
return
site_docs = []
region_docs = []
global_docs = []
# The layering policy is not returned as it is immutable. So all docs
# should have a metadata.layeringDefinitionn.layer section.
rendered_documents = document_layering.render()
for doc in rendered_documents:
# No need to validate the LayeringPolicy: it remains unchanged.
if doc['schema'].startswith(types.LAYERING_POLICY_SCHEMA):
continue
layer = doc['metadata']['layeringDefinition']['layer']
if layer == 'site':
site_docs.append(doc)
if layer == 'region':
region_docs.append(doc)
if layer == 'global':
global_docs.append(doc)
if site_expected:
if not isinstance(site_expected, list):
site_expected = [site_expected]
for idx, expected in enumerate(site_expected):
self.assertEqual(expected, site_docs[idx].get('data'),
'Actual site data does not match expected.')
if region_expected:
if not isinstance(region_expected, list):
region_expected = [region_expected]
for idx, expected in enumerate(region_expected):
self.assertEqual(expected, region_docs[idx].get('data'),
'Actual region data does not match expected.')
if global_expected:
if not isinstance(global_expected, list):
global_expected = [global_expected]
for idx, expected in enumerate(global_expected):
self.assertEqual(expected, global_docs[idx].get('data'),
'Actual global data does not match expected.')
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {
'metadata.storagePolicy': ['cleartext'],
'deleted': False
}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
documents = self._retrieve_documents_for_rendering(revision_id,
**filters)
try:
# NOTE(fmontei): `validate` is False because documents have already
# been pre-validated during ingestion. Documents are post-validated
# below, regardless.
document_layering = layering.DocumentLayering(
documents, validate=False)
rendered_documents = document_layering.render()
except (errors.InvalidDocumentLayer,
errors.InvalidDocumentParent,
errors.InvalidDocumentReplacement,
errors.IndeterminateDocumentParent,
errors.LayeringPolicyNotFound,
errors.MissingDocumentKey,
errors.SubstitutionSourceDataNotFound,
errors.SubstitutionSourceNotFound,
errors.UnknownSubstitutionError,
errors.UnsupportedActionMethod) as e:
with excutils.save_and_reraise_exception():
LOG.exception(e.format_message())
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
limit = sanitized_params.pop('limit', None)
user_filters = sanitized_params.copy()
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
if limit is not None:
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def _test_layering(self,
documents,
site_expected=None,
region_expected=None,
global_expected=None,
exception_expected=None):
document_layering = layering.DocumentLayering(documents)
if all([
site_expected, region_expected, global_expected,
exception_expected
]):
raise ValueError(
'(site_expected|region_expected|global_expected) and '
'(exception_expected) are mutually exclusive.')
if exception_expected:
self.assertRaises(exception_expected, document_layering.render)
return
site_docs = []
region_docs = []
global_docs = []
# The layering policy is not returned as it is immutable. So all docs
# should have a metadata.layeringDefinitionn.layer section.
rendered_documents = document_layering.render()
for doc in rendered_documents:
layer = doc['metadata']['layeringDefinition']['layer']
if layer == 'site':
site_docs.append(doc)
if layer == 'region':
region_docs.append(doc)
if layer == 'global':
global_docs.append(doc)
if site_expected:
if not isinstance(site_expected, list):
site_expected = [site_expected]
for idx, expected in enumerate(site_expected):
self.assertEqual(expected, site_docs[idx].get('data'))
if region_expected:
if not isinstance(region_expected, list):
region_expected = [region_expected]
for idx, expected in enumerate(region_expected):
self.assertEqual(expected, region_docs[idx].get('data'))
if global_expected:
if not isinstance(global_expected, list):
global_expected = [global_expected]
for idx, expected in enumerate(global_expected):
self.assertEqual(expected, global_docs[idx].get('data'))
def test_layering_child_with_invalid_parent_selector(self, mock_log):
doc_factory = factories.DocumentFactory(2, [1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
for parent_selector in ({'key2': 'value2'}, {'key1': 'value2'}):
documents[-1]['metadata']['layeringDefinition'][
'parentSelector'] = parent_selector
layering.DocumentLayering(documents)
self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0],
'Could not find parent for document .*')
mock_log.info.reset_mock()
def test_layering_unreferenced_parent_label(self, mock_log):
doc_factory = factories.DocumentFactory(2, [1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
for parent_label in ({'key2': 'value2'}, {'key1': 'value2'}):
# Second doc is the global doc, or parent.
documents[1]['metadata']['labels'] = parent_label
layering.DocumentLayering(documents)
self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0],
'Could not find parent for document .*')
mock_log.info.reset_mock()
def test_layering_document_references_itself(self, mock_log):
# Test that a parentSelector cannot reference the document itself
# without an error being raised.
doc_factory = factories.DocumentFactory(3, [1, 1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
self_ref = {"self": "self"}
documents[2]['metadata']['labels'] = self_ref
documents[2]['metadata']['layeringDefinition'][
'parentSelector'] = self_ref
layering.DocumentLayering(documents)
self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0],
'Could not find parent for document .*')
def test_layering_unreferenced_parent_label(self, mock_log):
doc_factory = factories.DocumentFactory(2, [1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
for parent_label in ({'key2': 'value2'}, {'key1': 'value2'}):
# Second doc is the global doc, or parent.
documents[1]['metadata']['labels'] = parent_label
layering.DocumentLayering(documents, validate=False)
self.assertTrue(
any('Could not find parent for document' in
mock_log.debug.mock_calls[x][1][0])
for x in range(len(mock_log.debug.mock_calls)))
mock_log.debug.reset_mock()
def test_layering_child_with_invalid_parent_selector(self, mock_log):
doc_factory = factories.DocumentFactory(2, [1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
for parent_selector in ({'key2': 'value2'}, {'key1': 'value2'}):
documents[-1]['metadata']['layeringDefinition'][
'parentSelector'] = parent_selector
layering.DocumentLayering(documents, validate=False)
self.assertTrue(
any('Could not find parent for document' in
mock_log.debug.mock_calls[x][1][0])
for x in range(len(mock_log.debug.mock_calls)))
mock_log.debug.reset_mock()
def test_layering_document_references_itself(self, mock_log):
# Test that a parentSelector cannot reference the document itself
# without an error being raised.
doc_factory = factories.DocumentFactory(3, [1, 1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
self_ref = {"self": "self"}
documents[2]['metadata']['labels'] = self_ref
documents[2]['metadata']['layeringDefinition'][
'parentSelector'] = self_ref
layering.DocumentLayering(documents, validate=False)
self.assertTrue(
any('Could not find parent for document' in
mock_log.debug.mock_calls[x][1][0])
for x in range(len(mock_log.debug.mock_calls)))
def test_layering_with_broken_layer_order(self, mock_log):
doc_factory = factories.DocumentFactory(2, [1, 1])
documents = doc_factory.gen_test({}, site_abstract=False)
layering_policy = documents[0]
broken_layer_orders = [
['site', 'region', 'global'], ['broken', 'global'], ['broken'],
['site', 'broken']]
for broken_layer_order in broken_layer_orders:
layering_policy['data']['layerOrder'] = broken_layer_order
# The site will not be able to find a correct parent.
layering.DocumentLayering(documents)
self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0],
'%s is an empty layer with no documents. '
'It will be discarded from the layerOrder'
' during the layering process.')
mock_log.info.reset_mock()
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {'metadata.storagePolicy': ['cleartext'], 'deleted': False}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
documents = self._retrieve_documents_for_rendering(
revision_id, **filters)
substitution_sources = self._retrieve_substitution_sources()
try:
document_layering = layering.DocumentLayering(
documents, substitution_sources)
rendered_documents = document_layering.render()
except (errors.IndeterminateDocumentParent,
errors.UnsupportedActionMethod,
errors.MissingDocumentKey) as e:
raise falcon.HTTPBadRequest(description=e.format_message())
except (errors.LayeringPolicyNotFound,
errors.SubstitutionDependencyNotFound) as e:
raise falcon.HTTPConflict(description=e.format_message())
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
user_filters = sanitized_params.copy()
user_filters['metadata.layeringDefinition.abstract'] = False
rendered_documents = [
d for d in rendered_documents
if utils.deepfilter(d, **user_filters)
]
if sort_by:
rendered_documents = utils.multisort(rendered_documents, sort_by,
order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def test_layering_documents_with_different_schemas(self, mock_log):
"""Validate that attempting to layer documents with different schemas
results in errors.
"""
doc_factory = factories.DocumentFactory(3, [1, 1, 1])
documents = doc_factory.gen_test({})
# Region and site documents should result in no parent being found
# since their schemas will not match that of their parent's.
for idx in range(1, 3): # Only region/site have parent.
prev_schema = documents[idx]['schema']
documents[idx]['schema'] = test_utils.rand_name('schema')
layering.DocumentLayering(documents)
self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0],
'Could not find parent for document .*')
mock_log.info.reset_mock()
# Restore schema for next test run.
documents[idx]['schema'] = prev_schema
def deckhand_render(documents=[],
fail_on_missing_sub_src=False,
validate=False):
errors = []
rendered_documents = []
schemas, schema_errors = load_schemas_from_docs(documents)
errors.extend(schema_errors)
try:
deckhand_eng = layering.DocumentLayering(
documents,
substitution_sources=documents,
fail_on_missing_sub_src=fail_on_missing_sub_src,
validate=validate)
rendered_documents = [dict(d) for d in deckhand_eng.render()]
except dh_errors.DeckhandException as e:
errors.append((DECKHAND_RENDER_EXCEPTION,
'An unknown Deckhand exception occurred while trying'
' to render documents: %s' % str(e)))
return rendered_documents, errors
def do_render():
"""Perform document rendering for the revision."""
document_layering = layering.DocumentLayering(documents, **kwargs)
return document_layering.render()
