def deckhand_render(documents=None, fail_on_missing_sub_src=False, validate=True): documents = documents or [] errors = [] rendered_documents = [] schemas, schema_errors = load_schemas_from_docs(documents) errors.extend(schema_errors) try: deckhand_eng = layering.DocumentLayering( documents, fail_on_missing_sub_src=fail_on_missing_sub_src, validate=validate) rendered_documents = [dict(d) for d in deckhand_eng.render()] if validate: validator = document_validation.DocumentValidation( rendered_documents) results = validator.validate_all() for result in results: if result['errors']: errors.append( (DECKHAND_RENDER_EXCEPTION, 'During rendering Deckhand was unable to validate ' 'the following document, details: %s.' % (result['errors']))) except dh_errors.DeckhandException as e: errors.append((DECKHAND_RENDER_EXCEPTION, 'An unknown Deckhand exception occurred while trying' ' to render documents: %s. Details: %s.' % (str(e), e.error_list))) return rendered_documents, errors
def __init__(self, *, documents, debug=False, substitute=True, allow_missing_substitutions=True, leave_kubectl=False, validate=True): LOG.info("Parsing document schemas.") schema_set = validation.load_schemas_from_docs(documents) LOG.info("Parsed %d document schemas." % len(schema_set)) LOG.info("Building config from %d documents." % len(documents)) if substitute: LOG.info("Rendering documents via Deckhand engine.") try: deckhand_eng = layering.DocumentLayering( documents, fail_on_missing_sub_src=not allow_missing_substitutions) documents = [dict(d) for d in deckhand_eng.render()] except dh_errors.DeckhandException as e: LOG.exception( 'An unknown Deckhand exception occurred while trying' ' to render documents.') raise exceptions.DeckhandException(str(e)) LOG.info("Deckhand engine returned %d documents." % len(documents)) if validate: validation.check_schemas(documents, schemas=schema_set) self.debug = debug self.documents = documents self.leave_kubectl = leave_kubectl
def _test_layering(self, documents, site_expected=None, region_expected=None, global_expected=None, exception_expected=None, substitution_sources=None): document_layering = layering.DocumentLayering( documents, substitution_sources) if all([site_expected, region_expected, global_expected, exception_expected]): raise ValueError( '(site_expected|region_expected|global_expected) and ' '(exception_expected) are mutually exclusive.') if exception_expected: self.assertRaises(exception_expected, document_layering.render) return site_docs = [] region_docs = [] global_docs = [] # The layering policy is not returned as it is immutable. So all docs # should have a metadata.layeringDefinitionn.layer section. rendered_documents = document_layering.render() for doc in rendered_documents: # No need to validate the LayeringPolicy: it remains unchanged. if doc['schema'].startswith(types.LAYERING_POLICY_SCHEMA): continue layer = doc['metadata']['layeringDefinition']['layer'] if layer == 'site': site_docs.append(doc) if layer == 'region': region_docs.append(doc) if layer == 'global': global_docs.append(doc) if site_expected: if not isinstance(site_expected, list): site_expected = [site_expected] for idx, expected in enumerate(site_expected): self.assertEqual(expected, site_docs[idx].get('data'), 'Actual site data does not match expected.') if region_expected: if not isinstance(region_expected, list): region_expected = [region_expected] for idx, expected in enumerate(region_expected): self.assertEqual(expected, region_docs[idx].get('data'), 'Actual region data does not match expected.') if global_expected: if not isinstance(global_expected, list): global_expected = [global_expected] for idx, expected in enumerate(global_expected): self.assertEqual(expected, global_docs[idx].get('data'), 'Actual global data does not match expected.')
def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = { 'metadata.storagePolicy': ['cleartext'], 'deleted': False } if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') documents = self._retrieve_documents_for_rendering(revision_id, **filters) try: # NOTE(fmontei): `validate` is False because documents have already # been pre-validated during ingestion. Documents are post-validated # below, regardless. document_layering = layering.DocumentLayering( documents, validate=False) rendered_documents = document_layering.render() except (errors.InvalidDocumentLayer, errors.InvalidDocumentParent, errors.InvalidDocumentReplacement, errors.IndeterminateDocumentParent, errors.LayeringPolicyNotFound, errors.MissingDocumentKey, errors.SubstitutionSourceDataNotFound, errors.SubstitutionSourceNotFound, errors.UnknownSubstitutionError, errors.UnsupportedActionMethod) as e: with excutils.save_and_reraise_exception(): LOG.exception(e.format_message()) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) limit = sanitized_params.pop('limit', None) user_filters = sanitized_params.copy() rendered_documents = [ d for d in rendered_documents if utils.deepfilter( d, **user_filters)] if sort_by: rendered_documents = utils.multisort( rendered_documents, sort_by, order_by) if limit is not None: rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents) self._post_validate(rendered_documents)
def _test_layering(self, documents, site_expected=None, region_expected=None, global_expected=None, exception_expected=None): document_layering = layering.DocumentLayering(documents) if all([ site_expected, region_expected, global_expected, exception_expected ]): raise ValueError( '(site_expected|region_expected|global_expected) and ' '(exception_expected) are mutually exclusive.') if exception_expected: self.assertRaises(exception_expected, document_layering.render) return site_docs = [] region_docs = [] global_docs = [] # The layering policy is not returned as it is immutable. So all docs # should have a metadata.layeringDefinitionn.layer section. rendered_documents = document_layering.render() for doc in rendered_documents: layer = doc['metadata']['layeringDefinition']['layer'] if layer == 'site': site_docs.append(doc) if layer == 'region': region_docs.append(doc) if layer == 'global': global_docs.append(doc) if site_expected: if not isinstance(site_expected, list): site_expected = [site_expected] for idx, expected in enumerate(site_expected): self.assertEqual(expected, site_docs[idx].get('data')) if region_expected: if not isinstance(region_expected, list): region_expected = [region_expected] for idx, expected in enumerate(region_expected): self.assertEqual(expected, region_docs[idx].get('data')) if global_expected: if not isinstance(global_expected, list): global_expected = [global_expected] for idx, expected in enumerate(global_expected): self.assertEqual(expected, global_docs[idx].get('data'))
def test_layering_child_with_invalid_parent_selector(self, mock_log): doc_factory = factories.DocumentFactory(2, [1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) for parent_selector in ({'key2': 'value2'}, {'key1': 'value2'}): documents[-1]['metadata']['layeringDefinition'][ 'parentSelector'] = parent_selector layering.DocumentLayering(documents) self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0], 'Could not find parent for document .*') mock_log.info.reset_mock()
def test_layering_unreferenced_parent_label(self, mock_log): doc_factory = factories.DocumentFactory(2, [1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) for parent_label in ({'key2': 'value2'}, {'key1': 'value2'}): # Second doc is the global doc, or parent. documents[1]['metadata']['labels'] = parent_label layering.DocumentLayering(documents) self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0], 'Could not find parent for document .*') mock_log.info.reset_mock()
def test_layering_document_references_itself(self, mock_log): # Test that a parentSelector cannot reference the document itself # without an error being raised. doc_factory = factories.DocumentFactory(3, [1, 1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) self_ref = {"self": "self"} documents[2]['metadata']['labels'] = self_ref documents[2]['metadata']['layeringDefinition'][ 'parentSelector'] = self_ref layering.DocumentLayering(documents) self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0], 'Could not find parent for document .*')
def test_layering_unreferenced_parent_label(self, mock_log): doc_factory = factories.DocumentFactory(2, [1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) for parent_label in ({'key2': 'value2'}, {'key1': 'value2'}): # Second doc is the global doc, or parent. documents[1]['metadata']['labels'] = parent_label layering.DocumentLayering(documents, validate=False) self.assertTrue( any('Could not find parent for document' in mock_log.debug.mock_calls[x][1][0]) for x in range(len(mock_log.debug.mock_calls))) mock_log.debug.reset_mock()
def test_layering_child_with_invalid_parent_selector(self, mock_log): doc_factory = factories.DocumentFactory(2, [1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) for parent_selector in ({'key2': 'value2'}, {'key1': 'value2'}): documents[-1]['metadata']['layeringDefinition'][ 'parentSelector'] = parent_selector layering.DocumentLayering(documents, validate=False) self.assertTrue( any('Could not find parent for document' in mock_log.debug.mock_calls[x][1][0]) for x in range(len(mock_log.debug.mock_calls))) mock_log.debug.reset_mock()
def test_layering_document_references_itself(self, mock_log): # Test that a parentSelector cannot reference the document itself # without an error being raised. doc_factory = factories.DocumentFactory(3, [1, 1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) self_ref = {"self": "self"} documents[2]['metadata']['labels'] = self_ref documents[2]['metadata']['layeringDefinition'][ 'parentSelector'] = self_ref layering.DocumentLayering(documents, validate=False) self.assertTrue( any('Could not find parent for document' in mock_log.debug.mock_calls[x][1][0]) for x in range(len(mock_log.debug.mock_calls)))
def test_layering_with_broken_layer_order(self, mock_log): doc_factory = factories.DocumentFactory(2, [1, 1]) documents = doc_factory.gen_test({}, site_abstract=False) layering_policy = documents[0] broken_layer_orders = [ ['site', 'region', 'global'], ['broken', 'global'], ['broken'], ['site', 'broken']] for broken_layer_order in broken_layer_orders: layering_policy['data']['layerOrder'] = broken_layer_order # The site will not be able to find a correct parent. layering.DocumentLayering(documents) self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0], '%s is an empty layer with no documents. ' 'It will be discarded from the layerOrder' ' during the layering process.') mock_log.info.reset_mock()
def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = {'metadata.storagePolicy': ['cleartext'], 'deleted': False} if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') documents = self._retrieve_documents_for_rendering( revision_id, **filters) substitution_sources = self._retrieve_substitution_sources() try: document_layering = layering.DocumentLayering( documents, substitution_sources) rendered_documents = document_layering.render() except (errors.IndeterminateDocumentParent, errors.UnsupportedActionMethod, errors.MissingDocumentKey) as e: raise falcon.HTTPBadRequest(description=e.format_message()) except (errors.LayeringPolicyNotFound, errors.SubstitutionDependencyNotFound) as e: raise falcon.HTTPConflict(description=e.format_message()) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) user_filters = sanitized_params.copy() user_filters['metadata.layeringDefinition.abstract'] = False rendered_documents = [ d for d in rendered_documents if utils.deepfilter(d, **user_filters) ] if sort_by: rendered_documents = utils.multisort(rendered_documents, sort_by, order_by) resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents) self._post_validate(rendered_documents)
def test_layering_documents_with_different_schemas(self, mock_log): """Validate that attempting to layer documents with different schemas results in errors. """ doc_factory = factories.DocumentFactory(3, [1, 1, 1]) documents = doc_factory.gen_test({}) # Region and site documents should result in no parent being found # since their schemas will not match that of their parent's. for idx in range(1, 3): # Only region/site have parent. prev_schema = documents[idx]['schema'] documents[idx]['schema'] = test_utils.rand_name('schema') layering.DocumentLayering(documents) self.assertRegexpMatches(mock_log.info.mock_calls[0][1][0], 'Could not find parent for document .*') mock_log.info.reset_mock() # Restore schema for next test run. documents[idx]['schema'] = prev_schema
def deckhand_render(documents=[], fail_on_missing_sub_src=False, validate=False): errors = [] rendered_documents = [] schemas, schema_errors = load_schemas_from_docs(documents) errors.extend(schema_errors) try: deckhand_eng = layering.DocumentLayering( documents, substitution_sources=documents, fail_on_missing_sub_src=fail_on_missing_sub_src, validate=validate) rendered_documents = [dict(d) for d in deckhand_eng.render()] except dh_errors.DeckhandException as e: errors.append((DECKHAND_RENDER_EXCEPTION, 'An unknown Deckhand exception occurred while trying' ' to render documents: %s' % str(e))) return rendered_documents, errors
def do_render(): """Perform document rendering for the revision.""" document_layering = layering.DocumentLayering(documents, **kwargs) return document_layering.render()