def _test_doc_substitution(self, document_mapping, substitution_sources, expected_data, encryption_sources=None, cleartext_secrets=True): payload = self.document_factory.gen_test(document_mapping, global_abstract=False) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents(bucket_name, substitution_sources + [payload[-1]]) expected_document = copy.deepcopy(documents[-1]) expected_document['data'] = expected_data secret_substitution = secrets_manager.SecretsSubstitution( encryption_sources=encryption_sources, substitution_sources=substitution_sources, cleartext_secrets=cleartext_secrets) substituted_docs = list(secret_substitution.substitute_all(documents)) self.assertIn(expected_document, substituted_docs)
def test_show_revision(self): payload = [ base.DocumentFixture.get_minimal_fixture() for _ in range(4) ] bucket_name = test_utils.rand_name('bucket') documents = self.create_documents(bucket_name, payload) # Validate that each document points to the same revision. revision_ids = set([d['revision_id'] for d in documents]) self.assertEqual(1, len(revision_ids)) revision = self.show_revision(documents[0]['revision_id']) revision_view = self.view_builder.show(revision) expected_attrs = ('id', 'url', 'createdAt', 'validationPolicies', 'status') for attr in expected_attrs: self.assertIn(attr, revision_view) self.assertIsInstance(revision_view['validationPolicies'], list) self.assertEqual(revision_view['validationPolicies'], [])
def test_delete_all_documents(self): payload = self.documents_factory.gen_test(self.document_mapping) bucket_name = test_utils.rand_name('bucket') created_documents = self.create_documents(bucket_name, payload) self.assertIsInstance(created_documents, list) self.assertEqual(3, len(created_documents)) deleted_documents = self.create_documents(bucket_name, []) # Verify that all the expected documents were deleted. self.assertEqual( sorted([(d['metadata']['name'], d['schema']) for d in created_documents]), sorted([(d['name'], d['schema']) for d in deleted_documents])) # Verify that all their attributes have been cleared and that the # deleted/deleted_at attributes have been set to True. for deleted_document in deleted_documents: self.assertTrue(deleted_document['deleted']) self.assertTrue(deleted_document['deleted_at']) self.assertEmpty(deleted_document['data'])
def test_create_validation(self): rules = { 'deckhand:create_cleartext_documents': '@', 'deckhand:create_validation': '@' } self.policy.set_rules(rules) revision_id = self._create_revision() validation_name = test_utils.rand_name('validation') resp = self._create_validation(revision_id, validation_name, VALIDATION_RESULT) self.assertEqual(201, resp.status_code) expected_body = { 'status': 'failure', 'validator': { 'name': 'promenade', 'version': '1.1.2' } } self.assertEqual(expected_body, yaml.safe_load(resp.text))
def test_show_nonexistent_validation_entry_returns_404(self): rules = {'deckhand:create_cleartext_documents': '@', 'deckhand:create_validation': '@', 'deckhand:show_validation': '@'} self.policy.set_rules(rules) revision_id = self._create_revision() validation_name = test_utils.rand_name('validation') resp = self._create_validation(revision_id, validation_name, VALIDATION_FAILURE_RESULT) self.assertEqual(201, resp.status_code) expected_error = ('The requested validation entry 5 was not found for ' 'validation name %s and revision ID %d.' % ( validation_name, revision_id)) resp = self.app.simulate_get( '/api/v1.0/revisions/%s/validations/%s/entries/5' % ( revision_id, validation_name), headers={'Content-Type': 'application/x-yaml'}) self.assertEqual(404, resp.status_code) self.assertEqual(expected_error, yaml.safe_load(resp.text)['message'])
def test_list_documents_by_revision_id_and_filters(self): payload = self.documents_factory.gen_test(self.document_mapping) bucket_name = test_utils.rand_name('bucket') document = self.create_documents(bucket_name, payload)[1] filters = { 'schema': document['schema'], 'metadata.name': document['metadata']['name'], 'metadata.layeringDefinition.abstract': document['metadata']['layeringDefinition']['abstract'], 'metadata.layeringDefinition.layer': document['metadata']['layeringDefinition']['layer'] } documents = self.list_revision_documents(document['revision_id'], **filters) self.assertEqual(1, len(documents)) self.assertEqual(document, documents[0])
def test_get_documents_by_revision_id_and_wrong_filters(self): payload = base.DocumentFixture.get_minimal_fixture() bucket_name = test_utils.rand_name('bucket') document = self.create_documents(bucket_name, payload)[0] filters = { 'schema': 'fake_schema', 'metadata.name': 'fake_meta_name', 'metadata.layeringDefinition.abstract': not document['metadata']['layeringDefinition']['abstract'], 'metadata.layeringDefinition.layer': 'fake_layer', 'metadata.label': 'fake_label' } documents = self.list_revision_documents( document['revision_id'], **filters) self.assertEmpty(documents) for filter_key, filter_val in filters.items(): documents = self.list_revision_documents( document['revision_id'], filter_key=filter_val) self.assertEmpty(documents)
def test_create_show_and_list_many_tags_with_data(self): expected_tags = [] for _ in range(4): rand_prefix = test_utils.rand_name(self.__class__.__name__) tag = rand_prefix + '-Tag' data_key = rand_prefix + '-Key' data_val = rand_prefix + '-Val' db_api.revision_tag_create( self.revision_id, tag, {data_key: data_val}) expected_tags.append({'tag': tag, 'data': {data_key: data_val}}) expected_tags = sorted(expected_tags, key=lambda t: t['tag']) retrieved_tags = db_api.revision_tag_get_all(self.revision_id) self.assertEqual(4, len(retrieved_tags)) retrieved_tags = [ {k: t[k] for k in t.keys() if k in ('data', 'tag')} for t in retrieved_tags] self.assertEqual(sorted(expected_tags, key=lambda t: t['tag']), retrieved_tags)
def test_show_validation_entry(self): rules = {'deckhand:create_cleartext_documents': '@', 'deckhand:create_validation': '@', 'deckhand:show_validation': '@'} self.policy.set_rules(rules) revision_id = self._create_revision() validation_name = test_utils.rand_name('validation') resp = self._create_validation(revision_id, validation_name, VALIDATION_FAILURE_RESULT) resp = self.app.simulate_get( '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( revision_id, validation_name), headers={'Content-Type': 'application/x-yaml'}) self.assertEqual(200, resp.status_code) body = yaml.safe_load(resp.text) expected_body = { 'name': validation_name, 'status': 'failure', 'createdAt': None, 'expiresAfter': None, 'errors': [ { 'documents': [ { 'name': 'node-document-name', 'schema': 'promenade/Node/v1' }, { 'name': 'kubernetes-masters', 'schema': 'promenade/Masters/v1' } ], 'message': 'Node has master role, but not included in ' 'cluster masters list.' } ] } self.assertEqual(expected_body, body)
def test_layering_documents_with_different_schemas(self): """Validate that attempting to layer documents with different schemas results in errors. """ doc_factory = factories.DocumentFactory(3, [1, 1, 1]) documents = doc_factory.gen_test({}) # Region and site documents should result in no parent being found # since their schemas will not match that of their parent's. for idx in range(2, 4): # Only region/site have parent. prev_schema = documents[idx]['schema'] documents[idx]['schema'] = test_utils.rand_name('schema') # Escape '[' and ']' for regex to work. expected_err = ("Missing parent document for document %s." % documents[idx]).replace('[', '\[').replace(']', '\]') self.assertRaisesRegex(errors.MissingDocumentParent, expected_err, layering.DocumentLayering, documents) # Restore schema for next test run. documents[idx]['schema'] = prev_schema
def test_create_duplicate_bucket(self): """Validates that creating a bucket with the exact same documents references the last revision via orig_revision_id, as in reality the bucket is merely a revision history placeholder for the first bucket, since nothing was changed. Note that this is different from creating a duplicate document or creating a duplicate document in a separate bucket. """ bucket_name = test_utils.rand_name('bucket') payload = base.DocumentFixture.get_minimal_fixture() orig_documents = self.create_documents(bucket_name, [payload]) duplicate_documents = self.create_documents(bucket_name, [payload]) self.assertEqual(orig_documents[0]['revision_id'], duplicate_documents[0]['orig_revision_id']) self.assertDictItemsAlmostEqual( sorted(orig_documents, key=lambda d: d['created_at']), sorted(duplicate_documents, key=lambda d: d['created_at']), ignore=['created_at', 'updated_at', 'revision_id', 'id'])
def test_list_multiple_revisions(self): docs_count = [] for _ in range(3): doc_count = test_utils.rand_int(3, 9) docs_count.append(doc_count) payload = [ base.DocumentFixture.get_minimal_fixture() for _ in range(doc_count) ] bucket_name = test_utils.rand_name('bucket') self.create_documents(bucket_name, payload) revisions = self.list_revisions() revisions_view = self.view_builder.list(revisions) self.assertIn('results', revisions_view) # Validate that only 1 revision was returned. self.assertEqual(3, revisions_view['count']) # Validate that each revision has correct number of documents. for idx, doc_count in enumerate(docs_count): self.assertIn('id', revisions_view['results'][idx])
def test_get_documents_by_revision_id_and_filters(self): payload = self.documents_factory.gen_test(self.document_mapping) bucket_name = test_utils.rand_name('bucket') created_documents = self.create_documents(bucket_name, payload) for document in created_documents[1:]: filters = { 'schema': document['schema'], 'metadata.name': document['metadata']['name'], 'metadata.layeringDefinition.abstract': document['metadata']['layeringDefinition']['abstract'], 'metadata.layeringDefinition.layer': document['metadata']['layeringDefinition']['layer'] } filtered_documents = self.list_revision_documents( document['revision_id'], **filters) self.assertEqual(1, len(filtered_documents)) self.assertIsNone(filtered_documents[0].pop('orig_revision_id')) self.assertEqual(document, filtered_documents[0])
def _test_document_creation_view(self, count): # Test document creation view with the number of documents being # created specified by `count`. payload = [ base.DocumentFixture.get_minimal_fixture() for _ in range(count) ] bucket_name = test_utils.rand_name('bucket') created_documents = self.create_documents(bucket_name, payload) document_view = self.view_builder.list(created_documents) self.assertIsInstance(document_view, list) self.assertEqual(count, len(document_view)) expected_attrs = ('id', 'status', 'metadata', 'data', 'schema') for idx in range(count): for attr in expected_attrs: self.assertIn(attr, document_view[idx]) for attr in ('bucket', 'revision'): self.assertIn(attr, document_view[idx]['status']) revision_ids = set([v['status']['revision'] for v in document_view]) self.assertEqual([1], list(revision_ids))
def test_revision_diff_multi_bucket_created(self): revision_ids = [] bucket_names = [] for _ in range(3): payload = base.DocumentFixture.get_minimal_multi_fixture(count=3) bucket_name = test_utils.rand_name('bucket') bucket_names.append(bucket_name) documents = self.create_documents(bucket_name, payload) revision_id = documents[0]['revision_id'] revision_ids.append(revision_id) # Between revision 1 and 0, 1 bucket is created. self._verify_buckets_status( 0, revision_ids[0], {b: 'created' for b in bucket_names[:1]}) # Between revision 2 and 0, 2 buckets are created. self._verify_buckets_status( 0, revision_ids[1], {b: 'created' for b in bucket_names[:2]}) # Between revision 3 and 0, 3 buckets are created. self._verify_buckets_status( 0, revision_ids[2], {b: 'created' for b in bucket_names})
def test_revision_diff_delete_then_recreate(self): payload = base.DocumentFixture.get_minimal_fixture() bucket_name = test_utils.rand_name('bucket') created_documents = self.create_documents(bucket_name, payload) revision_id_1 = created_documents[0]['revision_id'] # Delete the previously created document. deleted_documents = self.create_documents(bucket_name, []) revision_id_2 = deleted_documents[0]['revision_id'] # Recreate the previously deleted document. recreated_documents = self.create_documents(bucket_name, payload) revision_id_3 = recreated_documents[0]['revision_id'] # Verify that the revision for recreated document compared to revision # for deleted document is created, ignoring order. self._verify_buckets_status( revision_id_2, revision_id_3, {bucket_name: 'created'}) # Verify that the revision for recreated document compared to revision # for created document is unmodified, ignoring order. self._verify_buckets_status( revision_id_1, revision_id_3, {bucket_name: 'unmodified'})
def test_delete_and_create_document_in_same_payload(self): payload = self.documents_factory.gen_test(self.document_mapping) bucket_name = test_utils.rand_name('bucket') # Create just 1 document. self.create_documents(bucket_name, payload[0]) # Create the document in payload[0] but create a new document for # payload[1]. documents = self.create_documents(bucket_name, payload[1]) # Information about the deleted and created document should've been # returned. The 1st document is the deleted one and the 2nd document # is the created one. self.assertEqual(2, len(documents)) # Check that deleted doc is formatted correctly. self.assertTrue(documents[0]['deleted']) self.assertTrue(documents[0]['deleted_at']) self.assertEmpty(documents[0]['data']) # Check that created doc isn't deleted. self.assertFalse(documents[1]['deleted']) for idx in range(2): self.assertEqual(documents[idx]['schema'], payload[idx]['schema']) self.assertEqual(documents[idx]['name'], payload[idx]['metadata']['name'])
def test_create_duplicate_document_same_bucket_raises_exc(self): bucket_name = test_utils.rand_name('bucket') document = base.DocumentFixture.get_minimal_fixture() payload = [document, document.copy()] self.assertRaises(errors.DuplicateDocumentExists, self.create_documents, bucket_name, payload)
def gen_test(self, mapping, site_abstract=True, region_abstract=True, global_abstract=True, site_parent_selectors=None): """Generate the document template. Generate the document template based on the arguments passed to the constructor and to this function. :param mapping: A list of dictionaries that specify the "data" and "actions" parameters for each document. A valid mapping is:: mapping = { "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}, "_SITE_DATA_1_": {"data": {"a": {"x": 7, "z": 3}, "b": 4}}, "_SITE_ACTIONS_1_": { "actions": [{"method": "merge", "path": path}]} } Each key must be of the form "_{LAYER_NAME}_{KEY_NAME}_{N}_" where: - {LAYER_NAME} is the name of the layer ("global", "region", "site") - {KEY_NAME} is either "DATA" or "ACTIONS" - {N} is the occurrence of the document based on the values in ``docs_per_layer``. If ``docs_per_layer`` is (1, 2) then _GLOBAL_DATA_1_, _SITE_DATA_1_, _SITE_DATA_2_, _SITE_ACTIONS_1_ and _SITE_ACTIONS_2_ must be provided. _GLOBAL_ACTIONS_{N}_ is ignored. :type mapping: dict :param site_abstract: Whether site layers are abstract/concrete. :type site_abstract: boolean :param region_abstract: Whether region layers are abstract/concrete. :type region_abstract: boolean :param global_abstract: Whether global layers are abstract/concrete. :type global_abstract: boolean :param site_parent_selectors: Override the default parent selector for each site. Assuming that ``docs_per_layer`` is (2, 2), for example, a valid value is:: [{'global': 'global1'}, {'global': 'global2'}] If not specified, each site will default to the first parent. :type site_parent_selectors: list :returns: Rendered template of the form specified above. """ rendered_template = [self.layering_policy] layer_order = rendered_template[0]['data']['layerOrder'] for layer_idx in range(self.num_layers): for count in range(self.docs_per_layer[layer_idx]): layer_template = copy.deepcopy(self.DOCUMENT_TEMPLATE) layer_name = layer_order[layer_idx] layer_template = copy.deepcopy(layer_template) # Set name. name_key = "_%s_NAME_%d_" % (layer_name.upper(), count + 1) if name_key in mapping: layer_template['metadata']['name'] = mapping[name_key] else: layer_template['metadata']['name'] = "%s%d" % ( test_utils.rand_name(layer_name), count + 1) # Set schema. schema_key = "_%s_SCHEMA_%d_" % (layer_name.upper(), count + 1) if schema_key in mapping: layer_template['schema'] = mapping[schema_key] # Set layer. layer_template['metadata']['layeringDefinition'][ 'layer'] = layer_name # Set labels. layer_template['metadata']['labels'] = { layer_name: "%s%d" % (layer_name, count + 1) } # Set parentSelector. if layer_name == 'site' and site_parent_selectors: parent_selector = site_parent_selectors[count] layer_template['metadata']['layeringDefinition'][ 'parentSelector'] = parent_selector elif layer_idx > 0: parent_selector = rendered_template[layer_idx]['metadata'][ 'labels'] layer_template['metadata']['layeringDefinition'][ 'parentSelector'] = parent_selector # Set abstract. if layer_name == 'site': layer_template['metadata']['layeringDefinition'][ 'abstract'] = site_abstract if layer_name == 'region': layer_template['metadata']['layeringDefinition'][ 'abstract'] = region_abstract if layer_name == 'global': layer_template['metadata']['layeringDefinition'][ 'abstract'] = global_abstract # Set data and actions. data_key = "_%s_DATA_%d_" % (layer_name.upper(), count + 1) actions_key = "_%s_ACTIONS_%d_" % (layer_name.upper(), count + 1) sub_key = "_%s_SUBSTITUTIONS_%d_" % (layer_name.upper(), count + 1) try: layer_template['data'] = mapping[data_key]['data'] except KeyError as e: LOG.debug( 'Could not map %s because it was not found in ' 'the `mapping` dict.', e.args[0]) try: layer_template['metadata']['layeringDefinition'][ 'actions'] = mapping[actions_key]['actions'] except KeyError as e: LOG.debug( 'Could not map %s because it was not found in ' 'the `mapping` dict.', e.args[0]) try: layer_template['metadata']['substitutions'] = mapping[ sub_key] except KeyError as e: LOG.debug( 'Could not map %s because it was not found in ' 'the `mapping` dict.', e.args[0]) rendered_template.append(layer_template) return rendered_template
def __init__(self, num_layers, docs_per_layer): """Constructor for ``DocumentFactory``. Returns a template whose JSON representation is of the form:: [{'data': {'layerOrder': ['global', 'region', 'site']}, 'metadata': {'name': 'layering-policy', 'schema': 'metadata/Control/v1'}, 'schema': 'deckhand/LayeringPolicy/v1'}, {'data': {'a': 1, 'b': 2}, 'metadata': {'labels': {'global': 'global1'}, 'layeringDefinition': {'abstract': True, 'actions': [], 'layer': 'global', 'parentSelector': ''}, 'name': 'global1', 'schema': 'metadata/Document/v1'}, 'schema': 'example/Kind/v1'} ... ] :param num_layers: Total number of layers. Only supported values include 1, 2 or 3. :type num_layers: integer :param docs_per_layer: The number of documents to be included per layer. For example, if ``num_layers`` is 3, then ``docs_per_layer`` can be (1, 1, 1) for 1 document for each layer or (1, 2, 3) for 1 doc for the 1st layer, 2 docs for the 2nd layer, and 3 docs for the 3rd layer. :type docs_per_layer: tuple, list :raises TypeError: If ``docs_per_layer`` is not the right type. :raises ValueError: If ``num_layers`` is not the right value or isn't compatible with ``docs_per_layer``. """ # Set up the layering definition's layerOrder. if num_layers == 1: layer_order = ["global"] elif num_layers == 2: layer_order = ["global", "site"] elif num_layers == 3: layer_order = ["global", "region", "site"] else: raise ValueError("'num_layers' must be a value between 1 - 3.") self.layering_policy = copy.deepcopy(self.LAYERING_POLICY_TEMPLATE) self.layering_policy['metadata']['name'] = test_utils.rand_name( 'layering-policy') self.layering_policy['data']['layerOrder'] = layer_order self.layering_policy['metadata']['layeringDefinition'][ 'layer'] = layer_order[0] if not isinstance(docs_per_layer, (list, tuple)): raise TypeError("'docs_per_layer' must be a list or tuple " "indicating the number of documents per layer.") elif not len(docs_per_layer) == num_layers: raise ValueError("The number of entries in 'docs_per_layer' must" "be equal to the value of 'num_layers'.") for doc_count in docs_per_layer: if doc_count < 0: raise ValueError( "Each entry in 'docs_per_layer' must be >= 1.") self.num_layers = num_layers self.docs_per_layer = docs_per_layer