def _find_layering_policy(self): """Retrieve the current layering policy. :raises LayeringPolicyMalformed: If the `layerOrder` could not be found in the LayeringPolicy or if it is not a list. :raises LayeringPolicyNotFound: If system has no layering policy. """ # TODO(fmontei): There should be a DB call here to fetch the layering # policy from the DB. for doc in self.documents: if doc.to_dict()['schema'] == self.LAYERING_POLICY_SCHEMA: self.layering_policy = doc break if not hasattr(self, 'layering_policy'): raise errors.LayeringPolicyNotFound( schema=self.LAYERING_POLICY_SCHEMA) # TODO(fmontei): Rely on schema validation or some such for this. try: self.layer_order = list(self.layering_policy['data']['layerOrder']) except KeyError: raise errors.LayeringPolicyMalformed( schema=self.LAYERING_POLICY_SCHEMA, document=self.layering_policy) if not isinstance(self.layer_order, list): raise errors.LayeringPolicyMalformed( schema=self.LAYERING_POLICY_SCHEMA, document=self.layering_policy)
def __init__(self, documents, substitution_sources=None): """Contructor for ``DocumentLayering``. :param layering_policy: The document with schema ``deckhand/LayeringPolicy`` needed for layering. :param documents: List of all other documents to be layered together in accordance with the ``layerOrder`` defined by the LayeringPolicy document. :type documents: List[dict] :param substitution_sources: List of documents that are potential sources for substitution. Should only include concrete documents. :type substitution_sources: List[dict] """ self._layering_policy, self._documents = self._extract_layering_policy( documents) if self._layering_policy is None: error_msg = ( 'No layering policy found in the system so could not reder ' 'documents.') LOG.error(error_msg) raise errors.LayeringPolicyNotFound() self._layered_docs = list( filter(lambda x: 'layeringDefinition' in x.metadata, self._documents)) self._layer_order = self._get_layering_order(self._layering_policy) self._parentless_documents = [] self._layered_documents = self._calc_document_children() self._substitution_sources = substitution_sources or []
def __init__(self, documents, validate=True, fail_on_missing_sub_src=True, encryption_sources=None, cleartext_secrets=False): """Contructor for ``DocumentLayering``. :param layering_policy: The document with schema ``deckhand/LayeringPolicy`` needed for layering. :param documents: List of all other documents to be layered together in accordance with the ``layerOrder`` defined by the LayeringPolicy document. :type documents: List[dict] :param validate: Whether to pre-validate documents using built-in schema validation. Skips over externally registered ``DataSchema`` documents to avoid false positives. Default is True. :type validate: bool :param fail_on_missing_sub_src: Whether to fail on a missing substitution source. Default is True. :type fail_on_missing_sub_src: bool :param encryption_sources: A dictionary that maps the reference contained in the destination document's data section to the actual unecrypted data. If encrypting data with Barbican, the reference will be a Barbican secret reference. :type encryption_sources: dict :param cleartext_secrets: Whether to show unencrypted data as cleartext. :type cleartext_secrets: bool :raises LayeringPolicyNotFound: If no LayeringPolicy was found among list of ``documents``. :raises InvalidDocumentLayer: If document layer not found in layerOrder for provided LayeringPolicy. :raises InvalidDocumentParent: If child references parent but they don't have the same schema or their layers are incompatible. :raises IndeterminateDocumentParent: If more than one parent document was found for a document. """ self._documents_by_layer = {} self._documents_by_labels = {} self._layering_policy = None self._sorted_documents = {} self._documents_by_index = {} # TODO(felipemonteiro): Add a hook for post-validation too. if validate: self._pre_validate_documents(documents) layering_policies = list( filter(lambda x: x.get('schema').startswith( types.LAYERING_POLICY_SCHEMA), documents)) if layering_policies: self._layering_policy = dd(layering_policies[0]) if len(layering_policies) > 1: LOG.warning('More than one layering policy document was ' 'passed in. Using the first one found: [%s] %s.', self._layering_policy.schema, self._layering_policy.name) if self._layering_policy is None: error_msg = ( 'No layering policy found in the system so could not render ' 'documents.') LOG.error(error_msg) raise errors.LayeringPolicyNotFound() for document in documents: document = dd(document) self._documents_by_index.setdefault(document.meta, document) if document.layer: if document.layer not in self._layering_policy.layer_order: LOG.error('Document layer %s for document [%s] %s not ' 'in layerOrder: %s.', document.layer, document.schema, document.name, self._layering_policy.layer_order) raise errors.InvalidDocumentLayer( document_layer=document.layer, document_schema=document.schema, document_name=document.name, layer_order=', '.join( self._layering_policy.layer_order), layering_policy_name=self._layering_policy.name) self._documents_by_layer.setdefault(document.layer, []) self._documents_by_layer[document.layer].append(document) if document.parent_selector: for label_key, label_val in document.parent_selector.items(): self._documents_by_labels.setdefault( (label_key, label_val), []) self._documents_by_labels[ (label_key, label_val)].append(document) self._layer_order = self._get_layering_order(self._layering_policy) self._calc_all_document_children() substitution_sources = self._calc_replacements_and_substitutions( [ d for d in self._documents_by_index.values() if not d.is_abstract ]) self.secrets_substitution = secrets_manager.SecretsSubstitution( substitution_sources, encryption_sources=encryption_sources, fail_on_missing_sub_src=fail_on_missing_sub_src, cleartext_secrets=cleartext_secrets) self._sorted_documents = self._topologically_sort_documents( substitution_sources) del self._documents_by_layer del self._documents_by_labels
def __init__(self, documents, substitution_sources=None, validate=True): """Contructor for ``DocumentLayering``. :param layering_policy: The document with schema ``deckhand/LayeringPolicy`` needed for layering. :param documents: List of all other documents to be layered together in accordance with the ``layerOrder`` defined by the LayeringPolicy document. :type documents: List[dict] :param substitution_sources: List of documents that are potential sources for substitution. Should only include concrete documents. :type substitution_sources: List[dict] :param validate: Whether to pre-validate documents using built-in schema validation. Default is True. :type validate: bool :raises LayeringPolicyNotFound: If no LayeringPolicy was found among list of ``documents``. :raises InvalidDocumentLayer: If document layer not found in layerOrder for provided LayeringPolicy. :raises InvalidDocumentParent: If child references parent but they don't have the same schema or their layers are incompatible. :raises IndeterminateDocumentParent: If more than one parent document was found for a document. """ self._documents_to_layer = [] self._documents_by_layer = {} self._documents_by_labels = {} self._layering_policy = None if validate: self._validate_documents(documents) layering_policies = list( filter( lambda x: x.get('schema').startswith( types.LAYERING_POLICY_SCHEMA), documents)) if layering_policies: self._layering_policy = document_wrapper.DocumentDict( layering_policies[0]) if len(layering_policies) > 1: LOG.warning( 'More than one layering policy document was ' 'passed in. Using the first one found: [%s] %s.', self._layering_policy.schema, self._layering_policy.name) if self._layering_policy is None: error_msg = ( 'No layering policy found in the system so could not render ' 'documents.') LOG.error(error_msg) raise errors.LayeringPolicyNotFound() sorted_documents = self._topologically_sort_documents(documents) for document in sorted_documents: document = document_wrapper.DocumentDict(document) if document.layering_definition: self._documents_to_layer.append(document) if document.layer: if document.layer not in self._layering_policy.layer_order: LOG.error( 'Document layer %s for document [%s] %s not ' 'in layerOrder: %s.', document.layer, document.schema, document.name, self._layering_policy.layer_order) raise errors.InvalidDocumentLayer( document_layer=document.layer, document_schema=document.schema, document_name=document.name, layer_order=', '.join( self._layering_policy.layer_order), layering_policy_name=self._layering_policy.name) self._documents_by_layer.setdefault(document.layer, []) self._documents_by_layer[document.layer].append(document) if document.parent_selector: for label_key, label_val in document.parent_selector.items(): self._documents_by_labels.setdefault( (label_key, label_val), []) self._documents_by_labels[(label_key, label_val)].append(document) self._layer_order = self._get_layering_order(self._layering_policy) self._calc_all_document_children() self._substitution_sources = substitution_sources or [] self.secrets_substitution = secrets_manager.SecretsSubstitution( self._substitution_sources) del self._documents_by_layer del self._documents_by_labels