def filter_revision_documents(documents, unique_only, **filters): """Return the list of documents that match filters. :param documents: List of documents to apply ``filters`` to. :param unique_only: Return only unique documents if ``True``. :param filters: Dictionary attributes (including nested) used to filter out revision documents. :returns: List of documents that match specified filters. """ filtered_documents = {} unique_filters = ('schema', 'name', 'layer') exclude_deleted = filters.pop('deleted', None) is False if exclude_deleted: documents = exclude_deleted_documents(documents) for document in documents: if utils.deepfilter(document, **filters): # Filter out redundant documents from previous revisions, i.e. # documents schema and metadata.name are repeated. if unique_only: unique_key = tuple( [document[filter] for filter in unique_filters]) else: unique_key = document['id'] if unique_key not in filtered_documents: filtered_documents[unique_key] = document return list(filtered_documents.values())
def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = { 'metadata.storagePolicy': ['cleartext'], 'deleted': False } if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') rendered_documents = common.get_rendered_docs(revision_id, **filters) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) limit = sanitized_params.pop('limit', None) user_filters = sanitized_params.copy() rendered_documents = [ d for d in rendered_documents if utils.deepfilter( d, **user_filters)] if sort_by: rendered_documents = utils.multisort( rendered_documents, sort_by, order_by) if limit is not None: rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 self._post_validate(rendered_documents) resp.body = self.view_builder.list(rendered_documents)
def on_get(self, req, resp, sanitized_params, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = { 'metadata.storagePolicy': ['cleartext'], 'deleted': False } if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') documents = self._retrieve_documents_for_rendering(revision_id, **filters) try: # NOTE(fmontei): `validate` is False because documents have already # been pre-validated during ingestion. Documents are post-validated # below, regardless. document_layering = layering.DocumentLayering( documents, validate=False) rendered_documents = document_layering.render() except (errors.InvalidDocumentLayer, errors.InvalidDocumentParent, errors.InvalidDocumentReplacement, errors.IndeterminateDocumentParent, errors.LayeringPolicyNotFound, errors.MissingDocumentKey, errors.SubstitutionSourceDataNotFound, errors.SubstitutionSourceNotFound, errors.UnknownSubstitutionError, errors.UnsupportedActionMethod) as e: with excutils.save_and_reraise_exception(): LOG.exception(e.format_message()) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) limit = sanitized_params.pop('limit', None) user_filters = sanitized_params.copy() rendered_documents = [ d for d in rendered_documents if utils.deepfilter( d, **user_filters)] if sort_by: rendered_documents = utils.multisort( rendered_documents, sort_by, order_by) if limit is not None: rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents) self._post_validate(rendered_documents)
def on_get(self, req, resp, revision_id): include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = { 'metadata.storagePolicy': ['cleartext'], 'deleted': False } if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') cleartext_secrets = req.get_param_as_bool('cleartext-secrets') if cleartext_secrets is None: cleartext_secrets = True req.params.pop('cleartext-secrets', None) rendered_documents, cache_hit = common.get_rendered_docs( revision_id, cleartext_secrets, **filters) # If the rendered documents result set is cached, then post-validation # for that result set has already been performed successfully, so it # can be safely skipped over as an optimization. if not cache_hit: data_schemas = db_api.revision_documents_get( schema=types.DATA_SCHEMA_SCHEMA, deleted=False) validator = document_validation.DocumentValidation( rendered_documents, data_schemas, pre_validate=False) engine.validate_render(revision_id, rendered_documents, validator) # Filters to be applied post-rendering, because many documents are # involved in rendering. User filters can only be applied once all # documents have been rendered. Note that `layering` module only # returns concrete documents, so no filtering for that is needed here. order_by = req.params.pop('order', None) sort_by = req.params.pop('sort', None) limit = req.params.pop('limit', None) user_filters = req.params.copy() if not cleartext_secrets: rendered_documents = utils.redact_documents(rendered_documents) rendered_documents = [ d for d in rendered_documents if utils.deepfilter( d, **user_filters)] if sort_by: rendered_documents = utils.multisort( rendered_documents, sort_by, order_by) if limit is not None: rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 resp.body = self.view_builder.list(rendered_documents)
def document_get_all(session=None, raw_dict=False, revision_id=None, **filters): """Retrieve all documents for ``revision_id`` that match ``filters``. :param session: Database session object. :param raw_dict: Whether to retrieve the exact way the data is stored in DB if ``True``, else the way users expect the data. :param revision_id: The ID corresponding to the ``Revision`` object. If the it is "latest", then retrieve the latest revision, if one exists. :param filters: Dictionary attributes (including nested) used to filter out revision documents. :returns: Dictionary representation of each retrieved document. """ session = session or get_session() if revision_id == 'latest': revision = session.query(models.Revision)\ .order_by(models.Revision.created_at.desc())\ .first() if revision: filters['revision_id'] = revision.id elif revision_id: filters['revision_id'] = revision_id # TODO(fmontei): Currently Deckhand doesn't support filtering by nested # JSON fields via sqlalchemy. For now, filter the documents using all # "regular" filters via sqlalchemy and all nested filters via Python. nested_filters = {} for f in filters.copy(): if any([x in f for x in ('.', 'schema')]): nested_filters.setdefault(f, filters.pop(f)) # Retrieve the most recently created documents for the revision, because # documents with the same metadata.name and schema can exist across # different revisions. documents = session.query(models.Document)\ .filter_by(**filters)\ .order_by(models.Document.created_at.desc())\ .all() final_documents = [] for doc in documents: d = doc.to_dict(raw_dict=raw_dict) if utils.deepfilter(d, **nested_filters): final_documents.append(d) return final_documents
def bucket_get_all(session=None, **filters): """Return list of all buckets. :param session: Database session object. :returns: List of dictionary representations of retrieved buckets. """ session = session or get_session() buckets = session.query(models.Bucket)\ .all() result = [] for bucket in buckets: revision_dict = bucket.to_dict() if utils.deepfilter(revision_dict, **filters): result.append(bucket) return result
def revision_get_all(session=None, **filters): """Return list of all revisions. :param session: Database session object. :returns: List of dictionary representations of retrieved revisions. """ session = session or get_session() revisions = session.query(models.Revision)\ .all() result = [] for revision in revisions: revision_dict = revision.to_dict() if utils.deepfilter(revision_dict, **filters): revision_dict['documents'] = _update_revision_history( revision_dict['documents']) result.append(revision_dict) return result