def filter_revision_documents(documents, unique_only, **filters):
"""Return the list of documents that match filters.
:param documents: List of documents to apply ``filters`` to.
:param unique_only: Return only unique documents if ``True``.
:param filters: Dictionary attributes (including nested) used to filter
out revision documents.
:returns: List of documents that match specified filters.
"""
filtered_documents = {}
unique_filters = ('schema', 'name', 'layer')
exclude_deleted = filters.pop('deleted', None) is False
if exclude_deleted:
documents = exclude_deleted_documents(documents)
for document in documents:
if utils.deepfilter(document, **filters):
# Filter out redundant documents from previous revisions, i.e.
# documents schema and metadata.name are repeated.
if unique_only:
unique_key = tuple(
[document[filter] for filter in unique_filters])
else:
unique_key = document['id']
if unique_key not in filtered_documents:
filtered_documents[unique_key] = document
return list(filtered_documents.values())
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {
'metadata.storagePolicy': ['cleartext'],
'deleted': False
}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
rendered_documents = common.get_rendered_docs(revision_id, **filters)
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
limit = sanitized_params.pop('limit', None)
user_filters = sanitized_params.copy()
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
if limit is not None:
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
self._post_validate(rendered_documents)
resp.body = self.view_builder.list(rendered_documents)
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {
'metadata.storagePolicy': ['cleartext'],
'deleted': False
}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
documents = self._retrieve_documents_for_rendering(revision_id,
**filters)
try:
# NOTE(fmontei): `validate` is False because documents have already
# been pre-validated during ingestion. Documents are post-validated
# below, regardless.
document_layering = layering.DocumentLayering(
documents, validate=False)
rendered_documents = document_layering.render()
except (errors.InvalidDocumentLayer,
errors.InvalidDocumentParent,
errors.InvalidDocumentReplacement,
errors.IndeterminateDocumentParent,
errors.LayeringPolicyNotFound,
errors.MissingDocumentKey,
errors.SubstitutionSourceDataNotFound,
errors.SubstitutionSourceNotFound,
errors.UnknownSubstitutionError,
errors.UnsupportedActionMethod) as e:
with excutils.save_and_reraise_exception():
LOG.exception(e.format_message())
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
limit = sanitized_params.pop('limit', None)
user_filters = sanitized_params.copy()
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
if limit is not None:
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def on_get(self, req, resp, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {
'metadata.storagePolicy': ['cleartext'],
'deleted': False
}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
cleartext_secrets = req.get_param_as_bool('cleartext-secrets')
if cleartext_secrets is None:
cleartext_secrets = True
req.params.pop('cleartext-secrets', None)
rendered_documents, cache_hit = common.get_rendered_docs(
revision_id, cleartext_secrets, **filters)
# If the rendered documents result set is cached, then post-validation
# for that result set has already been performed successfully, so it
# can be safely skipped over as an optimization.
if not cache_hit:
data_schemas = db_api.revision_documents_get(
schema=types.DATA_SCHEMA_SCHEMA, deleted=False)
validator = document_validation.DocumentValidation(
rendered_documents, data_schemas, pre_validate=False)
engine.validate_render(revision_id, rendered_documents, validator)
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = req.params.pop('order', None)
sort_by = req.params.pop('sort', None)
limit = req.params.pop('limit', None)
user_filters = req.params.copy()
if not cleartext_secrets:
rendered_documents = utils.redact_documents(rendered_documents)
rendered_documents = [
d for d in rendered_documents if utils.deepfilter(
d, **user_filters)]
if sort_by:
rendered_documents = utils.multisort(
rendered_documents, sort_by, order_by)
if limit is not None:
rendered_documents = rendered_documents[:limit]
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
def document_get_all(session=None,
raw_dict=False,
revision_id=None,
**filters):
"""Retrieve all documents for ``revision_id`` that match ``filters``.
:param session: Database session object.
:param raw_dict: Whether to retrieve the exact way the data is stored in
DB if ``True``, else the way users expect the data.
:param revision_id: The ID corresponding to the ``Revision`` object. If the
it is "latest", then retrieve the latest revision, if one exists.
:param filters: Dictionary attributes (including nested) used to filter
out revision documents.
:returns: Dictionary representation of each retrieved document.
"""
session = session or get_session()
if revision_id == 'latest':
revision = session.query(models.Revision)\
.order_by(models.Revision.created_at.desc())\
.first()
if revision:
filters['revision_id'] = revision.id
elif revision_id:
filters['revision_id'] = revision_id
# TODO(fmontei): Currently Deckhand doesn't support filtering by nested
# JSON fields via sqlalchemy. For now, filter the documents using all
# "regular" filters via sqlalchemy and all nested filters via Python.
nested_filters = {}
for f in filters.copy():
if any([x in f for x in ('.', 'schema')]):
nested_filters.setdefault(f, filters.pop(f))
# Retrieve the most recently created documents for the revision, because
# documents with the same metadata.name and schema can exist across
# different revisions.
documents = session.query(models.Document)\
.filter_by(**filters)\
.order_by(models.Document.created_at.desc())\
.all()
final_documents = []
for doc in documents:
d = doc.to_dict(raw_dict=raw_dict)
if utils.deepfilter(d, **nested_filters):
final_documents.append(d)
return final_documents
def bucket_get_all(session=None, **filters):
"""Return list of all buckets.
:param session: Database session object.
:returns: List of dictionary representations of retrieved buckets.
"""
session = session or get_session()
buckets = session.query(models.Bucket)\
.all()
result = []
for bucket in buckets:
revision_dict = bucket.to_dict()
if utils.deepfilter(revision_dict, **filters):
result.append(bucket)
return result
def revision_get_all(session=None, **filters):
"""Return list of all revisions.
:param session: Database session object.
:returns: List of dictionary representations of retrieved revisions.
"""
session = session or get_session()
revisions = session.query(models.Revision)\
.all()
result = []
for revision in revisions:
revision_dict = revision.to_dict()
if utils.deepfilter(revision_dict, **filters):
revision_dict['documents'] = _update_revision_history(
revision_dict['documents'])
result.append(revision_dict)
return result
