def _filter_revision_documents(documents, unique_only, **filters):
"""Return the list of documents that match filters.
:param documents: List of documents to apply ``filters`` to.
:param unique_only: Return only unique documents if ``True``.
:param filters: Dictionary attributes (including nested) used to filter
out revision documents.
:returns: List of documents that match specified filters.
"""
# TODO(fmontei): Implement this as an sqlalchemy query.
filtered_documents = {}
unique_filters = ('schema', 'name')
exclude_deleted = filters.pop('deleted', None) is False
if exclude_deleted:
documents = _exclude_deleted_documents(documents)
for document in documents:
if utils.deepfilter(document, **filters):
# Filter out redundant documents from previous revisions, i.e.
# documents schema and metadata.name are repeated.
if unique_only:
unique_key = tuple(
[document[filter] for filter in unique_filters])
else:
unique_key = document['id']
if unique_key not in filtered_documents:
filtered_documents[unique_key] = document
return list(filtered_documents.values())
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {'metadata.storagePolicy': ['cleartext'], 'deleted': False}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
documents = self._retrieve_documents_for_rendering(
revision_id, **filters)
substitution_sources = self._retrieve_substitution_sources()
try:
# NOTE(fmontei): `validate` is False because documents have already
# been pre-validated during ingestion. Documents are post-validated
# below, regardless.
document_layering = layering.DocumentLayering(documents,
substitution_sources,
validate=False)
rendered_documents = document_layering.render()
except (errors.InvalidDocumentLayer, errors.InvalidDocumentParent,
errors.IndeterminateDocumentParent, errors.MissingDocumentKey,
errors.UnsupportedActionMethod) as e:
raise falcon.HTTPBadRequest(description=e.format_message())
except (errors.LayeringPolicyNotFound,
errors.SubstitutionSourceNotFound) as e:
raise falcon.HTTPConflict(description=e.format_message())
except errors.errors.UnknownSubstitutionError as e:
raise falcon.HTTPInternalServerError(
description=e.format_message())
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered. Note that `layering` module only
# returns concrete documents, so no filtering for that is needed here.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
user_filters = sanitized_params.copy()
rendered_documents = [
d for d in rendered_documents
if utils.deepfilter(d, **user_filters)
]
if sort_by:
rendered_documents = utils.multisort(rendered_documents, sort_by,
order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def document_get_all(session=None,
raw_dict=False,
revision_id=None,
**filters):
"""Retrieve all documents for ``revision_id`` that match ``filters``.
:param session: Database session object.
:param raw_dict: Whether to retrieve the exact way the data is stored in
DB if ``True``, else the way users expect the data.
:param revision_id: The ID corresponding to the ``Revision`` object. If the
it is "latest", then retrieve the latest revision, if one exists.
:param filters: Dictionary attributes (including nested) used to filter
out revision documents.
:returns: Dictionary representation of each retrieved document.
"""
session = session or get_session()
if revision_id == 'latest':
revision = session.query(models.Revision)\
.order_by(models.Revision.created_at.desc())\
.first()
if revision:
filters['revision_id'] = revision.id
elif revision_id:
filters['revision_id'] = revision_id
# TODO(fmontei): Currently Deckhand doesn't support filtering by nested
# JSON fields via sqlalchemy. For now, filter the documents using all
# "regular" filters via sqlalchemy and all nested filters via Python.
nested_filters = {}
for f in filters.copy():
if any([x in f for x in ('.', 'schema')]):
nested_filters.setdefault(f, filters.pop(f))
# Retrieve the most recently created documents for the revision, because
# documents with the same metadata.name and schema can exist across
# different revisions.
documents = session.query(models.Document)\
.filter_by(**filters)\
.order_by(models.Document.created_at.desc())\
.all()
final_documents = []
for doc in documents:
d = doc.to_dict(raw_dict=raw_dict)
if utils.deepfilter(d, **nested_filters):
final_documents.append(d)
return final_documents
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = {'metadata.storagePolicy': ['cleartext'], 'deleted': False}
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
documents = self._retrieve_documents_for_rendering(
revision_id, **filters)
substitution_sources = self._retrieve_substitution_sources()
try:
document_layering = layering.DocumentLayering(
documents, substitution_sources)
rendered_documents = document_layering.render()
except (errors.IndeterminateDocumentParent,
errors.UnsupportedActionMethod,
errors.MissingDocumentKey) as e:
raise falcon.HTTPBadRequest(description=e.format_message())
except (errors.LayeringPolicyNotFound,
errors.SubstitutionDependencyNotFound) as e:
raise falcon.HTTPConflict(description=e.format_message())
# Filters to be applied post-rendering, because many documents are
# involved in rendering. User filters can only be applied once all
# documents have been rendered.
order_by = sanitized_params.pop('order', None)
sort_by = sanitized_params.pop('sort', None)
user_filters = sanitized_params.copy()
user_filters['metadata.layeringDefinition.abstract'] = False
rendered_documents = [
d for d in rendered_documents
if utils.deepfilter(d, **user_filters)
]
if sort_by:
rendered_documents = utils.multisort(rendered_documents, sort_by,
order_by)
resp.status = falcon.HTTP_200
resp.body = self.view_builder.list(rendered_documents)
self._post_validate(rendered_documents)
def revision_get_all(session=None, **filters):
"""Return list of all revisions.
:param session: Database session object.
:returns: List of dictionary representations of retrieved revisions.
"""
session = session or get_session()
revisions = session.query(models.Revision)\
.all()
result = []
for revision in revisions:
revision_dict = revision.to_dict()
if utils.deepfilter(revision_dict, **filters):
revision_dict['documents'] = _update_revision_history(
revision_dict['documents'])
result.append(revision_dict)
return result
