def test_paginate_query_no_pagination(self):
sqlalchemy.asc("user").AndReturn("asc")
self.query.order_by("asc").AndReturn(self.query)
sqlalchemy.desc("project").AndReturn("desc")
self.query.order_by("desc").AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5, ["user_id", "project_id"], sort_dirs=["asc", "desc"])
def test_paginate_query_no_pagination(self):
sqlalchemy.asc('user').AndReturn('asc')
self.query.order_by('asc').AndReturn(self.query)
sqlalchemy.desc('project').AndReturn('desc')
self.query.order_by('desc').AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5,
['user_id', 'project_id'],
sort_dirs=['asc', 'desc'])
def test_paginate_query_no_pagination(self):
sqlalchemy.asc(self.model.user_id).AndReturn('asc')
self.query.order_by('asc').AndReturn(self.query)
sqlalchemy.desc(self.model.project_id).AndReturn('desc')
self.query.order_by('desc').AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5,
['user_id', 'project_id'],
sort_dirs=['asc', 'desc'])
def test_paginate_query_no_pagination_no_sort_dirs(self):
sqlalchemy.asc("user").AndReturn("asc_3")
self.query.order_by("asc_3").AndReturn(self.query)
sqlalchemy.asc("project").AndReturn("asc_2")
self.query.order_by("asc_2").AndReturn(self.query)
sqlalchemy.asc("snapshot").AndReturn("asc_1")
self.query.order_by("asc_1").AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5, ["user_id", "project_id", "snapshot_id"])
def test_paginate_query_no_pagination_no_sort_dirs(self):
sqlalchemy.asc('user').AndReturn('asc_3')
self.query.order_by('asc_3').AndReturn(self.query)
sqlalchemy.asc('project').AndReturn('asc_2')
self.query.order_by('asc_2').AndReturn(self.query)
sqlalchemy.asc('snapshot').AndReturn('asc_1')
self.query.order_by('asc_1').AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5,
['user_id', 'project_id', 'snapshot_id'])
def test_paginate_query_no_pagination_no_sort_dirs(self):
sqlalchemy.asc('user').AndReturn('asc_3')
self.query.order_by('asc_3').AndReturn(self.query)
sqlalchemy.asc('project').AndReturn('asc_2')
self.query.order_by('asc_2').AndReturn(self.query)
sqlalchemy.asc('snapshot').AndReturn('asc_1')
self.query.order_by('asc_1').AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5,
['user_id', 'project_id', 'snapshot_id'])
def test_paginate_query(self):
sqlalchemy.asc("user").AndReturn("asc_1")
self.query.order_by("asc_1").AndReturn(self.query)
sqlalchemy.desc("project").AndReturn("desc_1")
self.query.order_by("desc_1").AndReturn(self.query)
self.mox.StubOutWithMock(sqlalchemy.sql, "and_")
sqlalchemy.sql.and_(False).AndReturn("some_crit")
sqlalchemy.sql.and_(True, False).AndReturn("another_crit")
self.mox.StubOutWithMock(sqlalchemy.sql, "or_")
sqlalchemy.sql.or_("some_crit", "another_crit").AndReturn("some_f")
self.query.filter("some_f").AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(
self.query, self.model, 5, ["user_id", "project_id"], marker=self.marker, sort_dirs=["asc", "desc"]
)
def _events_paginate_query(context,
query,
model,
limit=None,
sort_keys=None,
marker=None,
sort_dir=None):
default_sort_keys = ['created_at']
if not sort_keys:
sort_keys = default_sort_keys
if not sort_dir:
sort_dir = 'desc'
# This assures the order of the stacks will always be the same
# even for sort_key values that are not unique in the database
sort_keys = sort_keys + ['id']
model_marker = None
if marker:
# not to use model_query(context, model).get(marker), because
# user can only see the ID(column 'uuid') and the ID as the marker
model_marker = model_query(context,
model).filter_by(uuid=marker).first()
try:
query = utils.paginate_query(query, model, limit, sort_keys,
model_marker, sort_dir)
except utils.InvalidSortKey as exc:
raise exception.Invalid(reason=exc.message)
return query
def get_all(context, namespace_name, session, filters=None, marker=None,
limit=None, sort_key='created_at', sort_dir='desc'):
"""Get all tags that match zero or more filters.
:param filters: dict of filter keys and values.
:param marker: tag id after which to start page
:param limit: maximum number of namespaces to return
:param sort_key: namespace attribute by which results should be sorted
:param sort_dir: direction in which results should be sorted (asc, desc)
"""
namespace = namespace_api.get(context, namespace_name, session)
query = (session.query(models.MetadefTag).filter_by(
namespace_id=namespace['id']))
marker_tag = None
if marker is not None:
marker_tag = _get(context, marker, session)
sort_keys = ['created_at', 'id']
sort_keys.insert(0, sort_key) if sort_key not in sort_keys else sort_keys
query = paginate_query(query=query,
model=models.MetadefTag,
limit=limit,
sort_keys=sort_keys,
marker=marker_tag, sort_dir=sort_dir)
metadef_tag = query.all()
metadef_tag_list = []
for tag in metadef_tag:
metadef_tag_list.append(tag.to_dict())
return metadef_tag_list
def _events_paginate_query(context, query, model, limit=None, sort_keys=None,
marker=None, sort_dir=None):
default_sort_keys = ['created_at']
if not sort_keys:
sort_keys = default_sort_keys
if not sort_dir:
sort_dir = 'desc'
# This assures the order of the stacks will always be the same
# even for sort_key values that are not unique in the database
sort_keys = sort_keys + ['id']
model_marker = None
if marker:
# not to use model_query(context, model).get(marker), because
# user can only see the ID(column 'uuid') and the ID as the marker
model_marker = model_query(context, model).filter_by(uuid=marker).\
first()
try:
query = utils.paginate_query(query, model, limit, sort_keys,
model_marker, sort_dir)
except utils.InvalidSortKey as exc:
raise exception.Invalid(reason=exc.message)
return query
def _paginate_query(context, query, model, limit=None, sort_keys=None,
marker=None, sort_dir=None, offset=None):
default_sort_keys = ['created_at']
if not sort_keys:
sort_keys = default_sort_keys
if not sort_dir:
sort_dir = 'desc'
# This assures the order of the stacks will always be the same
# even for sort_key values that are not unique in the database
sort_keys = sort_keys + ['id']
model_marker = None
if marker:
model_marker = model_query(context, model).get(marker)
try:
query = utils.paginate_query(query, model, None, sort_keys,
model_marker, sort_dir)
if offset is not None:
query=query.offset(offset)
except utils.InvalidSortKey as exc:
raise exception.Invalid(reason=exc.message)
if limit is not None:
return query.limit(limit)
else:
return query
def _paginate_query(context,
query,
model,
limit=None,
sort_keys=None,
marker=None,
sort_dir=None):
default_sort_keys = ['created_at']
if not sort_keys:
sort_keys = default_sort_keys
if not sort_dir:
sort_dir = 'desc'
# This assures the order of the stacks will always be the same
# even for sort_key values that are not unique in the database
sort_keys = sort_keys + ['id']
model_marker = None
if marker:
model_marker = model_query(context, model).get(marker)
try:
query = utils.paginate_query(query, model, limit, sort_keys,
model_marker, sort_dir)
except utils.InvalidSortKey as exc:
raise exception.Invalid(reason=exc.message)
return query
def test_paginate_query(self):
sqlalchemy.asc('user').AndReturn('asc_1')
self.query.order_by('asc_1').AndReturn(self.query)
sqlalchemy.desc('project').AndReturn('desc_1')
self.query.order_by('desc_1').AndReturn(self.query)
self.mox.StubOutWithMock(sqlalchemy.sql, 'and_')
sqlalchemy.sql.and_(False).AndReturn('some_crit')
sqlalchemy.sql.and_(True, False).AndReturn('another_crit')
self.mox.StubOutWithMock(sqlalchemy.sql, 'or_')
sqlalchemy.sql.or_('some_crit', 'another_crit').AndReturn('some_f')
self.query.filter('some_f').AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5,
['user_id', 'project_id'],
marker=self.marker,
sort_dirs=['asc', 'desc'])
def test_paginate_query(self):
sqlalchemy.asc(self.model.user_id).AndReturn('asc_1')
self.query.order_by('asc_1').AndReturn(self.query)
sqlalchemy.desc(self.model.project_id).AndReturn('desc_1')
self.query.order_by('desc_1').AndReturn(self.query)
self.mox.StubOutWithMock(sqlalchemy.sql, 'and_')
sqlalchemy.sql.and_(mock.ANY).AndReturn('some_crit')
sqlalchemy.sql.and_(mock.ANY, mock.ANY).AndReturn('another_crit')
self.mox.StubOutWithMock(sqlalchemy.sql, 'or_')
sqlalchemy.sql.or_('some_crit', 'another_crit').AndReturn('some_f')
self.query.filter('some_f').AndReturn(self.query)
self.query.limit(5).AndReturn(self.query)
self.mox.ReplayAll()
utils.paginate_query(self.query, self.model, 5,
['user_id', 'project_id'],
marker=self.marker,
sort_dirs=['asc', 'desc'])
def _paginate_query(model, limit=None, marker=None, sort_keys=None,
sort_dir=None, query=None):
if type(sort_keys) is str:
sort_keys = [sort_keys]
if not query:
query = model_query(model)
query = db_utils.paginate_query(query, model, limit, sort_keys,
marker=marker, sort_dir=sort_dir)
return query.all()
def _paginate_query(model, limit=None, marker=None, sort_key=None,
sort_dir=None, query=None):
if not query:
query = model_query(model)
sort_keys = ['id']
if sort_key and sort_key not in sort_keys:
sort_keys.insert(0, sort_key)
query = db_utils.paginate_query(query, model, limit, sort_keys,
marker=marker, sort_dir=sort_dir)
return query.all()
def _paginate_query(model, limit=None, marker=None, sort_key=None,
sort_dir=None, query=None):
if not query:
query = model_query(model)
sort_keys = ['id']
if sort_key and sort_key not in sort_keys:
sort_keys.insert(0, sort_key)
query = db_utils.paginate_query(query, model, limit, sort_keys,
marker=marker, sort_dir=sort_dir)
return query.all()
def _apply_pagination(self, model_cls, query, marker=None, limit=None):
marker_entity = None
if marker:
marker_entity = api_base.entity_get(model_cls, marker, True)
return utils.paginate_query(query=query,
model=model_cls,
limit=limit,
sort_keys=["id"],
marker=marker_entity)
def _find(self, model, context, criterion, one=False,
marker=None, limit=None, sort_key=None, sort_dir=None):
"""
Base "finder" method
Used to abstract these details from all the _find_*() methods.
"""
# First up, create a query and apply the various filters
query = self.session.query(model)
query = self._apply_criterion(model, query, criterion)
query = self._apply_tenant_criteria(context, model, query)
query = self._apply_deleted_criteria(context, model, query)
if one:
# If we're asked to return exactly one record, but multiple or
# none match, raise a NotFound
try:
return query.one()
except (exc.NoResultFound, exc.MultipleResultsFound):
raise exceptions.NotFound()
else:
# If marker is not none and basestring we query it.
# Otherwise, return all matching records
if marker is not None:
try:
marker = self._find(model, context, {'id': marker},
one=True)
except exceptions.NotFound:
raise exceptions.MarkerNotFound(
'Marker %s could not be found' % marker)
# Malformed UUIDs return StatementError
except sqlalchemy_exc.StatementError as statement_error:
raise exceptions.InvalidMarker(statement_error.message)
sort_key = sort_key or 'created_at'
sort_dir = sort_dir or 'asc'
try:
query = oslo_utils.paginate_query(
query, model, limit,
[sort_key, 'id', 'created_at'], marker=marker,
sort_dir=sort_dir)
return query.all()
except oslo_utils.InvalidSortKey as sort_key_error:
raise exceptions.InvalidSortKey(sort_key_error.message)
# Any ValueErrors are propagated back to the user as is.
# Limits, sort_dir and sort_key are checked at the API layer.
# If however central or storage is called directly, invalid values
# show up as ValueError
except ValueError as value_error:
raise exceptions.ValueError(value_error.message)
def _get_all(context,
session,
filters=None,
marker=None,
limit=None,
sort_key='created_at',
sort_dir='desc'):
"""Get all namespaces that match zero or more filters.
:param filters: dict of filter keys and values.
:param marker: namespace id after which to start page
:param limit: maximum number of namespaces to return
:param sort_key: namespace attribute by which results should be sorted
:param sort_dir: direction in which results should be sorted (asc, desc)
"""
filters = filters or {}
query = _select_namespaces_query(context, session)
# if visibility filter, apply it to the context based query
visibility = filters.pop('visibility', None)
if visibility is not None:
query = query.filter(models.MetadefNamespace.visibility == visibility)
# if id_list filter, apply it to the context based query
id_list = filters.pop('id_list', None)
if id_list is not None:
query = query.filter(models.MetadefNamespace.id.in_(id_list))
marker_namespace = None
if marker is not None:
marker_namespace = _get(context, marker, session)
sort_keys = ['created_at', 'id']
sort_keys.insert(0, sort_key) if sort_key not in sort_keys else sort_keys
query = paginate_query(query=query,
model=models.MetadefNamespace,
limit=limit,
sort_keys=sort_keys,
marker=marker_namespace,
sort_dir=sort_dir)
return query.all()
def _paginate_query(model,
limit=None,
marker=None,
sort_keys=None,
sort_dir=None,
query=None):
if type(sort_keys) is str:
sort_keys = [sort_keys]
if not query:
query = model_query(model)
query = db_utils.paginate_query(query,
model,
limit,
sort_keys,
marker=marker,
sort_dir=sort_dir)
return query.all()
def get_all(context,
namespace_name,
session,
filters=None,
marker=None,
limit=None,
sort_key='created_at',
sort_dir='desc'):
"""Get all tags that match zero or more filters.
:param filters: dict of filter keys and values.
:param marker: tag id after which to start page
:param limit: maximum number of namespaces to return
:param sort_key: namespace attribute by which results should be sorted
:param sort_dir: direction in which results should be sorted (asc, desc)
"""
namespace = namespace_api.get(context, namespace_name, session)
query = (session.query(
models.MetadefTag).filter_by(namespace_id=namespace['id']))
marker_tag = None
if marker is not None:
marker_tag = _get(context, marker, session)
sort_keys = ['created_at', 'id']
sort_keys.insert(0, sort_key) if sort_key not in sort_keys else sort_keys
query = paginate_query(query=query,
model=models.MetadefTag,
limit=limit,
sort_keys=sort_keys,
marker=marker_tag,
sort_dir=sort_dir)
metadef_tag = query.all()
metadef_tag_list = []
for tag in metadef_tag:
metadef_tag_list.append(tag.to_dict())
return metadef_tag_list
def _get_all(context, session, filters=None, marker=None,
limit=None, sort_key='created_at', sort_dir='desc'):
"""Get all namespaces that match zero or more filters.
:param filters: dict of filter keys and values.
:param marker: namespace id after which to start page
:param limit: maximum number of namespaces to return
:param sort_key: namespace attribute by which results should be sorted
:param sort_dir: direction in which results should be sorted (asc, desc)
"""
filters = filters or {}
query = _select_namespaces_query(context, session)
# if visibility filter, apply it to the context based query
visibility = filters.pop('visibility', None)
if visibility is not None:
query = query.filter(models.MetadefNamespace.visibility == visibility)
# if id_list filter, apply it to the context based query
id_list = filters.pop('id_list', None)
if id_list is not None:
query = query.filter(models.MetadefNamespace.id.in_(id_list))
marker_namespace = None
if marker is not None:
marker_namespace = _get(context, marker, session)
sort_keys = ['created_at', 'id']
sort_keys.insert(0, sort_key) if sort_key not in sort_keys else sort_keys
query = paginate_query(query=query,
model=models.MetadefNamespace,
limit=limit,
sort_keys=sort_keys,
marker=marker_namespace, sort_dir=sort_dir)
return query.all()
def entity_get_all(kls, filter_non_public=False, marker=None, limit=None,
sort_field='id', sort_dir='asc', **kwargs):
# Sanity checks, in case someone accidentally explicitly passes in 'None'
if not sort_field:
sort_field = 'id'
if not sort_dir:
sort_dir = 'asc'
# Construct the query
query = model_query(kls)
# Sanity check on input parameters
query = apply_query_filters(query=query, model=kls, **kwargs)
# Construct the query
try:
query = paginate_query(query=query,
model=kls,
limit=limit,
sort_keys=[sort_field],
marker=marker,
sort_dir=sort_dir)
except InvalidSortKey:
raise ClientSideError("Invalid sort_field [%s]" % (sort_field,),
status_code=400)
except ValueError as ve:
raise ClientSideError("%s" % (ve,), status_code=400)
# Execute the query
entities = query.all()
if len(entities) > 0 and filter_non_public:
sample_entity = entities[0] if len(entities) > 0 else None
public_fields = getattr(sample_entity, "_public_fields", [])
entities = [_filter_non_public_fields(entity, public_fields)
for entity in entities]
return entities
def package_search(filters, context, limit=None):
"""Search packages with different filters
* Admin is allowed to browse all the packages
* Regular user is allowed to browse all packages belongs to user tenant
and all other packages marked is_public.
Also all packages should be enabled.
* Use marker (inside filters param) and limit for pagination:
The typical pattern of limit and marker is to make an initial limited
request and then to use the ID of the last package from the response
as the marker parameter in a subsequent limited request.
"""
session = db_session.get_session()
pkg = models.Package
# If the packages search specifies the inclusion of disabled packages,
# we handle this differently for admins vs non-admins:
# For admins: *don't* require pkg.enabled == True (no changes to query)
# For non-admins: add an OR-condition to filter for packages that are owned
# by the tenant in the current context
# Otherwise: in all other cases, we return only enabled packages
if filters.get('include_disabled', '').lower() == 'true':
include_disabled = True
else:
include_disabled = False
if context.is_admin:
if not include_disabled:
#NOTE(efedorova): is needed for SA 0.7.9, but could be done
# simpler in SA 0.8. See http://goo.gl/9stlKu for a details
query = session.query(pkg).filter(pkg.__table__.c.enabled)
else:
query = session.query(pkg)
elif filters.get('owned', '').lower() == 'true':
if not include_disabled:
query = session.query(pkg).filter((pkg.owner_id == context.tenant)
& pkg.enabled)
else:
query = session.query(pkg).filter(pkg.owner_id == context.tenant)
else:
if not include_disabled:
query = session.query(pkg).filter(
or_((pkg.is_public & pkg.enabled),
((pkg.owner_id == context.tenant) & pkg.enabled)))
else:
query = session.query(pkg).filter(
or_((pkg.is_public & pkg.enabled),
pkg.owner_id == context.tenant))
if 'type' in filters.keys():
query = query.filter(pkg.type == filters['type'].title())
if 'category' in filters.keys():
query = query.filter(
pkg.categories.any(models.Category.name.in_(filters['category'])))
if 'tag' in filters.keys():
query = query.filter(pkg.tags.any(models.Tag.name.in_(filters['tag'])))
if 'class_name' in filters.keys():
query = query.filter(
pkg.class_definitions.any(
models.Class.name == filters['class_name']))
if 'fqn' in filters.keys():
query = query.filter(pkg.fully_qualified_name == filters['fqn'])
if 'search' in filters.keys():
fk_fields = {
'categories': 'Category',
'tags': 'Tag',
'class_definitions': 'Class'
}
conditions = []
for attr in dir(pkg):
if attr.startswith('_'):
continue
if isinstance(getattr(pkg, attr),
attributes.InstrumentedAttribute):
search_str = filters['search']
for delim in ',;':
search_str = search_str.replace(delim, ' ')
for key_word in search_str.split():
_word = '%{value}%'.format(value=key_word)
if attr in fk_fields.keys():
condition = getattr(pkg, attr).any(
getattr(models, fk_fields[attr]).name.like(_word))
conditions.append(condition)
else:
conditions.append(getattr(pkg, attr).like(_word))
query = query.filter(or_(*conditions))
sort_keys = [
SEARCH_MAPPING[sort_key]
for sort_key in filters.get('order_by', []) or ['created']
]
marker = filters.get('marker')
sort_dir = filters.get('sort_dir')
if marker is not None: # set marker to real object instead of its id
marker = _package_get(marker, session)
query = utils.paginate_query(query, pkg, limit, sort_keys, marker,
sort_dir)
return query.all()
def package_search(filters, context, limit=None):
"""Search packages with different filters
* Admin is allowed to browse all the packages
* Regular user is allowed to browse all packages belongs to user tenant
and all other packages marked is_public.
Also all packages should be enabled.
* Use marker (inside filters param) and limit for pagination:
The typical pattern of limit and marker is to make an initial limited
request and then to use the ID of the last package from the response
as the marker parameter in a subsequent limited request.
"""
session = db_session.get_session()
pkg = models.Package
# If the packages search specifies the inclusion of disabled packages,
# we handle this differently for admins vs non-admins:
# For admins: *don't* require pkg.enabled == True (no changes to query)
# For non-admins: add an OR-condition to filter for packages that are owned
# by the tenant in the current context
# Otherwise: in all other cases, we return only enabled packages
if filters.get('include_disabled', '').lower() == 'true':
include_disabled = True
else:
include_disabled = False
if context.is_admin:
if not include_disabled:
#NOTE(efedorova): is needed for SA 0.7.9, but could be done
# simpler in SA 0.8. See http://goo.gl/9stlKu for a details
query = session.query(pkg).filter(pkg.__table__.c.enabled)
else:
query = session.query(pkg)
elif filters.get('owned', '').lower() == 'true':
if not include_disabled:
query = session.query(pkg).filter(
(pkg.owner_id == context.tenant) & pkg.enabled
)
else:
query = session.query(pkg).filter(pkg.owner_id == context.tenant)
else:
if not include_disabled:
query = session.query(pkg).filter(
or_((pkg.is_public & pkg.enabled),
((pkg.owner_id == context.tenant) & pkg.enabled))
)
else:
query = session.query(pkg).filter(
or_((pkg.is_public & pkg.enabled),
pkg.owner_id == context.tenant)
)
if 'type' in filters.keys():
query = query.filter(pkg.type == filters['type'].title())
if 'category' in filters.keys():
query = query.filter(pkg.categories.any(
models.Category.name.in_(filters['category'])))
if 'tag' in filters.keys():
query = query.filter(pkg.tags.any(
models.Tag.name.in_(filters['tag'])))
if 'class_name' in filters.keys():
query = query.filter(pkg.class_definitions.any(
models.Class.name == filters['class_name']))
if 'fqn' in filters.keys():
query = query.filter(pkg.fully_qualified_name == filters['fqn'])
if 'search' in filters.keys():
fk_fields = {'categories': 'Category',
'tags': 'Tag',
'class_definitions': 'Class'}
conditions = []
for attr in dir(pkg):
if attr.startswith('_'):
continue
if isinstance(getattr(pkg, attr),
attributes.InstrumentedAttribute):
search_str = filters['search']
for delim in ',;':
search_str = search_str.replace(delim, ' ')
for key_word in search_str.split():
_word = '%{value}%'.format(value=key_word)
if attr in fk_fields.keys():
condition = getattr(pkg, attr).any(
getattr(models, fk_fields[attr]).name.like(_word))
conditions.append(condition)
else:
conditions.append(getattr(pkg, attr).like(_word))
query = query.filter(or_(*conditions))
sort_keys = [SEARCH_MAPPING[sort_key] for sort_key in
filters.get('order_by', []) or ['created']]
marker = filters.get('marker')
sort_dir = filters.get('sort_dir')
if marker is not None: # set marker to real object instead of its id
marker = _package_get(marker, session)
query = utils.paginate_query(
query, pkg, limit, sort_keys, marker, sort_dir)
return query.all()
