def test_split_expression(self):
sq = SQ(foo='bar')
self.assertEqual(sq.split_expression('foo'), ('foo', 'exact'))
self.assertEqual(sq.split_expression('foo__exact'), ('foo', 'exact'))
self.assertEqual(sq.split_expression('foo__lt'), ('foo', 'lt'))
self.assertEqual(sq.split_expression('foo__lte'), ('foo', 'lte'))
self.assertEqual(sq.split_expression('foo__gt'), ('foo', 'gt'))
self.assertEqual(sq.split_expression('foo__gte'), ('foo', 'gte'))
self.assertEqual(sq.split_expression('foo__in'), ('foo', 'in'))
self.assertEqual(sq.split_expression('foo__startswith'), ('foo', 'startswith'))
# Unrecognized filter. Fall back to exact.
self.assertEqual(sq.split_expression('foo__moof'), ('foo', 'exact'))
def autocomplete(self, **kwargs):
"""
A shortcut method to perform an autocomplete search.
Must be run against fields that are either ``NgramField`` or
``EdgeNgramField``.
"""
clone = self._clone()
query_bits = []
for field_name, query in kwargs.items():
for word in query.split(' '):
bit = clone.query.clean(word.strip())
if bit:
kwargs = {
field_name: bit,
}
query_bits.append(SQ(**kwargs))
return clone.filter(six.moves.reduce(operator.__and__, query_bits))
def test_log_query(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Stow.
self.old_unified_index = connections['default']._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi])
connections['default']._index = self.ui
# Update the "index".
backend = connections['default'].get_backend()
backend.clear()
self.bmmsi.update()
with self.settings(DEBUG=False):
msq = connections['default'].get_query()
self.assertEqual(len(msq.get_results()), 23)
self.assertEqual(len(connections['default'].queries), 0)
with self.settings(DEBUG=True):
# Redefine it to clear out the cached results.
msq2 = connections['default'].get_query()
self.assertEqual(len(msq2.get_results()), 23)
self.assertEqual(len(connections['default'].queries), 1)
self.assertEqual(connections['default'].queries[0]['query_string'],
'')
msq3 = connections['default'].get_query()
msq3.add_filter(SQ(foo='bar'))
len(msq3.get_results())
self.assertEqual(len(connections['default'].queries), 2)
self.assertEqual(connections['default'].queries[0]['query_string'],
'')
self.assertEqual(connections['default'].queries[1]['query_string'],
'')
# Restore.
connections['default']._index = self.old_unified_index
def _member_sqs(self, sqs, user, **kwargs):
"""
Filter the query set for members.
(status AND status_detail AND
(anon_view OR user_view OR member_view))
OR
(users_can_view__in user.pk)
OR
(groups_can_view__in user's groups)
user is a required argument since we'll be filtering by user.pk.
"""
groups = [g.pk for g in user.group_set.all()]
status_detail = kwargs.get('status_detail', 'active')
status = kwargs.get('status', True)
anon_q = SQ(allow_anonymous_view=True)
user_q = SQ(allow_user_view=True)
member_q = SQ(allow_member_view=True)
status_q = SQ(status=status, status_detail=status_detail)
user_perm_q = SQ(users_can_view__in=[user.pk])
group_perm_q = SQ(groups_can_view__in=groups)
creator_q = SQ(creator_username=user.username)
owner_q = SQ(owner_username=user.username)
if groups:
sqs = sqs.filter((status_q
& (anon_q | user_q | creator_q | owner_q))
| (user_perm_q | group_perm_q))
else:
sqs = sqs.filter((status_q
& (anon_q | user_q | creator_q | owner_q))
| (user_perm_q))
return sqs
def narrow_sqs(self, sqs):
"""
TODO: Currently this is an AND conjunction. It should vary depending
on the value of self.select_many.
"""
if self.select_many:
sq = None
for value in self.get_applicable_values():
q = SQ(**{self.field_name: sqs.query.clean(value.value)})
if sq:
sq = sq | q
else:
sq = q
if sq:
sqs = sqs.narrow(sq)
else:
for value in self.get_applicable_values():
sqs = sqs.narrow(
u'%s:"%s"' % (self.field_name, sqs.query.clean(value.value))
)
return sqs
def test_simple_nesting(self):
sq1 = SQ(foo='bar')
sq2 = SQ(foo='bar')
bigger_sq = SQ(sq1 & sq2)
self.assertEqual(repr(bigger_sq),
'<SQ: AND (foo__content=bar AND foo__content=bar)>')
another_bigger_sq = SQ(sq1 | sq2)
self.assertEqual(repr(another_bigger_sq),
'<SQ: AND (foo__content=bar OR foo__content=bar)>')
one_more_bigger_sq = SQ(sq1 & ~sq2)
self.assertEqual(
repr(one_more_bigger_sq),
'<SQ: AND (foo__content=bar AND NOT (foo__content=bar))>')
mega_sq = SQ(bigger_sq & SQ(another_bigger_sq | ~one_more_bigger_sq))
self.assertEqual(
repr(mega_sq),
'<SQ: AND ((foo__content=bar AND foo__content=bar) AND ((foo__content=bar OR foo__content=bar) OR NOT ((foo__content=bar AND NOT (foo__content=bar)))))>'
)
def _user_sqs(self, sqs, user, **kwargs):
"""
Filter the query set for people between admin and anon permission
(status AND status_detail AND ((anon_view OR user_view)
OR
(users_can_view__in user.pk)
OR
(groups_can_view__in user's groups)
user required since we'll filter by user.pk.
"""
groups = [g.pk for g in user.group_set.all()]
status_detail = kwargs.get('status_detail', 'active')
status = kwargs.get('status', True)
anon_q = SQ(allow_anonymous_view=True)
user_q = SQ(allow_user_view=True)
status_q = SQ(status=status, status_detail=status_detail)
user_perm_q = SQ(users_can_view__in=[user.pk])
group_perm_q = SQ(groups_can_view__in=groups)
creator_q = SQ(creator_id=user.id)
owner_q = SQ(owner_id=user.id)
if groups:
sqs = sqs.filter(
(status_q & (anon_q | user_q | creator_q | owner_q)) |
(user_perm_q | group_perm_q))
else:
sqs = sqs.filter(
(status_q & (anon_q | user_q | creator_q | owner_q)) |
(user_perm_q)
)
return sqs
def test_add_filter(self):
self.assertEqual(len(self.bsq.query_filter), 0)
self.bsq.add_filter(SQ(foo='bar'))
self.assertEqual(len(self.bsq.query_filter), 1)
self.bsq.add_filter(SQ(foo__lt='10'))
self.bsq.add_filter(~SQ(claris='moof'))
self.bsq.add_filter(SQ(claris='moof'), use_or=True)
self.assertEqual(repr(self.bsq.query_filter), '<SQ: OR ((foo__contains=bar AND foo__lt=10 AND NOT (claris__contains=moof)) OR claris__contains=moof)>')
self.bsq.add_filter(SQ(claris='moof'))
self.assertEqual(repr(self.bsq.query_filter), '<SQ: AND (((foo__contains=bar AND foo__lt=10 AND NOT (claris__contains=moof)) OR claris__contains=moof) AND claris__contains=moof)>')
self.bsq.add_filter(SQ(claris='wtf mate'))
self.assertEqual(repr(self.bsq.query_filter), '<SQ: AND (((foo__contains=bar AND foo__lt=10 AND NOT (claris__contains=moof)) OR claris__contains=moof) AND claris__contains=moof AND claris__contains=wtf mate)>')
def filter_and(self, *args, **kwargs):
"""Narrows the search by looking for (and including) certain attributes."""
clone = self._clone()
clone.query.add_filter(SQ(*args, **kwargs))
return clone
def exclude(self, *args, **kwargs):
"""Narrows the search by ensuring certain attributes are not included."""
clone = self._clone()
clone.query.add_filter(~SQ(*args, **kwargs))
return clone
def test_build_query(self):
self.bsq.add_filter(SQ(foo='bar'))
self.assertRaises(NotImplementedError, self.bsq.build_query)
def test_get_count(self):
self.bsq.add_filter(SQ(foo='bar'))
self.assertRaises(NotImplementedError, self.bsq.get_count)
def test_repr(self):
self.assertEqual(repr(SQ(foo='bar')), '<SQ: AND foo__contains=bar>')
self.assertEqual(repr(SQ(foo=1)), '<SQ: AND foo__contains=1>')
self.assertEqual(repr(SQ(foo=datetime.datetime(2009, 5, 12, 23, 17))),
'<SQ: AND foo__contains=2009-05-12 23:17:00>')
def test_split_expression(self):
sq = SQ(foo='bar')
self.assertEqual(sq.split_expression('foo'), ('foo', 'contains'))
self.assertEqual(sq.split_expression('foo__exact'), ('foo', 'exact'))
self.assertEqual(sq.split_expression('foo__contains'),
('foo', 'contains'))
self.assertEqual(sq.split_expression('foo__lt'), ('foo', 'lt'))
self.assertEqual(sq.split_expression('foo__lte'), ('foo', 'lte'))
self.assertEqual(sq.split_expression('foo__gt'), ('foo', 'gt'))
self.assertEqual(sq.split_expression('foo__gte'), ('foo', 'gte'))
self.assertEqual(sq.split_expression('foo__in'), ('foo', 'in'))
self.assertEqual(sq.split_expression('foo__startswith'),
('foo', 'startswith'))
self.assertEqual(sq.split_expression('foo__range'), ('foo', 'range'))
self.assertEqual(sq.split_expression('foo__fuzzy'), ('foo', 'fuzzy'))
# Unrecognized filter. Fall back to exact.
self.assertEqual(sq.split_expression('foo__moof'), ('foo', 'contains'))
def filter_searchqueryset_for_read_access(sqs, user):
"""
Given a SearchQuerySet, this function adds a filter that limits the
result set to only include elements with read access.
"""
public_node = (
SQ(public__exact=True) | # public on the object itself (applicable for groups)
SQ(mt_visibility__exact=BaseTagObject.VISIBILITY_ALL) | # public via "everyone can see me" visibility meta attribute
SQ(always_visible__exact=True) # special marker for indexed objects that should always show up in search
)
if user.is_authenticated:
if check_user_superuser(user):
pass
else:
logged_in_user_visibility = (
SQ(user_visibility_mode__exact=True) & # for UserProfile search index objects
SQ(mt_visibility__exact=BaseTagObject.VISIBILITY_GROUP) # logged in users can see users who are visible
)
my_item = (
SQ(creator__exact=user.id)
)
visible_for_all_authenticated_users = (
SQ(visible_for_all_authenticated_users=True)
)
# FIXME: known problem: ``group_members`` is a stale indexed representation of the members
# of an items group. New members of a group won't be able to find old indexed items if the index
# is not refreshed regularly
group_visible_and_in_my_group = (
SQ(mt_visibility__exact=BaseTagObject.VISIBILITY_GROUP) &
SQ(group_members__contains=user.id)
)
ored_query = public_node | group_visible_and_in_my_group | my_item \
| logged_in_user_visibility | visible_for_all_authenticated_users
users_group_ids = get_cosinnus_group_model().objects.get_for_user_pks(user)
if users_group_ids:
group_member_user_visibility = (
SQ(user_visibility_mode__exact=True) & # for UserProfile search index objects
SQ(mt_visibility__exact=BaseTagObject.VISIBILITY_USER) & # team mambers can see this user
SQ(membership_groups__in=users_group_ids)
)
ored_query = ored_query | group_member_user_visibility
sqs = sqs.filter_and(ored_query)
else:
sqs = sqs.filter_and(public_node)
return sqs
def test_split_expression(self):
sq = SQ(foo="bar")
self.assertEqual(sq.split_expression("foo"), ("foo", "content"))
self.assertEqual(sq.split_expression("foo__exact"), ("foo", "exact"))
self.assertEqual(sq.split_expression("foo__content"), ("foo", "content"))
self.assertEqual(sq.split_expression("foo__contains"), ("foo", "contains"))
self.assertEqual(sq.split_expression("foo__lt"), ("foo", "lt"))
self.assertEqual(sq.split_expression("foo__lte"), ("foo", "lte"))
self.assertEqual(sq.split_expression("foo__gt"), ("foo", "gt"))
self.assertEqual(sq.split_expression("foo__gte"), ("foo", "gte"))
self.assertEqual(sq.split_expression("foo__in"), ("foo", "in"))
self.assertEqual(sq.split_expression("foo__startswith"), ("foo", "startswith"))
self.assertEqual(sq.split_expression("foo__endswith"), ("foo", "endswith"))
self.assertEqual(sq.split_expression("foo__range"), ("foo", "range"))
self.assertEqual(sq.split_expression("foo__fuzzy"), ("foo", "fuzzy"))
# Unrecognized filter. Fall back to exact.
self.assertEqual(sq.split_expression("foo__moof"), ("foo", "content"))
def _user_or_member_sqs(self, sqs, user, **kwargs):
"""
Filter the query set for user or members.
Here we check the existence of the fields to be tested,
and handle the queries that need to be directly pulled from db.
(status AND status_detail AND
(anon_view OR user_view OR member_view))
OR
(users_can_view__in user.pk)
OR
(groups_can_view__in user's groups)
user is a required argument since we'll be filtering by user.pk.
"""
groups = [g.pk for g in user.group_set.all()]
status_detail = kwargs.get('status_detail', 'active')
status = kwargs.get('status', True)
is_member = kwargs.get('is_member', True)
# allow_xxx and creator and owner fields to be checked
fields_ors_sq = None
fields_ors_sq_list = []
model_fiels = [field.name for field in self.model._meta.fields]
if 'allow_anonymous_view' in model_fiels:
fields_ors_sq_list.append(SQ(allow_anonymous_view=True))
if 'allow_user_view' in model_fiels:
fields_ors_sq_list.append(SQ(allow_user_view=True))
if is_member:
if 'allow_member_view' in model_fiels:
fields_ors_sq_list.append(SQ(allow_member_view=True))
if 'creator_username' in model_fiels:
fields_ors_sq_list.append(SQ(creator_username=user.username))
if 'owner_username' in model_fiels:
fields_ors_sq_list.append(SQ(owner_username=user.username))
for field_sq in fields_ors_sq_list:
if fields_ors_sq:
fields_ors_sq = fields_ors_sq | field_sq
else:
fields_ors_sq = field_sq
# status and status_detail if exist
status_d = {}
if 'status' in model_fiels:
status_d.update({'status': status})
if 'status_detail' in model_fiels:
status_d.update({'status_detail': status_detail})
if status_d:
status_q = SQ(**status_d)
else:
status_q = None
direct_db = kwargs.get('direct_db', 0)
# object permissions
if not direct_db:
group_perm_q = SQ(groups_can_view__in=groups)
# user is not being recorded in the ObjectPermission
#user_perm_q = SQ(users_can_view__in=[user.pk])
else:
# to pull directly from db, get a list of object_ids instead of groups
from tendenci.apps.perms.object_perms import ObjectPermission
app_label = self.model._meta.app_label
model_name = self.model.__name__
content_type = ContentType.objects.filter(app_label=app_label,
model=model_name)[:1]
codename = 'view_%s' % model_name
group_allowed_object_ids = ObjectPermission.objects.filter(
content_type=content_type[0],
codename=codename,
group__in=groups).values_list('id', flat=True)
group_perm_q = SQ(id__in=group_allowed_object_ids)
# user is not being recorded in the ObjectPermission
# so, commenting it out for now
# user_allowed_object_ids = ObjectPermission.objects.filter(
# content_type=content_type[0],
# codename=codename,
# user__in=[user]).values_list('id', flat=True)
group_perm_q = SQ(id__in=group_allowed_object_ids)
filters = None
if status_q or fields_ors_sq:
if status_q and fields_ors_sq:
filters = status_q & fields_ors_sq
elif status_q:
filters = status_q
else:
filters = fields_ors_sq
if groups:
if not filters:
filters = group_perm_q
else:
filters = filters | group_perm_q
if filters:
sqs = sqs.filter(filters)
return sqs
def get_queryset(self):
query_params = self.request.query_params
query = SQ()
if query_params.get('pincode'):
query &= SQ(
pincode__startswith=query_params.get('pincode').lower())
if query_params.get('state'):
query &= SQ(state__startswith=query_params.get('state').lower())
if query_params.get('division_name'):
query &= SQ(division_name__startswith=query_params.get(
'division_name').lower())
if query_params.get('region_name'):
query &= SQ(region_name__startswith=query_params.get(
'region_name').lower())
if query_params.get('circle_name'):
query &= SQ(circle_name__startswith=query_params.get(
'circle_name').lower())
if query_params.get('taluk'):
query &= SQ(
taluk_name__startswith=query_params.get('taluk').lower())
if query_params.get('q'):
value = query_params.get('q').lower()
query = (SQ(pincode__startswith=value)
| SQ(state__startswith=value)
| SQ(division_name__startswith=value)
| SQ(region_name__startswith=value)
| SQ(circle_name__startswith=value)
| SQ(taluk_name__startswith=value))
sqs = SearchQuerySet().filter(query)
sqs.query.set_limits(low=0, high=20)
return sqs.query.get_results()
def filter_or(self, *args, **kwargs):
"""Narrows the search by ensuring certain attributes are not included."""
clone = self._clone()
clone.query.add_filter(SQ(*args, **kwargs), use_or=True)
return clone
def results_for_members(self, team):
base_qs = SearchQuerySet().models(models.TeamVideo)
public = SQ(is_public=True)
mine = SQ(is_public=False, owned_by_team_id=team.pk)
return base_qs.filter(public | mine)
def test_split_expression(self):
sq = SQ(foo="bar")
self.assertEqual(sq.split_expression("foo"), ("foo", "content"))
self.assertEqual(sq.split_expression("foo__exact"), ("foo", "exact"))
self.assertEqual(sq.split_expression("foo__content"), ("foo", "content"))
self.assertEqual(sq.split_expression("foo__contains"), ("foo", "contains"))
self.assertEqual(sq.split_expression("foo__lt"), ("foo", "lt"))
self.assertEqual(sq.split_expression("foo__lte"), ("foo", "lte"))
self.assertEqual(sq.split_expression("foo__gt"), ("foo", "gt"))
self.assertEqual(sq.split_expression("foo__gte"), ("foo", "gte"))
self.assertEqual(sq.split_expression("foo__in"), ("foo", "in"))
self.assertEqual(sq.split_expression("foo__startswith"), ("foo", "startswith"))
self.assertEqual(sq.split_expression("foo__endswith"), ("foo", "endswith"))
self.assertEqual(sq.split_expression("foo__range"), ("foo", "range"))
self.assertEqual(sq.split_expression("foo__fuzzy"), ("foo", "fuzzy"))
# Unrecognized filter. Fall back to exact.
self.assertEqual(sq.split_expression("foo__moof"), ("foo", "content"))
def _boost_search_query(self, sqs):
q = self.cleaned_data['q']
sqs = sqs.filter(SQ(boosted=AutoQuery(q)) | SQ(text=AutoQuery(q)))
return sqs
