def _test_unit_text_search(qs, text, sfields, exact, empty=True):
unit_search = UnitTextSearch(qs)
result = unit_search.search(text, sfields, exact).order_by("pk")
words = unit_search.get_words(text, exact)
fields = unit_search.get_search_fields(sfields)
# ensure result meets our expectation
assert (list(result) == _expected_text_search_results(qs, words, fields))
# ensure that there are no dupes in result qs
assert list(result) == list(result.distinct())
if not empty:
assert result.count()
for item in result:
# item is in original qs
assert item in qs
for word in words:
searchword_found = False
for field in fields:
if word.lower() in getattr(item, field).lower():
# one of the items attrs matches search
searchword_found = True
break
assert searchword_found
def _test_unit_text_search(qs, text, sfields, exact, case, empty=True):
unit_search = UnitTextSearch(qs)
result = unit_search.search(text, sfields, exact, case).order_by("pk")
words = unit_search.get_words(text, exact)
fields = unit_search.get_search_fields(sfields)
# ensure result meets our expectation
assert (
list(result)
== _expected_text_search_results(qs, words, fields, exact, case))
# ensure that there are no dupes in result qs
assert list(result) == list(result.distinct())
if not empty:
assert result.count()
for item in result:
# item is in original qs
assert item in qs
for word in words:
searchword_found = False
for field in fields:
if word.lower() in getattr(item, field).lower():
# one of the items attrs matches search
searchword_found = True
break
assert searchword_found
def test_get_units_text_search(units_text_searches):
search = units_text_searches
sfields = search["sfields"]
fields = _expected_text_search_fields(sfields)
words = _expected_text_search_words(search['text'], search["exact"])
# ensure the fields parser works correctly
assert (
UnitTextSearch(Unit.objects.all()).get_search_fields(sfields)
== fields)
# ensure the text tokeniser works correctly
assert (
UnitTextSearch(Unit.objects.all()).get_words(
search['text'], search["exact"])
== words)
assert isinstance(words, list)
# run the all units test first and check its not empty if it shouldnt be
_test_unit_text_search(
Unit.objects.all(),
search["text"], search["sfields"], search["exact"],
search["empty"])
for qs in [Unit.objects.none(), Unit.objects.live()]:
# run tests against different qs
_test_unit_text_search(
qs, search["text"], search["sfields"], search["exact"])
def filter_qs(self, qs):
kwargs = self.kwargs
category = kwargs['category']
checks = kwargs['checks']
exact = 'exact' in kwargs['soptions']
case = 'case' in kwargs['soptions']
modified_since = kwargs['modified-since']
month = kwargs['month']
search = kwargs['search']
sfields = kwargs['sfields']
user = kwargs['user']
if self.unit_filter:
qs = UnitSearchFilter().filter(
qs, self.unit_filter,
user=user, checks=checks, category=category)
if modified_since is not None:
qs = qs.filter(
change__submitted_on__gt=modified_since).distinct()
if month is not None:
qs = qs.filter(
change__submitted_on__gte=month[0],
change__submitted_on__lte=month[1]).distinct()
if sfields and search:
qs = UnitTextSearch(qs).search(
search, sfields, exact=exact, case=case)
return qs
def filter_qs(self, qs):
kwargs = self.kwargs
category = kwargs["category"]
checks = kwargs["checks"]
exact = "exact" in kwargs["soptions"]
month = kwargs["month"]
search = kwargs["search"]
sfields = kwargs["sfields"]
user = kwargs["user"]
if self.unit_filter:
qs = UnitSearchFilter().filter(qs,
self.unit_filter,
user=user,
checks=checks,
category=category)
if month is not None:
qs = qs.filter(submitted_on__gte=month[0],
submitted_on__lte=month[1]).distinct()
if sfields and search:
qs = UnitTextSearch(qs).search(search, sfields, exact=exact)
return qs
def calculate_search_results(kwargs, user):
pootle_path = kwargs["pootle_path"]
category = kwargs.get("category")
checks = kwargs.get("checks")
offset = kwargs.get("offset", 0)
limit = kwargs.get("count", 9)
modified_since = kwargs.get("modified-since")
month = kwargs.get("month")
search = kwargs.get("search")
sfields = kwargs.get("sfields")
soptions = kwargs.get("soptions", [])
sort = kwargs.get("sort", None)
vfolder = kwargs.get("vfolder", None)
language_code, project_code, dir_path_, filename = (split_pootle_path(
kwargs["pootle_path"]))
uids = [int(x) for x in kwargs.get("uids", "").split(",") if x]
unit_filter = kwargs.get("filter")
if modified_since:
modified_since = parse_datetime(modified_since)
if month:
month = get_date_interval(month)
path_kwargs = {
k: v
for k, v in resolve(pootle_path).kwargs.items()
if k in ["language_code", "project_code", "dir_path", "filename"]
}
qs = (Unit.objects.get_translatable(user=user, **path_kwargs).order_by(
"store", "index"))
if vfolder is not None:
qs = qs.filter(store__vfolders=vfolder)
# if "filter" is present in request vars...
if unit_filter:
# filter the results accordingly
qs = UnitSearchFilter().filter(qs,
unit_filter,
user=user,
checks=checks,
category=get_category_id(category))
# filter by modified
if modified_since:
qs = qs.filter(submitted_on__gt=modified_since).distinct()
if month is not None:
qs = qs.filter(submitted_on__gte=month[0],
submitted_on__lte=month[1]).distinct()
# sort results
if unit_filter in ["my-suggestions", "user-suggestions"]:
sort_on = "suggestions"
elif unit_filter in ["my-submissions", "user-submissions"]:
sort_on = "submissions"
else:
sort_on = "units"
sort_by = ALLOWED_SORTS[sort_on].get(sort, None)
if sort_by is not None:
# filtered sort
if sort_on in SIMPLY_SORTED:
qs = qs.order_by(sort_by, "store__pootle_path", "index")
else:
max_field, sort_order = get_max_and_order_fields(sort_by)
qs = (qs.annotate(sort_by_field=Max(max_field)).order_by(
sort_order, "store__pootle_path", "index"))
# text search
if search and sfields:
qs = UnitTextSearch(qs).search(search, [sfields], "exact" in soptions)
find_unit = (not offset and language_code and project_code and filename
and uids)
start = offset
total = qs.count()
if find_unit:
# find the uid in the Store
uid_list = list(qs.values_list("pk", flat=True))
unit_index = uid_list.index(uids[0])
start = int(unit_index / (2 * limit)) * (2 * limit)
end = min(start + (2 * limit), total)
unit_groups = []
units_by_path = groupby(
qs.values(*GroupedResults.select_fields)[start:end],
lambda x: x["store__pootle_path"])
for pootle_path, units in units_by_path:
unit_groups.append({pootle_path: StoreResults(units).data})
total = qs.count()
return total, start, min(end, total), unit_groups
type__in=SubmissionTypes.EDIT_TYPES).filter(
suggestion__isnull=True).filter(submitter=user).values_list(
"unit_id", flat=True)
# next the suggestions that are accepted and the user is this user
user_suggestions = Suggestion.objects.filter(state__name="accepted",
user=user).values_list(
"unit_id", flat=True)
expected = qs.filter(id__in=(set(user_edit_subs)
| set(user_suggestions))).exclude(
change__submitted_by=user)
assert (list(expected.order_by("pk")) == list(result.order_by("pk")))
def _test_unit_text_search(qs, text, sfields, exact, case, empty=True):
unit_search = UnitTextSearch(qs)
result = unit_search.search(text, sfields, exact, case).order_by("pk")
words = unit_search.get_words(text, exact)
fields = unit_search.get_search_fields(sfields)
# ensure result meets our expectation
assert (list(result) == _expected_text_search_results(
qs, words, fields, exact, case))
# ensure that there are no dupes in result qs
assert list(result) == list(result.distinct())
if not empty:
assert result.count()
for item in result:
def calculate_search_results(kwargs, user):
pootle_path = kwargs["pootle_path"]
category = kwargs.get("category")
checks = kwargs.get("checks")
initial = kwargs.get("initial", False)
limit = kwargs.get("count", 9)
modified_since = kwargs.get("modified-since")
search = kwargs.get("search")
sfields = kwargs.get("sfields")
soptions = kwargs.get("soptions", [])
sort = kwargs.get("sort", None)
uids = [int(x) for x in kwargs.get("uids", "").split(",") if x]
unit_filter = kwargs.get("filter")
if modified_since:
modified_since = parse_datetime(modified_since)
vfolder = None
if 'virtualfolder' in settings.INSTALLED_APPS:
vfolder, pootle_path = extract_vfolder_from_path(
pootle_path,
vfti=VirtualFolderTreeItem.objects.select_related(
"directory", "vfolder"))
qs = (Unit.objects.get_translatable(
user=user, **resolve(pootle_path).kwargs).filter(
store__pootle_path__startswith=pootle_path).order_by(
"store", "index"))
if vfolder is not None:
qs = qs.filter(vfolders=vfolder)
# if "filter" is present in request vars...
if unit_filter:
# filter the results accordingly
qs = UnitSearchFilter().filter(qs,
unit_filter,
user=user,
checks=checks,
category=get_category_id(category))
# filter by modified
if modified_since:
qs = qs.filter(submitted_on__gt=modified_since).distinct()
# sort results
if unit_filter in ["my-suggestions", "user-suggestions"]:
sort_on = "suggestions"
elif unit_filter in ["my-submissions", "user-submissions"]:
sort_on = "submissions"
else:
sort_on = "units"
sort_by = ALLOWED_SORTS[sort_on].get(sort, None)
if sort_by is not None:
# filtered sort
if sort_on in SIMPLY_SORTED:
qs = qs.order_by(sort_by, "store__pootle_path", "index")
else:
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
qs = (qs.annotate(sort_by_field=Max(max_field)).order_by(
sort_order, "store__pootle_path", "index"))
# text search
if search and sfields:
qs = UnitTextSearch(qs).search(search, [sfields], "exact" in soptions)
begin = 0
end = None
uid_list = []
if initial:
uid_list = list(qs.values_list("pk", flat=True))
if uids and len(uids) == 1:
# if uid is set get the index of the uid
index = uid_list.index(uids[0])
begin = max(index - limit, 0)
end = min(index + limit + 1, len(uid_list))
elif uids:
qs = qs.filter(id__in=uids)
if end is None:
end = 2 * limit
unit_groups = []
units_by_path = groupby(
qs.values(*GroupedResults.select_fields)[begin:end],
lambda x: x["store__pootle_path"])
for pootle_path, units in units_by_path:
unit_groups.append({pootle_path: StoreResults(units).data})
return uid_list, unit_groups