def _get_revision_from_since(self, since):
from pootle.core.dateparse import parse_datetime
from pootle.core.utils.timezone import make_aware
from pootle_statistics.models import Submission
if " " in since:
since = "%sT%s%s" % tuple(since.split(" "))
since = make_aware(parse_datetime(since))
submissions_qs = Submission.objects.filter(creation_time__lt=since)
if self.projects:
submissions_qs = submissions_qs.filter(
translation_project__project__code__in=self.projects,
)
if self.languages:
submissions_qs = submissions_qs.filter(
translation_project__language__code__in=self.languages,
)
submission = submissions_qs.last()
if submission is None:
return 0
return submission.unit.revision
def _get_revision_from_since(self, since):
from pootle.core.dateparse import parse_datetime
from pootle.core.utils.timezone import make_aware
from pootle_statistics.models import Submission
if " " in since:
since = "%sT%s%s" % tuple(since.split(" "))
since = make_aware(parse_datetime(since))
submissions_qs = Submission.objects.filter(creation_time__lt=since)
if self.projects:
submissions_qs = submissions_qs.filter(
translation_project__project__code__in=self.projects, )
if self.languages:
submissions_qs = submissions_qs.filter(
translation_project__language__code__in=self.languages, )
submission = submissions_qs.last()
if submission is None:
return 0
return submission.unit.revision
def calculate_search_results(kwargs, user):
pootle_path = kwargs["pootle_path"]
category = kwargs.get("category")
checks = kwargs.get("checks")
offset = kwargs.get("offset", 0)
limit = kwargs.get("count", 9)
modified_since = kwargs.get("modified-since")
month = kwargs.get("month")
search = kwargs.get("search")
sfields = kwargs.get("sfields")
soptions = kwargs.get("soptions", [])
sort = kwargs.get("sort", None)
vfolder = kwargs.get("vfolder", None)
language_code, project_code, dir_path_, filename = (split_pootle_path(
kwargs["pootle_path"]))
uids = [int(x) for x in kwargs.get("uids", "").split(",") if x]
unit_filter = kwargs.get("filter")
if modified_since:
modified_since = parse_datetime(modified_since)
if month:
month = get_date_interval(month)
path_kwargs = {
k: v
for k, v in resolve(pootle_path).kwargs.items()
if k in ["language_code", "project_code", "dir_path", "filename"]
}
qs = (Unit.objects.get_translatable(user=user, **path_kwargs).order_by(
"store", "index"))
if vfolder is not None:
qs = qs.filter(store__vfolders=vfolder)
# if "filter" is present in request vars...
if unit_filter:
# filter the results accordingly
qs = UnitSearchFilter().filter(qs,
unit_filter,
user=user,
checks=checks,
category=get_category_id(category))
# filter by modified
if modified_since:
qs = qs.filter(submitted_on__gt=modified_since).distinct()
if month is not None:
qs = qs.filter(submitted_on__gte=month[0],
submitted_on__lte=month[1]).distinct()
# sort results
if unit_filter in ["my-suggestions", "user-suggestions"]:
sort_on = "suggestions"
elif unit_filter in ["my-submissions", "user-submissions"]:
sort_on = "submissions"
else:
sort_on = "units"
sort_by = ALLOWED_SORTS[sort_on].get(sort, None)
if sort_by is not None:
# filtered sort
if sort_on in SIMPLY_SORTED:
qs = qs.order_by(sort_by, "store__pootle_path", "index")
else:
max_field, sort_order = get_max_and_order_fields(sort_by)
qs = (qs.annotate(sort_by_field=Max(max_field)).order_by(
sort_order, "store__pootle_path", "index"))
# text search
if search and sfields:
qs = UnitTextSearch(qs).search(search, [sfields], "exact" in soptions)
find_unit = (not offset and language_code and project_code and filename
and uids)
start = offset
total = qs.count()
if find_unit:
# find the uid in the Store
uid_list = list(qs.values_list("pk", flat=True))
unit_index = uid_list.index(uids[0])
start = int(unit_index / (2 * limit)) * (2 * limit)
end = min(start + (2 * limit), total)
unit_groups = []
units_by_path = groupby(
qs.values(*GroupedResults.select_fields)[start:end],
lambda x: x["store__pootle_path"])
for pootle_path, units in units_by_path:
unit_groups.append({pootle_path: StoreResults(units).data})
total = qs.count()
return total, start, min(end, total), unit_groups
def get_step_query(request, units_queryset):
"""Narrows down unit query to units matching conditions in GET."""
if 'filter' in request.GET:
unit_filter = request.GET['filter']
username = request.GET.get('user', None)
modified_since = request.GET.get('modified-since', None)
month = request.GET.get('month', None)
sort_by_param = request.GET.get('sort', None)
sort_on = 'units'
user = request.profile
if username is not None:
User = get_user_model()
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
pass
if unit_filter:
match_queryset = units_queryset.none()
if unit_filter == 'all':
match_queryset = units_queryset
elif unit_filter == 'translated':
match_queryset = units_queryset.filter(state=TRANSLATED)
elif unit_filter == 'untranslated':
match_queryset = units_queryset.filter(state=UNTRANSLATED)
elif unit_filter == 'fuzzy':
match_queryset = units_queryset.filter(state=FUZZY)
elif unit_filter == 'incomplete':
match_queryset = units_queryset.filter(
Q(state=UNTRANSLATED) | Q(state=FUZZY), )
elif unit_filter == 'suggestions':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING).distinct()
elif unit_filter in ('my-suggestions', 'user-suggestions'):
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING,
suggestion__user=user,
).distinct()
sort_on = 'suggestions'
elif unit_filter == 'user-suggestions-accepted':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.ACCEPTED,
suggestion__user=user,
).distinct()
elif unit_filter == 'user-suggestions-rejected':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.REJECTED,
suggestion__user=user,
).distinct()
elif unit_filter in ('my-submissions', 'user-submissions'):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).distinct()
sort_on = 'submissions'
elif (unit_filter in ('my-submissions-overwritten',
'user-submissions-overwritten')):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).exclude(submitted_by=user).distinct()
elif unit_filter == 'checks' and 'checks' in request.GET:
checks = request.GET['checks'].split(',')
if checks:
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False,
qualitycheck__name__in=checks,
).distinct()
if modified_since is not None:
datetime_obj = parse_datetime(modified_since)
if datetime_obj is not None:
match_queryset = match_queryset.filter(
submitted_on__gt=datetime_obj, ).distinct()
if month is not None:
[start, end] = get_date_interval(month)
match_queryset = match_queryset.filter(
submitted_on__gte=start,
submitted_on__lte=end,
).distinct()
sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None)
if sort_by is not None:
if sort_on in SIMPLY_SORTED:
if sort_by == 'priority':
# TODO: Replace the following extra() with Coalesce
# https://docs.djangoproject.com/en/1.8/ref/models/database-functions/#coalesce
# once we drop support for Django<1.8.x:
# .annotate(
# sort_by_field=Coalesce(
# Max("vfolders__priority"),
# Value(1)
# )
# ).order_by("-sort_by_field")
match_queryset = match_queryset.extra(
select={
'sort_by_field':
"""
SELECT COALESCE(MAX(virtualfolder_virtualfolder.priority), 1)
FROM virtualfolder_virtualfolder
INNER JOIN virtualfolder_virtualfolder_units
ON virtualfolder_virtualfolder.id = virtualfolder_virtualfolder_units.virtualfolder_id
WHERE virtualfolder_virtualfolder_units.unit_id = pootle_store_unit.id
"""
}).extra(order_by=['-sort_by_field'])
else:
match_queryset = match_queryset.order_by(sort_by)
else:
# Omit leading `-` sign
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
# It's necessary to use `Max()` here because we can't
# use `distinct()` and `order_by()` at the same time
# (unless PostreSQL is used and `distinct(field_name)`)
match_queryset = match_queryset \
.annotate(sort_by_field=Max(max_field)) \
.order_by(sort_order)
units_queryset = match_queryset
if 'search' in request.GET and 'sfields' in request.GET:
# Accept `sfields` to be a comma-separated string of fields (#46)
GET = request.GET.copy()
sfields = GET['sfields']
if isinstance(sfields, unicode) and u',' in sfields:
GET.setlist('sfields', sfields.split(u','))
# use the search form for validation only
search_form = make_search_form(GET)
if search_form.is_valid():
units_queryset = get_search_step_query(search_form, units_queryset)
return units_queryset
def get_step_query(request, units_queryset):
"""Narrows down unit query to units matching conditions in GET."""
if "filter" in request.GET:
unit_filter = request.GET["filter"]
username = request.GET.get("user", None)
modified_since = request.GET.get("modified-since", None)
month = request.GET.get("month", None)
sort_by_param = request.GET.get("sort", None)
sort_on = "units"
user = request.profile
if username is not None:
User = get_user_model()
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
pass
if unit_filter:
match_queryset = units_queryset.none()
if unit_filter == "all":
match_queryset = units_queryset
elif unit_filter == "translated":
match_queryset = units_queryset.filter(state=TRANSLATED)
elif unit_filter == "untranslated":
match_queryset = units_queryset.filter(state=UNTRANSLATED)
elif unit_filter == "fuzzy":
match_queryset = units_queryset.filter(state=FUZZY)
elif unit_filter == "incomplete":
match_queryset = units_queryset.filter(Q(state=UNTRANSLATED) | Q(state=FUZZY))
elif unit_filter == "suggestions":
match_queryset = units_queryset.filter(suggestion__state=SuggestionStates.PENDING).distinct()
elif unit_filter in ("my-suggestions", "user-suggestions"):
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING, suggestion__user=user
).distinct()
sort_on = "suggestions"
elif unit_filter == "user-suggestions-accepted":
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.ACCEPTED, suggestion__user=user
).distinct()
elif unit_filter == "user-suggestions-rejected":
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.REJECTED, suggestion__user=user
).distinct()
elif unit_filter in ("my-submissions", "user-submissions"):
match_queryset = units_queryset.filter(
submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES
).distinct()
sort_on = "submissions"
elif unit_filter in ("my-submissions-overwritten", "user-submissions-overwritten"):
match_queryset = (
units_queryset.filter(submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES)
.exclude(submitted_by=user)
.distinct()
)
elif unit_filter == "checks":
if "checks" in request.GET:
checks = request.GET["checks"].split(",")
if checks:
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False, qualitycheck__name__in=checks
).distinct()
elif "category" in request.GET:
category_name = request.GET["category"]
try:
category = get_category_id(category_name)
except KeyError:
raise Http404
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False, qualitycheck__category=category
).distinct()
if modified_since is not None:
datetime_obj = parse_datetime(modified_since)
if datetime_obj is not None:
match_queryset = match_queryset.filter(submitted_on__gt=datetime_obj).distinct()
if month is not None:
[start, end] = get_date_interval(month)
match_queryset = match_queryset.filter(submitted_on__gte=start, submitted_on__lte=end).distinct()
sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None)
if sort_by is not None:
if sort_on in SIMPLY_SORTED:
if sort_by == "priority":
# TODO: Replace the following extra() with Coalesce
# https://docs.djangoproject.com/en/1.8/ref/models/database-functions/#coalesce
# once we drop support for Django<1.8.x:
# .annotate(
# sort_by_field=Coalesce(
# Max("vfolders__priority"),
# Value(1)
# )
# ).order_by("-sort_by_field")
match_queryset = match_queryset.extra(
select={
"sort_by_field": """
SELECT COALESCE(MAX(virtualfolder_virtualfolder.priority), 1)
FROM virtualfolder_virtualfolder
INNER JOIN virtualfolder_virtualfolder_units
ON virtualfolder_virtualfolder.id = virtualfolder_virtualfolder_units.virtualfolder_id
WHERE virtualfolder_virtualfolder_units.unit_id = pootle_store_unit.id
"""
}
).extra(order_by=["-sort_by_field"])
else:
match_queryset = match_queryset.order_by(sort_by)
else:
# Omit leading `-` sign
if sort_by[0] == "-":
max_field = sort_by[1:]
sort_order = "-sort_by_field"
else:
max_field = sort_by
sort_order = "sort_by_field"
# It's necessary to use `Max()` here because we can't
# use `distinct()` and `order_by()` at the same time
# (unless PostreSQL is used and `distinct(field_name)`)
match_queryset = match_queryset.annotate(sort_by_field=Max(max_field)).order_by(sort_order)
units_queryset = match_queryset
if "search" in request.GET and "sfields" in request.GET:
# Accept `sfields` to be a comma-separated string of fields (#46)
GET = request.GET.copy()
sfields = GET["sfields"]
if isinstance(sfields, unicode) and u"," in sfields:
GET.setlist("sfields", sfields.split(u","))
# use the search form for validation only
search_form = make_search_form(GET)
if search_form.is_valid():
units_queryset = get_search_step_query(search_form, units_queryset)
return units_queryset
def to_python(self, value):
if value is not None:
return parse_datetime(value)
def calculate_search_results(kwargs, user):
pootle_path = kwargs["pootle_path"]
category = kwargs.get("category")
checks = kwargs.get("checks")
initial = kwargs.get("initial", False)
limit = kwargs.get("count", 9)
modified_since = kwargs.get("modified-since")
search = kwargs.get("search")
sfields = kwargs.get("sfields")
soptions = kwargs.get("soptions", [])
sort = kwargs.get("sort", None)
uids = [
int(x)
for x
in kwargs.get("uids", "").split(",")
if x]
unit_filter = kwargs.get("filter")
if modified_since:
modified_since = parse_datetime(modified_since)
vfolder = None
if 'virtualfolder' in settings.INSTALLED_APPS:
vfolder, pootle_path = extract_vfolder_from_path(
pootle_path,
vfti=VirtualFolderTreeItem.objects.select_related(
"directory", "vfolder"))
qs = (
Unit.objects.get_translatable(user=user, **resolve(pootle_path).kwargs)
.filter(store__pootle_path__startswith=pootle_path)
.order_by("store", "index"))
if vfolder is not None:
qs = qs.filter(vfolders=vfolder)
# if "filter" is present in request vars...
if unit_filter:
# filter the results accordingly
qs = UnitSearchFilter().filter(
qs,
unit_filter,
user=user,
checks=checks,
category=get_category_id(category))
# filter by modified
if modified_since:
qs = qs.filter(submitted_on__gt=modified_since).distinct()
# sort results
if unit_filter in ["my-suggestions", "user-suggestions"]:
sort_on = "suggestions"
elif unit_filter in ["my-submissions", "user-submissions"]:
sort_on = "submissions"
else:
sort_on = "units"
sort_by = ALLOWED_SORTS[sort_on].get(sort, None)
if sort_by is not None:
# filtered sort
if sort_on in SIMPLY_SORTED:
qs = qs.order_by(sort_by, "store__pootle_path", "index")
else:
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
qs = (
qs.annotate(sort_by_field=Max(max_field))
.order_by(sort_order, "store__pootle_path", "index"))
# text search
if search and sfields:
qs = UnitTextSearch(qs).search(
search,
[sfields],
"exact" in soptions)
begin = 0
end = None
uid_list = []
if initial:
uid_list = list(qs.values_list("pk", flat=True))
if uids and len(uids) == 1:
# if uid is set get the index of the uid
index = uid_list.index(uids[0])
begin = max(index - limit, 0)
end = min(index + limit + 1, len(uid_list))
elif uids:
qs = qs.filter(id__in=uids)
if end is None:
end = 2 * limit
unit_groups = []
units_by_path = groupby(
qs.values(*GroupedResults.select_fields)[begin:end],
lambda x: x["store__pootle_path"])
for pootle_path, units in units_by_path:
unit_groups.append(
{pootle_path: StoreResults(units).data})
return uid_list, unit_groups
def calculate_search_results(kwargs, user):
pootle_path = kwargs["pootle_path"]
category = kwargs.get("category")
checks = kwargs.get("checks")
offset = kwargs.get("offset", 0)
limit = kwargs.get("count", 9)
modified_since = kwargs.get("modified-since")
month = kwargs.get("month")
search = kwargs.get("search")
sfields = kwargs.get("sfields")
soptions = kwargs.get("soptions", [])
sort = kwargs.get("sort", None)
vfolder = kwargs.get("vfolder", None)
language_code, project_code, dir_path_, filename = (
split_pootle_path(kwargs["pootle_path"]))
uids = [
int(x)
for x
in kwargs.get("uids", "").split(",")
if x]
unit_filter = kwargs.get("filter")
if modified_since:
modified_since = parse_datetime(modified_since)
if month:
month = get_date_interval(month)
path_kwargs = {
k: v
for k, v
in resolve(pootle_path).kwargs.items()
if k in [
"language_code",
"project_code",
"dir_path",
"filename"]}
qs = (
Unit.objects.get_translatable(user=user, **path_kwargs)
.order_by("store", "index"))
if vfolder is not None:
qs = qs.filter(store__vfolders=vfolder)
# if "filter" is present in request vars...
if unit_filter:
# filter the results accordingly
qs = UnitSearchFilter().filter(
qs,
unit_filter,
user=user,
checks=checks,
category=get_category_id(category))
# filter by modified
if modified_since:
qs = qs.filter(change__submitted_on__gt=modified_since).distinct()
if month is not None:
qs = qs.filter(
change__submitted_on__gte=month[0],
change__submitted_on__lte=month[1]).distinct()
# sort results
if unit_filter in ["my-suggestions", "user-suggestions"]:
sort_on = "suggestions"
elif unit_filter in ["my-submissions", "user-submissions"]:
sort_on = "submissions"
else:
sort_on = "units"
sort_by = ALLOWED_SORTS[sort_on].get(sort, None)
if sort_by is not None:
# filtered sort
if sort_on in SIMPLY_SORTED:
qs = qs.order_by(sort_by, "store__pootle_path", "index")
else:
max_field, sort_order = get_max_and_order_fields(sort_by)
qs = (
qs.annotate(sort_by_field=Max(max_field))
.order_by(sort_order, "store__pootle_path", "index"))
# text search
if search and sfields:
qs = UnitTextSearch(qs).search(
search,
[sfields],
"exact" in soptions)
find_unit = (
not offset
and language_code
and project_code
and filename
and uids)
start = offset
total = qs.count()
if find_unit:
# find the uid in the Store
uid_list = list(qs.values_list("pk", flat=True))
unit_index = uid_list.index(uids[0])
start = int(unit_index / (2 * limit)) * (2 * limit)
end = min(start + (2 * limit), total)
unit_groups = []
units_by_path = groupby(
qs.values(*GroupedResults.select_fields)[start:end],
lambda x: x["store__pootle_path"])
for pootle_path, units in units_by_path:
unit_groups.append(
{pootle_path: StoreResults(units).data})
total = qs.count()
return total, start, min(end, total), unit_groups
def get_step_query(request, units_queryset):
"""Narrows down unit query to units matching conditions in GET."""
if 'filter' in request.GET:
unit_filter = request.GET['filter']
username = request.GET.get('user', None)
modified_since = request.GET.get('modified-since', None)
month = request.GET.get('month', None)
sort_by_param = request.GET.get('sort', None)
sort_on = 'units'
user = request.profile
if username is not None:
User = get_user_model()
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
pass
if unit_filter:
match_queryset = units_queryset.none()
if unit_filter == 'all':
match_queryset = units_queryset
elif unit_filter == 'translated':
match_queryset = units_queryset.filter(state=TRANSLATED)
elif unit_filter == 'untranslated':
match_queryset = units_queryset.filter(state=UNTRANSLATED)
elif unit_filter == 'fuzzy':
match_queryset = units_queryset.filter(state=FUZZY)
elif unit_filter == 'incomplete':
match_queryset = units_queryset.filter(
Q(state=UNTRANSLATED) | Q(state=FUZZY),
)
elif unit_filter == 'suggestions':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING).distinct()
elif unit_filter in ('my-suggestions', 'user-suggestions'):
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING,
suggestion__user=user,
).distinct()
sort_on = 'suggestions'
elif unit_filter == 'user-suggestions-accepted':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.ACCEPTED,
suggestion__user=user,
).distinct()
elif unit_filter == 'user-suggestions-rejected':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.REJECTED,
suggestion__user=user,
).distinct()
elif unit_filter in ('my-submissions', 'user-submissions'):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).distinct()
sort_on = 'submissions'
elif (unit_filter in ('my-submissions-overwritten',
'user-submissions-overwritten')):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).exclude(submitted_by=user).distinct()
elif unit_filter == 'checks':
if 'checks' in request.GET:
checks = request.GET['checks'].split(',')
if checks:
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False,
qualitycheck__name__in=checks,
).distinct()
elif 'category' in request.GET:
category_name = request.GET['category']
try:
category = get_category_id(category_name)
except KeyError:
raise Http404
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False,
qualitycheck__category=category,
).distinct()
if modified_since is not None:
datetime_obj = parse_datetime(modified_since)
if datetime_obj is not None:
match_queryset = match_queryset.filter(
submitted_on__gt=datetime_obj,
).distinct()
if month is not None:
[start, end] = get_date_interval(month)
match_queryset = match_queryset.filter(
submitted_on__gte=start,
submitted_on__lte=end,
).distinct()
sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None)
if sort_by is not None:
if sort_on in SIMPLY_SORTED:
match_queryset = match_queryset.order_by(sort_by)
else:
# Omit leading `-` sign
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
# It's necessary to use `Max()` here because we can't
# use `distinct()` and `order_by()` at the same time
# (unless PostreSQL is used and `distinct(field_name)`)
match_queryset = match_queryset \
.annotate(sort_by_field=Max(max_field)) \
.order_by(sort_order)
units_queryset = match_queryset
if 'search' in request.GET and 'sfields' in request.GET:
# Accept `sfields` to be a comma-separated string of fields (#46)
GET = request.GET.copy()
sfields = GET['sfields']
if isinstance(sfields, unicode) and u',' in sfields:
GET.setlist('sfields', sfields.split(u','))
# use the search form for validation only
search_form = make_search_form(GET)
if search_form.is_valid():
units_queryset = get_search_step_query(search_form, units_queryset)
return units_queryset
def get_step_query(request, units_queryset):
"""Narrows down unit query to units matching conditions in GET."""
if 'filter' in request.GET:
unit_filter = request.GET['filter']
username = request.GET.get('user', None)
modified_since = request.GET.get('modified-since', None)
month = request.GET.get('month', None)
sort_by_param = request.GET.get('sort', None)
sort_on = 'units'
user = request.profile
if username is not None:
User = get_user_model()
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
pass
if unit_filter:
match_queryset = units_queryset.none()
if unit_filter == 'all':
match_queryset = units_queryset
elif unit_filter == 'translated':
match_queryset = units_queryset.filter(state=TRANSLATED)
elif unit_filter == 'untranslated':
match_queryset = units_queryset.filter(state=UNTRANSLATED)
elif unit_filter == 'fuzzy':
match_queryset = units_queryset.filter(state=FUZZY)
elif unit_filter == 'incomplete':
match_queryset = units_queryset.filter(
Q(state=UNTRANSLATED) | Q(state=FUZZY),
)
elif unit_filter == 'suggestions':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING
).distinct()
elif unit_filter in ('my-suggestions', 'user-suggestions'):
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING,
suggestion__user=user,
).distinct()
sort_on = 'suggestions'
elif unit_filter == 'user-suggestions-accepted':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.ACCEPTED,
suggestion__user=user,
).distinct()
elif unit_filter == 'user-suggestions-rejected':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.REJECTED,
suggestion__user=user,
).distinct()
elif unit_filter in ('my-submissions', 'user-submissions'):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).distinct()
sort_on = 'submissions'
elif (unit_filter in ('my-submissions-overwritten',
'user-submissions-overwritten')):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).exclude(submitted_by=user).distinct()
elif unit_filter == 'checks' and 'checks' in request.GET:
checks = request.GET['checks'].split(',')
if checks:
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False,
qualitycheck__name__in=checks,
).distinct()
if modified_since is not None:
datetime_obj = parse_datetime(modified_since)
if datetime_obj is not None:
match_queryset = match_queryset.filter(
submitted_on__gt=datetime_obj,
).distinct()
if month is not None:
[start, end] = get_date_interval(month)
match_queryset = match_queryset.filter(
submitted_on__gte=start,
submitted_on__lte=end,
).distinct()
sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None)
if sort_by is not None:
if sort_on in SIMPLY_SORTED:
if sort_by == 'priority':
# TODO: Replace the following extra() with Coalesce
# https://docs.djangoproject.com/en/1.8/ref/models/database-functions/#coalesce
# once we drop support for Django<1.8.x:
# .annotate(
# sort_by_field=Coalesce(
# Max("vfolders__priority"),
# Value(1)
# )
# ).order_by("-sort_by_field")
match_queryset = match_queryset.extra(select={'sort_by_field': """
SELECT COALESCE(MAX(virtualfolder_virtualfolder.priority), 1)
FROM virtualfolder_virtualfolder
INNER JOIN virtualfolder_virtualfolder_units
ON virtualfolder_virtualfolder.id = virtualfolder_virtualfolder_units.virtualfolder_id
WHERE virtualfolder_virtualfolder_units.unit_id = pootle_store_unit.id
"""}).extra(order_by=['-sort_by_field'])
else:
match_queryset = match_queryset.order_by(sort_by)
else:
# Omit leading `-` sign
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
# It's necessary to use `Max()` here because we can't
# use `distinct()` and `order_by()` at the same time
# (unless PostreSQL is used and `distinct(field_name)`)
match_queryset = match_queryset \
.annotate(sort_by_field=Max(max_field)) \
.order_by(sort_order)
units_queryset = match_queryset
if 'search' in request.GET and 'sfields' in request.GET:
# Accept `sfields` to be a comma-separated string of fields (#46)
GET = request.GET.copy()
sfields = GET['sfields']
if isinstance(sfields, unicode) and u',' in sfields:
GET.setlist('sfields', sfields.split(u','))
# use the search form for validation only
search_form = make_search_form(GET)
if search_form.is_valid():
units_queryset = get_search_step_query(search_form, units_queryset)
return units_queryset
def to_python(self, value):
if value is not None:
return parse_datetime(value)
def calculate_search_results(kwargs, user):
pootle_path = kwargs["pootle_path"]
category = kwargs.get("category")
checks = kwargs.get("checks")
initial = kwargs.get("initial", False)
limit = kwargs.get("count", 9)
modified_since = kwargs.get("modified-since")
search = kwargs.get("search")
sfields = kwargs.get("sfields")
soptions = kwargs.get("soptions", [])
sort = kwargs.get("sort", None)
uids = [int(x) for x in kwargs.get("uids", "").split(",") if x]
unit_filter = kwargs.get("filter")
if modified_since:
modified_since = parse_datetime(modified_since)
vfolder = None
if 'virtualfolder' in settings.INSTALLED_APPS:
vfolder, pootle_path = extract_vfolder_from_path(
pootle_path,
vfti=VirtualFolderTreeItem.objects.select_related(
"directory", "vfolder"))
qs = (Unit.objects.get_translatable(
user=user, **resolve(pootle_path).kwargs).filter(
store__pootle_path__startswith=pootle_path).order_by(
"store", "index"))
if vfolder is not None:
qs = qs.filter(vfolders=vfolder)
# if "filter" is present in request vars...
if unit_filter:
# filter the results accordingly
qs = UnitSearchFilter().filter(qs,
unit_filter,
user=user,
checks=checks,
category=get_category_id(category))
# filter by modified
if modified_since:
qs = qs.filter(submitted_on__gt=modified_since).distinct()
# sort results
if unit_filter in ["my-suggestions", "user-suggestions"]:
sort_on = "suggestions"
elif unit_filter in ["my-submissions", "user-submissions"]:
sort_on = "submissions"
else:
sort_on = "units"
sort_by = ALLOWED_SORTS[sort_on].get(sort, None)
if sort_by is not None:
# filtered sort
if sort_on in SIMPLY_SORTED:
qs = qs.order_by(sort_by, "store__pootle_path", "index")
else:
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
qs = (qs.annotate(sort_by_field=Max(max_field)).order_by(
sort_order, "store__pootle_path", "index"))
# text search
if search and sfields:
qs = UnitTextSearch(qs).search(search, [sfields], "exact" in soptions)
begin = 0
end = None
uid_list = []
if initial:
uid_list = list(qs.values_list("pk", flat=True))
if uids and len(uids) == 1:
# if uid is set get the index of the uid
index = uid_list.index(uids[0])
begin = max(index - limit, 0)
end = min(index + limit + 1, len(uid_list))
elif uids:
qs = qs.filter(id__in=uids)
if end is None:
end = 2 * limit
unit_groups = []
units_by_path = groupby(
qs.values(*GroupedResults.select_fields)[begin:end],
lambda x: x["store__pootle_path"])
for pootle_path, units in units_by_path:
unit_groups.append({pootle_path: StoreResults(units).data})
return uid_list, unit_groups
def get_step_query(request, units_queryset):
"""Narrows down unit query to units matching conditions in GET."""
if 'filter' in request.GET:
unit_filter = request.GET['filter']
username = request.GET.get('user', None)
modified_since = request.GET.get('modified-since', None)
month = request.GET.get('month', None)
sort_by_param = request.GET.get('sort', None)
sort_on = 'units'
user = request.profile
if username is not None:
User = get_user_model()
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
pass
if unit_filter:
match_queryset = units_queryset.none()
if unit_filter == 'all':
match_queryset = units_queryset
elif unit_filter == 'translated':
match_queryset = units_queryset.filter(state=TRANSLATED)
elif unit_filter == 'untranslated':
match_queryset = units_queryset.filter(state=UNTRANSLATED)
elif unit_filter == 'fuzzy':
match_queryset = units_queryset.filter(state=FUZZY)
elif unit_filter == 'incomplete':
match_queryset = units_queryset.filter(
Q(state=UNTRANSLATED) | Q(state=FUZZY), )
elif unit_filter == 'suggestions':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING).distinct()
elif unit_filter in ('my-suggestions', 'user-suggestions'):
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.PENDING,
suggestion__user=user,
).distinct()
sort_on = 'suggestions'
elif unit_filter == 'user-suggestions-accepted':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.ACCEPTED,
suggestion__user=user,
).distinct()
elif unit_filter == 'user-suggestions-rejected':
match_queryset = units_queryset.filter(
suggestion__state=SuggestionStates.REJECTED,
suggestion__user=user,
).distinct()
elif unit_filter in ('my-submissions', 'user-submissions'):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).distinct()
sort_on = 'submissions'
elif (unit_filter in ('my-submissions-overwritten',
'user-submissions-overwritten')):
match_queryset = units_queryset.filter(
submission__submitter=user,
submission__type__in=SubmissionTypes.EDIT_TYPES,
).exclude(submitted_by=user).distinct()
elif unit_filter == 'checks' and 'checks' in request.GET:
checks = request.GET['checks'].split(',')
if checks:
match_queryset = units_queryset.filter(
qualitycheck__false_positive=False,
qualitycheck__name__in=checks,
).distinct()
if modified_since is not None:
datetime_obj = parse_datetime(modified_since)
if datetime_obj is not None:
match_queryset = match_queryset.filter(
submitted_on__gt=datetime_obj, ).distinct()
if month is not None:
[start, end] = get_date_interval(month)
match_queryset = match_queryset.filter(
submitted_on__gte=start,
submitted_on__lte=end,
).distinct()
sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None)
if sort_by is not None:
if sort_on in SIMPLY_SORTED:
match_queryset = match_queryset.order_by(sort_by)
else:
# Omit leading `-` sign
if sort_by[0] == '-':
max_field = sort_by[1:]
sort_order = '-sort_by_field'
else:
max_field = sort_by
sort_order = 'sort_by_field'
# It's necessary to use `Max()` here because we can't
# use `distinct()` and `order_by()` at the same time
# (unless PostreSQL is used and `distinct(field_name)`)
match_queryset = match_queryset \
.annotate(sort_by_field=Max(max_field)) \
.order_by(sort_order)
units_queryset = match_queryset
if 'search' in request.GET and 'sfields' in request.GET:
# Accept `sfields` to be a comma-separated string of fields (#46)
GET = request.GET.copy()
sfields = GET['sfields']
if isinstance(sfields, unicode) and u',' in sfields:
GET.setlist('sfields', sfields.split(u','))
# use the search form for validation only
search_form = make_search_form(GET)
if search_form.is_valid():
units_queryset = get_search_step_query(search_form, units_queryset)
return units_queryset
