def _get_revision_from_since(self, since): from pootle.core.dateparse import parse_datetime from pootle.core.utils.timezone import make_aware from pootle_statistics.models import Submission if " " in since: since = "%sT%s%s" % tuple(since.split(" ")) since = make_aware(parse_datetime(since)) submissions_qs = Submission.objects.filter(creation_time__lt=since) if self.projects: submissions_qs = submissions_qs.filter( translation_project__project__code__in=self.projects, ) if self.languages: submissions_qs = submissions_qs.filter( translation_project__language__code__in=self.languages, ) submission = submissions_qs.last() if submission is None: return 0 return submission.unit.revision
def calculate_search_results(kwargs, user): pootle_path = kwargs["pootle_path"] category = kwargs.get("category") checks = kwargs.get("checks") offset = kwargs.get("offset", 0) limit = kwargs.get("count", 9) modified_since = kwargs.get("modified-since") month = kwargs.get("month") search = kwargs.get("search") sfields = kwargs.get("sfields") soptions = kwargs.get("soptions", []) sort = kwargs.get("sort", None) vfolder = kwargs.get("vfolder", None) language_code, project_code, dir_path_, filename = (split_pootle_path( kwargs["pootle_path"])) uids = [int(x) for x in kwargs.get("uids", "").split(",") if x] unit_filter = kwargs.get("filter") if modified_since: modified_since = parse_datetime(modified_since) if month: month = get_date_interval(month) path_kwargs = { k: v for k, v in resolve(pootle_path).kwargs.items() if k in ["language_code", "project_code", "dir_path", "filename"] } qs = (Unit.objects.get_translatable(user=user, **path_kwargs).order_by( "store", "index")) if vfolder is not None: qs = qs.filter(store__vfolders=vfolder) # if "filter" is present in request vars... if unit_filter: # filter the results accordingly qs = UnitSearchFilter().filter(qs, unit_filter, user=user, checks=checks, category=get_category_id(category)) # filter by modified if modified_since: qs = qs.filter(submitted_on__gt=modified_since).distinct() if month is not None: qs = qs.filter(submitted_on__gte=month[0], submitted_on__lte=month[1]).distinct() # sort results if unit_filter in ["my-suggestions", "user-suggestions"]: sort_on = "suggestions" elif unit_filter in ["my-submissions", "user-submissions"]: sort_on = "submissions" else: sort_on = "units" sort_by = ALLOWED_SORTS[sort_on].get(sort, None) if sort_by is not None: # filtered sort if sort_on in SIMPLY_SORTED: qs = qs.order_by(sort_by, "store__pootle_path", "index") else: max_field, sort_order = get_max_and_order_fields(sort_by) qs = (qs.annotate(sort_by_field=Max(max_field)).order_by( sort_order, "store__pootle_path", "index")) # text search if search and sfields: qs = UnitTextSearch(qs).search(search, [sfields], "exact" in soptions) find_unit = (not offset and language_code and project_code and filename and uids) start = offset total = qs.count() if find_unit: # find the uid in the Store uid_list = list(qs.values_list("pk", flat=True)) unit_index = uid_list.index(uids[0]) start = int(unit_index / (2 * limit)) * (2 * limit) end = min(start + (2 * limit), total) unit_groups = [] units_by_path = groupby( qs.values(*GroupedResults.select_fields)[start:end], lambda x: x["store__pootle_path"]) for pootle_path, units in units_by_path: unit_groups.append({pootle_path: StoreResults(units).data}) total = qs.count() return total, start, min(end, total), unit_groups
def get_step_query(request, units_queryset): """Narrows down unit query to units matching conditions in GET.""" if 'filter' in request.GET: unit_filter = request.GET['filter'] username = request.GET.get('user', None) modified_since = request.GET.get('modified-since', None) month = request.GET.get('month', None) sort_by_param = request.GET.get('sort', None) sort_on = 'units' user = request.profile if username is not None: User = get_user_model() try: user = User.objects.get(username=username) except User.DoesNotExist: pass if unit_filter: match_queryset = units_queryset.none() if unit_filter == 'all': match_queryset = units_queryset elif unit_filter == 'translated': match_queryset = units_queryset.filter(state=TRANSLATED) elif unit_filter == 'untranslated': match_queryset = units_queryset.filter(state=UNTRANSLATED) elif unit_filter == 'fuzzy': match_queryset = units_queryset.filter(state=FUZZY) elif unit_filter == 'incomplete': match_queryset = units_queryset.filter( Q(state=UNTRANSLATED) | Q(state=FUZZY), ) elif unit_filter == 'suggestions': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING).distinct() elif unit_filter in ('my-suggestions', 'user-suggestions'): match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING, suggestion__user=user, ).distinct() sort_on = 'suggestions' elif unit_filter == 'user-suggestions-accepted': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.ACCEPTED, suggestion__user=user, ).distinct() elif unit_filter == 'user-suggestions-rejected': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.REJECTED, suggestion__user=user, ).distinct() elif unit_filter in ('my-submissions', 'user-submissions'): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).distinct() sort_on = 'submissions' elif (unit_filter in ('my-submissions-overwritten', 'user-submissions-overwritten')): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).exclude(submitted_by=user).distinct() elif unit_filter == 'checks' and 'checks' in request.GET: checks = request.GET['checks'].split(',') if checks: match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__name__in=checks, ).distinct() if modified_since is not None: datetime_obj = parse_datetime(modified_since) if datetime_obj is not None: match_queryset = match_queryset.filter( submitted_on__gt=datetime_obj, ).distinct() if month is not None: [start, end] = get_date_interval(month) match_queryset = match_queryset.filter( submitted_on__gte=start, submitted_on__lte=end, ).distinct() sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None) if sort_by is not None: if sort_on in SIMPLY_SORTED: if sort_by == 'priority': # TODO: Replace the following extra() with Coalesce # https://docs.djangoproject.com/en/1.8/ref/models/database-functions/#coalesce # once we drop support for Django<1.8.x: # .annotate( # sort_by_field=Coalesce( # Max("vfolders__priority"), # Value(1) # ) # ).order_by("-sort_by_field") match_queryset = match_queryset.extra( select={ 'sort_by_field': """ SELECT COALESCE(MAX(virtualfolder_virtualfolder.priority), 1) FROM virtualfolder_virtualfolder INNER JOIN virtualfolder_virtualfolder_units ON virtualfolder_virtualfolder.id = virtualfolder_virtualfolder_units.virtualfolder_id WHERE virtualfolder_virtualfolder_units.unit_id = pootle_store_unit.id """ }).extra(order_by=['-sort_by_field']) else: match_queryset = match_queryset.order_by(sort_by) else: # Omit leading `-` sign if sort_by[0] == '-': max_field = sort_by[1:] sort_order = '-sort_by_field' else: max_field = sort_by sort_order = 'sort_by_field' # It's necessary to use `Max()` here because we can't # use `distinct()` and `order_by()` at the same time # (unless PostreSQL is used and `distinct(field_name)`) match_queryset = match_queryset \ .annotate(sort_by_field=Max(max_field)) \ .order_by(sort_order) units_queryset = match_queryset if 'search' in request.GET and 'sfields' in request.GET: # Accept `sfields` to be a comma-separated string of fields (#46) GET = request.GET.copy() sfields = GET['sfields'] if isinstance(sfields, unicode) and u',' in sfields: GET.setlist('sfields', sfields.split(u',')) # use the search form for validation only search_form = make_search_form(GET) if search_form.is_valid(): units_queryset = get_search_step_query(search_form, units_queryset) return units_queryset
def get_step_query(request, units_queryset): """Narrows down unit query to units matching conditions in GET.""" if "filter" in request.GET: unit_filter = request.GET["filter"] username = request.GET.get("user", None) modified_since = request.GET.get("modified-since", None) month = request.GET.get("month", None) sort_by_param = request.GET.get("sort", None) sort_on = "units" user = request.profile if username is not None: User = get_user_model() try: user = User.objects.get(username=username) except User.DoesNotExist: pass if unit_filter: match_queryset = units_queryset.none() if unit_filter == "all": match_queryset = units_queryset elif unit_filter == "translated": match_queryset = units_queryset.filter(state=TRANSLATED) elif unit_filter == "untranslated": match_queryset = units_queryset.filter(state=UNTRANSLATED) elif unit_filter == "fuzzy": match_queryset = units_queryset.filter(state=FUZZY) elif unit_filter == "incomplete": match_queryset = units_queryset.filter(Q(state=UNTRANSLATED) | Q(state=FUZZY)) elif unit_filter == "suggestions": match_queryset = units_queryset.filter(suggestion__state=SuggestionStates.PENDING).distinct() elif unit_filter in ("my-suggestions", "user-suggestions"): match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING, suggestion__user=user ).distinct() sort_on = "suggestions" elif unit_filter == "user-suggestions-accepted": match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.ACCEPTED, suggestion__user=user ).distinct() elif unit_filter == "user-suggestions-rejected": match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.REJECTED, suggestion__user=user ).distinct() elif unit_filter in ("my-submissions", "user-submissions"): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES ).distinct() sort_on = "submissions" elif unit_filter in ("my-submissions-overwritten", "user-submissions-overwritten"): match_queryset = ( units_queryset.filter(submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES) .exclude(submitted_by=user) .distinct() ) elif unit_filter == "checks": if "checks" in request.GET: checks = request.GET["checks"].split(",") if checks: match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__name__in=checks ).distinct() elif "category" in request.GET: category_name = request.GET["category"] try: category = get_category_id(category_name) except KeyError: raise Http404 match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__category=category ).distinct() if modified_since is not None: datetime_obj = parse_datetime(modified_since) if datetime_obj is not None: match_queryset = match_queryset.filter(submitted_on__gt=datetime_obj).distinct() if month is not None: [start, end] = get_date_interval(month) match_queryset = match_queryset.filter(submitted_on__gte=start, submitted_on__lte=end).distinct() sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None) if sort_by is not None: if sort_on in SIMPLY_SORTED: if sort_by == "priority": # TODO: Replace the following extra() with Coalesce # https://docs.djangoproject.com/en/1.8/ref/models/database-functions/#coalesce # once we drop support for Django<1.8.x: # .annotate( # sort_by_field=Coalesce( # Max("vfolders__priority"), # Value(1) # ) # ).order_by("-sort_by_field") match_queryset = match_queryset.extra( select={ "sort_by_field": """ SELECT COALESCE(MAX(virtualfolder_virtualfolder.priority), 1) FROM virtualfolder_virtualfolder INNER JOIN virtualfolder_virtualfolder_units ON virtualfolder_virtualfolder.id = virtualfolder_virtualfolder_units.virtualfolder_id WHERE virtualfolder_virtualfolder_units.unit_id = pootle_store_unit.id """ } ).extra(order_by=["-sort_by_field"]) else: match_queryset = match_queryset.order_by(sort_by) else: # Omit leading `-` sign if sort_by[0] == "-": max_field = sort_by[1:] sort_order = "-sort_by_field" else: max_field = sort_by sort_order = "sort_by_field" # It's necessary to use `Max()` here because we can't # use `distinct()` and `order_by()` at the same time # (unless PostreSQL is used and `distinct(field_name)`) match_queryset = match_queryset.annotate(sort_by_field=Max(max_field)).order_by(sort_order) units_queryset = match_queryset if "search" in request.GET and "sfields" in request.GET: # Accept `sfields` to be a comma-separated string of fields (#46) GET = request.GET.copy() sfields = GET["sfields"] if isinstance(sfields, unicode) and u"," in sfields: GET.setlist("sfields", sfields.split(u",")) # use the search form for validation only search_form = make_search_form(GET) if search_form.is_valid(): units_queryset = get_search_step_query(search_form, units_queryset) return units_queryset
def to_python(self, value): if value is not None: return parse_datetime(value)
def calculate_search_results(kwargs, user): pootle_path = kwargs["pootle_path"] category = kwargs.get("category") checks = kwargs.get("checks") initial = kwargs.get("initial", False) limit = kwargs.get("count", 9) modified_since = kwargs.get("modified-since") search = kwargs.get("search") sfields = kwargs.get("sfields") soptions = kwargs.get("soptions", []) sort = kwargs.get("sort", None) uids = [ int(x) for x in kwargs.get("uids", "").split(",") if x] unit_filter = kwargs.get("filter") if modified_since: modified_since = parse_datetime(modified_since) vfolder = None if 'virtualfolder' in settings.INSTALLED_APPS: vfolder, pootle_path = extract_vfolder_from_path( pootle_path, vfti=VirtualFolderTreeItem.objects.select_related( "directory", "vfolder")) qs = ( Unit.objects.get_translatable(user=user, **resolve(pootle_path).kwargs) .filter(store__pootle_path__startswith=pootle_path) .order_by("store", "index")) if vfolder is not None: qs = qs.filter(vfolders=vfolder) # if "filter" is present in request vars... if unit_filter: # filter the results accordingly qs = UnitSearchFilter().filter( qs, unit_filter, user=user, checks=checks, category=get_category_id(category)) # filter by modified if modified_since: qs = qs.filter(submitted_on__gt=modified_since).distinct() # sort results if unit_filter in ["my-suggestions", "user-suggestions"]: sort_on = "suggestions" elif unit_filter in ["my-submissions", "user-submissions"]: sort_on = "submissions" else: sort_on = "units" sort_by = ALLOWED_SORTS[sort_on].get(sort, None) if sort_by is not None: # filtered sort if sort_on in SIMPLY_SORTED: qs = qs.order_by(sort_by, "store__pootle_path", "index") else: if sort_by[0] == '-': max_field = sort_by[1:] sort_order = '-sort_by_field' else: max_field = sort_by sort_order = 'sort_by_field' qs = ( qs.annotate(sort_by_field=Max(max_field)) .order_by(sort_order, "store__pootle_path", "index")) # text search if search and sfields: qs = UnitTextSearch(qs).search( search, [sfields], "exact" in soptions) begin = 0 end = None uid_list = [] if initial: uid_list = list(qs.values_list("pk", flat=True)) if uids and len(uids) == 1: # if uid is set get the index of the uid index = uid_list.index(uids[0]) begin = max(index - limit, 0) end = min(index + limit + 1, len(uid_list)) elif uids: qs = qs.filter(id__in=uids) if end is None: end = 2 * limit unit_groups = [] units_by_path = groupby( qs.values(*GroupedResults.select_fields)[begin:end], lambda x: x["store__pootle_path"]) for pootle_path, units in units_by_path: unit_groups.append( {pootle_path: StoreResults(units).data}) return uid_list, unit_groups
def calculate_search_results(kwargs, user): pootle_path = kwargs["pootle_path"] category = kwargs.get("category") checks = kwargs.get("checks") offset = kwargs.get("offset", 0) limit = kwargs.get("count", 9) modified_since = kwargs.get("modified-since") month = kwargs.get("month") search = kwargs.get("search") sfields = kwargs.get("sfields") soptions = kwargs.get("soptions", []) sort = kwargs.get("sort", None) vfolder = kwargs.get("vfolder", None) language_code, project_code, dir_path_, filename = ( split_pootle_path(kwargs["pootle_path"])) uids = [ int(x) for x in kwargs.get("uids", "").split(",") if x] unit_filter = kwargs.get("filter") if modified_since: modified_since = parse_datetime(modified_since) if month: month = get_date_interval(month) path_kwargs = { k: v for k, v in resolve(pootle_path).kwargs.items() if k in [ "language_code", "project_code", "dir_path", "filename"]} qs = ( Unit.objects.get_translatable(user=user, **path_kwargs) .order_by("store", "index")) if vfolder is not None: qs = qs.filter(store__vfolders=vfolder) # if "filter" is present in request vars... if unit_filter: # filter the results accordingly qs = UnitSearchFilter().filter( qs, unit_filter, user=user, checks=checks, category=get_category_id(category)) # filter by modified if modified_since: qs = qs.filter(change__submitted_on__gt=modified_since).distinct() if month is not None: qs = qs.filter( change__submitted_on__gte=month[0], change__submitted_on__lte=month[1]).distinct() # sort results if unit_filter in ["my-suggestions", "user-suggestions"]: sort_on = "suggestions" elif unit_filter in ["my-submissions", "user-submissions"]: sort_on = "submissions" else: sort_on = "units" sort_by = ALLOWED_SORTS[sort_on].get(sort, None) if sort_by is not None: # filtered sort if sort_on in SIMPLY_SORTED: qs = qs.order_by(sort_by, "store__pootle_path", "index") else: max_field, sort_order = get_max_and_order_fields(sort_by) qs = ( qs.annotate(sort_by_field=Max(max_field)) .order_by(sort_order, "store__pootle_path", "index")) # text search if search and sfields: qs = UnitTextSearch(qs).search( search, [sfields], "exact" in soptions) find_unit = ( not offset and language_code and project_code and filename and uids) start = offset total = qs.count() if find_unit: # find the uid in the Store uid_list = list(qs.values_list("pk", flat=True)) unit_index = uid_list.index(uids[0]) start = int(unit_index / (2 * limit)) * (2 * limit) end = min(start + (2 * limit), total) unit_groups = [] units_by_path = groupby( qs.values(*GroupedResults.select_fields)[start:end], lambda x: x["store__pootle_path"]) for pootle_path, units in units_by_path: unit_groups.append( {pootle_path: StoreResults(units).data}) total = qs.count() return total, start, min(end, total), unit_groups
def get_step_query(request, units_queryset): """Narrows down unit query to units matching conditions in GET.""" if 'filter' in request.GET: unit_filter = request.GET['filter'] username = request.GET.get('user', None) modified_since = request.GET.get('modified-since', None) month = request.GET.get('month', None) sort_by_param = request.GET.get('sort', None) sort_on = 'units' user = request.profile if username is not None: User = get_user_model() try: user = User.objects.get(username=username) except User.DoesNotExist: pass if unit_filter: match_queryset = units_queryset.none() if unit_filter == 'all': match_queryset = units_queryset elif unit_filter == 'translated': match_queryset = units_queryset.filter(state=TRANSLATED) elif unit_filter == 'untranslated': match_queryset = units_queryset.filter(state=UNTRANSLATED) elif unit_filter == 'fuzzy': match_queryset = units_queryset.filter(state=FUZZY) elif unit_filter == 'incomplete': match_queryset = units_queryset.filter( Q(state=UNTRANSLATED) | Q(state=FUZZY), ) elif unit_filter == 'suggestions': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING).distinct() elif unit_filter in ('my-suggestions', 'user-suggestions'): match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING, suggestion__user=user, ).distinct() sort_on = 'suggestions' elif unit_filter == 'user-suggestions-accepted': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.ACCEPTED, suggestion__user=user, ).distinct() elif unit_filter == 'user-suggestions-rejected': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.REJECTED, suggestion__user=user, ).distinct() elif unit_filter in ('my-submissions', 'user-submissions'): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).distinct() sort_on = 'submissions' elif (unit_filter in ('my-submissions-overwritten', 'user-submissions-overwritten')): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).exclude(submitted_by=user).distinct() elif unit_filter == 'checks': if 'checks' in request.GET: checks = request.GET['checks'].split(',') if checks: match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__name__in=checks, ).distinct() elif 'category' in request.GET: category_name = request.GET['category'] try: category = get_category_id(category_name) except KeyError: raise Http404 match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__category=category, ).distinct() if modified_since is not None: datetime_obj = parse_datetime(modified_since) if datetime_obj is not None: match_queryset = match_queryset.filter( submitted_on__gt=datetime_obj, ).distinct() if month is not None: [start, end] = get_date_interval(month) match_queryset = match_queryset.filter( submitted_on__gte=start, submitted_on__lte=end, ).distinct() sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None) if sort_by is not None: if sort_on in SIMPLY_SORTED: match_queryset = match_queryset.order_by(sort_by) else: # Omit leading `-` sign if sort_by[0] == '-': max_field = sort_by[1:] sort_order = '-sort_by_field' else: max_field = sort_by sort_order = 'sort_by_field' # It's necessary to use `Max()` here because we can't # use `distinct()` and `order_by()` at the same time # (unless PostreSQL is used and `distinct(field_name)`) match_queryset = match_queryset \ .annotate(sort_by_field=Max(max_field)) \ .order_by(sort_order) units_queryset = match_queryset if 'search' in request.GET and 'sfields' in request.GET: # Accept `sfields` to be a comma-separated string of fields (#46) GET = request.GET.copy() sfields = GET['sfields'] if isinstance(sfields, unicode) and u',' in sfields: GET.setlist('sfields', sfields.split(u',')) # use the search form for validation only search_form = make_search_form(GET) if search_form.is_valid(): units_queryset = get_search_step_query(search_form, units_queryset) return units_queryset
def get_step_query(request, units_queryset): """Narrows down unit query to units matching conditions in GET.""" if 'filter' in request.GET: unit_filter = request.GET['filter'] username = request.GET.get('user', None) modified_since = request.GET.get('modified-since', None) month = request.GET.get('month', None) sort_by_param = request.GET.get('sort', None) sort_on = 'units' user = request.profile if username is not None: User = get_user_model() try: user = User.objects.get(username=username) except User.DoesNotExist: pass if unit_filter: match_queryset = units_queryset.none() if unit_filter == 'all': match_queryset = units_queryset elif unit_filter == 'translated': match_queryset = units_queryset.filter(state=TRANSLATED) elif unit_filter == 'untranslated': match_queryset = units_queryset.filter(state=UNTRANSLATED) elif unit_filter == 'fuzzy': match_queryset = units_queryset.filter(state=FUZZY) elif unit_filter == 'incomplete': match_queryset = units_queryset.filter( Q(state=UNTRANSLATED) | Q(state=FUZZY), ) elif unit_filter == 'suggestions': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING ).distinct() elif unit_filter in ('my-suggestions', 'user-suggestions'): match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING, suggestion__user=user, ).distinct() sort_on = 'suggestions' elif unit_filter == 'user-suggestions-accepted': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.ACCEPTED, suggestion__user=user, ).distinct() elif unit_filter == 'user-suggestions-rejected': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.REJECTED, suggestion__user=user, ).distinct() elif unit_filter in ('my-submissions', 'user-submissions'): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).distinct() sort_on = 'submissions' elif (unit_filter in ('my-submissions-overwritten', 'user-submissions-overwritten')): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).exclude(submitted_by=user).distinct() elif unit_filter == 'checks' and 'checks' in request.GET: checks = request.GET['checks'].split(',') if checks: match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__name__in=checks, ).distinct() if modified_since is not None: datetime_obj = parse_datetime(modified_since) if datetime_obj is not None: match_queryset = match_queryset.filter( submitted_on__gt=datetime_obj, ).distinct() if month is not None: [start, end] = get_date_interval(month) match_queryset = match_queryset.filter( submitted_on__gte=start, submitted_on__lte=end, ).distinct() sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None) if sort_by is not None: if sort_on in SIMPLY_SORTED: if sort_by == 'priority': # TODO: Replace the following extra() with Coalesce # https://docs.djangoproject.com/en/1.8/ref/models/database-functions/#coalesce # once we drop support for Django<1.8.x: # .annotate( # sort_by_field=Coalesce( # Max("vfolders__priority"), # Value(1) # ) # ).order_by("-sort_by_field") match_queryset = match_queryset.extra(select={'sort_by_field': """ SELECT COALESCE(MAX(virtualfolder_virtualfolder.priority), 1) FROM virtualfolder_virtualfolder INNER JOIN virtualfolder_virtualfolder_units ON virtualfolder_virtualfolder.id = virtualfolder_virtualfolder_units.virtualfolder_id WHERE virtualfolder_virtualfolder_units.unit_id = pootle_store_unit.id """}).extra(order_by=['-sort_by_field']) else: match_queryset = match_queryset.order_by(sort_by) else: # Omit leading `-` sign if sort_by[0] == '-': max_field = sort_by[1:] sort_order = '-sort_by_field' else: max_field = sort_by sort_order = 'sort_by_field' # It's necessary to use `Max()` here because we can't # use `distinct()` and `order_by()` at the same time # (unless PostreSQL is used and `distinct(field_name)`) match_queryset = match_queryset \ .annotate(sort_by_field=Max(max_field)) \ .order_by(sort_order) units_queryset = match_queryset if 'search' in request.GET and 'sfields' in request.GET: # Accept `sfields` to be a comma-separated string of fields (#46) GET = request.GET.copy() sfields = GET['sfields'] if isinstance(sfields, unicode) and u',' in sfields: GET.setlist('sfields', sfields.split(u',')) # use the search form for validation only search_form = make_search_form(GET) if search_form.is_valid(): units_queryset = get_search_step_query(search_form, units_queryset) return units_queryset
def calculate_search_results(kwargs, user): pootle_path = kwargs["pootle_path"] category = kwargs.get("category") checks = kwargs.get("checks") initial = kwargs.get("initial", False) limit = kwargs.get("count", 9) modified_since = kwargs.get("modified-since") search = kwargs.get("search") sfields = kwargs.get("sfields") soptions = kwargs.get("soptions", []) sort = kwargs.get("sort", None) uids = [int(x) for x in kwargs.get("uids", "").split(",") if x] unit_filter = kwargs.get("filter") if modified_since: modified_since = parse_datetime(modified_since) vfolder = None if 'virtualfolder' in settings.INSTALLED_APPS: vfolder, pootle_path = extract_vfolder_from_path( pootle_path, vfti=VirtualFolderTreeItem.objects.select_related( "directory", "vfolder")) qs = (Unit.objects.get_translatable( user=user, **resolve(pootle_path).kwargs).filter( store__pootle_path__startswith=pootle_path).order_by( "store", "index")) if vfolder is not None: qs = qs.filter(vfolders=vfolder) # if "filter" is present in request vars... if unit_filter: # filter the results accordingly qs = UnitSearchFilter().filter(qs, unit_filter, user=user, checks=checks, category=get_category_id(category)) # filter by modified if modified_since: qs = qs.filter(submitted_on__gt=modified_since).distinct() # sort results if unit_filter in ["my-suggestions", "user-suggestions"]: sort_on = "suggestions" elif unit_filter in ["my-submissions", "user-submissions"]: sort_on = "submissions" else: sort_on = "units" sort_by = ALLOWED_SORTS[sort_on].get(sort, None) if sort_by is not None: # filtered sort if sort_on in SIMPLY_SORTED: qs = qs.order_by(sort_by, "store__pootle_path", "index") else: if sort_by[0] == '-': max_field = sort_by[1:] sort_order = '-sort_by_field' else: max_field = sort_by sort_order = 'sort_by_field' qs = (qs.annotate(sort_by_field=Max(max_field)).order_by( sort_order, "store__pootle_path", "index")) # text search if search and sfields: qs = UnitTextSearch(qs).search(search, [sfields], "exact" in soptions) begin = 0 end = None uid_list = [] if initial: uid_list = list(qs.values_list("pk", flat=True)) if uids and len(uids) == 1: # if uid is set get the index of the uid index = uid_list.index(uids[0]) begin = max(index - limit, 0) end = min(index + limit + 1, len(uid_list)) elif uids: qs = qs.filter(id__in=uids) if end is None: end = 2 * limit unit_groups = [] units_by_path = groupby( qs.values(*GroupedResults.select_fields)[begin:end], lambda x: x["store__pootle_path"]) for pootle_path, units in units_by_path: unit_groups.append({pootle_path: StoreResults(units).data}) return uid_list, unit_groups
def get_step_query(request, units_queryset): """Narrows down unit query to units matching conditions in GET.""" if 'filter' in request.GET: unit_filter = request.GET['filter'] username = request.GET.get('user', None) modified_since = request.GET.get('modified-since', None) month = request.GET.get('month', None) sort_by_param = request.GET.get('sort', None) sort_on = 'units' user = request.profile if username is not None: User = get_user_model() try: user = User.objects.get(username=username) except User.DoesNotExist: pass if unit_filter: match_queryset = units_queryset.none() if unit_filter == 'all': match_queryset = units_queryset elif unit_filter == 'translated': match_queryset = units_queryset.filter(state=TRANSLATED) elif unit_filter == 'untranslated': match_queryset = units_queryset.filter(state=UNTRANSLATED) elif unit_filter == 'fuzzy': match_queryset = units_queryset.filter(state=FUZZY) elif unit_filter == 'incomplete': match_queryset = units_queryset.filter( Q(state=UNTRANSLATED) | Q(state=FUZZY), ) elif unit_filter == 'suggestions': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING).distinct() elif unit_filter in ('my-suggestions', 'user-suggestions'): match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.PENDING, suggestion__user=user, ).distinct() sort_on = 'suggestions' elif unit_filter == 'user-suggestions-accepted': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.ACCEPTED, suggestion__user=user, ).distinct() elif unit_filter == 'user-suggestions-rejected': match_queryset = units_queryset.filter( suggestion__state=SuggestionStates.REJECTED, suggestion__user=user, ).distinct() elif unit_filter in ('my-submissions', 'user-submissions'): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).distinct() sort_on = 'submissions' elif (unit_filter in ('my-submissions-overwritten', 'user-submissions-overwritten')): match_queryset = units_queryset.filter( submission__submitter=user, submission__type__in=SubmissionTypes.EDIT_TYPES, ).exclude(submitted_by=user).distinct() elif unit_filter == 'checks' and 'checks' in request.GET: checks = request.GET['checks'].split(',') if checks: match_queryset = units_queryset.filter( qualitycheck__false_positive=False, qualitycheck__name__in=checks, ).distinct() if modified_since is not None: datetime_obj = parse_datetime(modified_since) if datetime_obj is not None: match_queryset = match_queryset.filter( submitted_on__gt=datetime_obj, ).distinct() if month is not None: [start, end] = get_date_interval(month) match_queryset = match_queryset.filter( submitted_on__gte=start, submitted_on__lte=end, ).distinct() sort_by = ALLOWED_SORTS[sort_on].get(sort_by_param, None) if sort_by is not None: if sort_on in SIMPLY_SORTED: match_queryset = match_queryset.order_by(sort_by) else: # Omit leading `-` sign if sort_by[0] == '-': max_field = sort_by[1:] sort_order = '-sort_by_field' else: max_field = sort_by sort_order = 'sort_by_field' # It's necessary to use `Max()` here because we can't # use `distinct()` and `order_by()` at the same time # (unless PostreSQL is used and `distinct(field_name)`) match_queryset = match_queryset \ .annotate(sort_by_field=Max(max_field)) \ .order_by(sort_order) units_queryset = match_queryset if 'search' in request.GET and 'sfields' in request.GET: # Accept `sfields` to be a comma-separated string of fields (#46) GET = request.GET.copy() sfields = GET['sfields'] if isinstance(sfields, unicode) and u',' in sfields: GET.setlist('sfields', sfields.split(u',')) # use the search form for validation only search_form = make_search_form(GET) if search_form.is_valid(): units_queryset = get_search_step_query(search_form, units_queryset) return units_queryset