def get_start_date(lessons: QuerySet):
"""Returns the date which should be displayed as first in calendar as string"""
lessons.order_by('start_time')
start_date = datetime.datetime.now()
for lesson in lessons:
if lesson.start_time > start_date:
start_date = lesson.start_time
return start_date.isoformat(timespec='seconds')
return start_date.isoformat(timespec='seconds')
def sort_queryset(query_set: QuerySet,
sort_multi: Iterable = None,
sort_field: str = None,
sort_order: Union[str, int, bool] = True,
default_sorting: Union[str, Iterable] = None,
sort_only_fields: Iterable = None,
check_filed_before_sort: bool = False,
**kwarg: dict) -> QuerySet:
"""
sort QuerySet
Сортировка набора данных с указанными параметрамиб с поддержкой сортировки по одному полю заданному sort_field или
множеству полей sort_multi (указанных списком вловарей со ключами field и order
Порядок сортировки задается значением 1 и -1 (по возрастснию и по убыванию соотвественно)
:rtype: QuerySet results with sorting
:param check_filed_before_sort: flag that indicate test sorted fields name in sort_only_fields list
(for sorting only specified fields)
:param default_sorting: if no sorting (sort_field or sort_multi) specified use default_sorting
:param sort_field: field for sorting (no multiply sorting)
:param sort_multi: complex multi sorting
(list of dict {
filed - field name,
order - "1" asc
"-1" desc
}
)
:param query_set: QuerySet for apply sorting operation
:param sort_order: order value 1 asc, value -1 is desc
:type sort_only_fields: object
"""
if query_set and (isinstance(query_set, QuerySet)
or hasattr(query_set, 'order_by')):
if sort_field:
if not check_filed_before_sort or sort_field in sort_only_fields:
sort_multi = [{'field': sort_field, 'order': sort_order}]
if sort_multi:
sorting = [
f"{sort_info['field']}" if sort_info['order'] == 1
or sort_info['order'] == '1' else f"-{sort_info['field']}"
for sort_info in sort_multi if not check_filed_before_sort
or sort_info['field'] in sort_only_fields
]
query_set = query_set.order_by(*sorting)
elif default_sorting:
if isinstance(default_sorting, str):
query_set = query_set.order_by(default_sorting)
elif isinstance(default_sorting, Iterable):
query_set = query_set.order_by(*default_sorting)
return query_set
def create_from_questions(cls,
user: User,
questions: QuerySet,
max_n=20,
randomise_order=True,
include_answered=False,
save=True) -> "QuizSession":
cls.check_no_current_session(user=user)
if include_answered is False:
questions = questions.exclude(records__in=Record.objects.filter(
user=user))
if randomise_order is True:
questions = questions.order_by("?")
max_n = max_n if max_n <= questions.count() else questions.count()
selected_qs = questions[0:max_n + 1]
if selected_qs.count() < 1:
raise RuntimeError(
"Cannot create Quiz Session without any questions!")
session = cls(user=user)
session.save()
session.questions.set(selected_qs)
session.save()
return session
def _sort_metadata(self, all_md: QuerySet):
""" Perform sorting using sortBy parameter
Values follow the syntax of `attribute:A|D`. The A or D indicates ASC or DESC ordering. Since the
attribute can contain `:` as well, the A|D part has to be separated without the attribute to be damaged.
Args:
all_md (QuerySet): The unsorted metadata queryset
Returns:
all_md (QuerySet): The sorted metadata queryset
"""
if self.param.sort_by is None:
return all_md
sort_by_components = self.param.sort_by.split(":")
attrib = ":".join(sort_by_components[:-1])
md_attrib = self.attribute_map.get(attrib, None)
if md_attrib is None:
raise NotImplementedError(
"{} not supported for sorting".format(attrib),
"sortBy"
)
desc = sort_by_components[-1] == "D"
if desc:
md_attrib = "-" + md_attrib
return all_md.order_by(md_attrib)
def generic_sort(query_set: QuerySet, request):
"""
Sorts by field given in request query parameter `sort_by`
with order given as `order`.
In case of missing order default order is asc.
When `sort_by` contains not existing field, sorting is ignored and
original query_set is returned
:param query_set: QuerySet to be sorted
:param request: GET request
:return: sorted query_set
"""
sort_by = request.query_params.get('sort_by', None)
order = request.query_params.get('order', None)
model = query_set.model
try:
if sort_by and hasattr(model, sort_by) and order in ['desc', 'asc']:
if order == "desc":
condition = F(sort_by).desc()
else:
condition = F(sort_by).asc()
return query_set.order_by(condition)
else:
return query_set
except FieldError:
return query_set
def process(self, request_fields: QueryDict,
queryset: QuerySet) -> QuerySet:
if request_fields.get('ordering'):
if request_fields.get('ordering') in ('desc', 'asc'):
self.ordering = request_fields.get('ordering')
to_order = self.rule_body.split(',')
for field in to_order:
key = field
if self.ordering == 'desc':
key = '-' + key
try:
list(queryset.order_by(key))
queryset = queryset.order_by(key)
except FieldError:
return queryset
return queryset
def queryset_iterator(queryset: QuerySet, chunksize: int=5000) -> Iterator[Any]:
queryset = queryset.order_by('id')
while queryset.exists():
for row in queryset[:chunksize]:
msg_id = row.id
yield row
queryset = queryset.filter(id__gt=msg_id)
def get_most_sold_products(self):
products = QuerySet(Product)
for subcategory in self.subcategories.all():
products = (products | subcategory.get_most_sold_products())
return products.order_by('-sold')[:5]
def paginate_queryset(
self,
queryset: QuerySet,
request: Request,
view: APIView = None,
) -> OptionalList:
serializer_class = self.get_request_serializer_class()
serializer = serializer_class(data=request.query_params,
max_per_page=self.max_page_size)
serializer.is_valid(raise_exception=True)
self.per_page = serializer.validated_data.get(
self.page_size_query_param, self.default_page_size)
after_uuid = serializer.validated_data.get(self.after_query_param)
before_uuid = serializer.validated_data.get(self.before_query_param)
order = get_order(self.default_order, serializer.validated_data.keys())
self.cursor_obj, self.order = get_cursor(queryset, after_uuid,
before_uuid, order)
lookup = self.get_lookup()
if lookup:
queryset = queryset.filter(**lookup)
order_sign = '-' if self.order == 'before' else ''
order_keyword = f'{order_sign}{self.order_by_field}'
queryset = queryset.order_by(order_keyword)
base_url = request.build_absolute_uri()
self.base_url = replace_query_param(base_url,
self.page_size_query_param,
self.per_page)
self.request = request
self.has_next = False
start_offset = 0
stop_offset = start_offset + self.per_page + 1
if self.use_count:
self.total = queryset.count()
paginated = list(queryset[:stop_offset])
if len(paginated) > self.per_page:
self.has_next = True
del paginated[-1]
if paginated:
self.page_boundaries = (
paginated[0].uuid,
paginated[-1].uuid,
)
elif self.cursor_obj:
self.page_boundaries = (self.cursor_obj.uuid, self.cursor_obj.uuid)
else:
self.page_boundaries = (None, None)
return paginated
def sort_by(queryset: QuerySet, request: Request) -> QuerySet:
sort_by = request.query_params.get('sort_by')
if sort_by is not None:
descending = request.query_params.get('descending',
'false') == 'true'
if sort_by == 'status':
queryset = queryset.order_by(
('-' if descending else '') + 'status_external',
('-' if descending else '') + 'status_internal',
)
else:
queryset = queryset.order_by(('-' if descending else '') +
sort_by)
return queryset
def queryset_iterator(queryset: QuerySet,
chunksize: int = 5000) -> Iterator[Any]:
queryset = queryset.order_by("id")
while queryset.exists():
for row in queryset[:chunksize]:
msg_id = row.id
yield row
queryset = queryset.filter(id__gt=msg_id)
def calc_net(queryset: QuerySet):
total = timedelta()
queryset = queryset.order_by("-date")
for i in range(queryset.count()):
if queryset[i].nature == 1 and i > 0:
total += queryset[i - 1].date - queryset[i].date
return format_delta(total)
def effort_user_timeline_chart_data(queryset: QuerySet) -> Dict:
queryset_values = queryset.order_by("date").values(
"date", "user__acronym").annotate(total_effort=Sum("effort"))
items_by_user = pydash.group_by(queryset_values, "user__acronym")
return {
user__acronym: items_by_user[user__acronym]
for user__acronym in items_by_user.keys()
}
def apply_order(queryset: QuerySet, field: str, asc: bool = False) -> QuerySet:
orderable_fields = ORDERABLE_FIELDS[queryset.model.__name__]
if field in orderable_fields:
order_by = field
if not asc:
order_by = '-%s' % order_by
queryset = queryset.order_by(order_by)
return queryset
def sort_by(queryset: QuerySet, request: Request) -> QuerySet:
sort_by = request.query_params.get('sort_by')
if sort_by is not None:
descending = request.query_params.get('descending',
'false') == 'true'
queryset = queryset.order_by(('-' if descending else '') + sort_by)
return queryset
def calculate_sessions(self, events: QuerySet, session_type: Optional[str],
date_filter: Dict[str, datetime], team: Team,
request: request.Request) -> List[Dict[str, Any]]:
if not events:
return []
# format date filter for session view
_date_gte = Q()
if session_type is None:
if request.GET.get('date_from', None):
_date_gte = Q(timestamp__gte=date_filter['timestamp__gte'],
timestamp__lte=date_filter['timestamp__gte'] +
relativedelta(days=1))
else:
dt = events.order_by('-timestamp').values(
'timestamp')[0]['timestamp']
if dt:
dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)
_date_gte = Q(timestamp__gte=dt,
timestamp__lte=dt + relativedelta(days=1))
sessions = events.filter(_date_gte)\
.annotate(previous_timestamp=Window(
expression=Lag('timestamp', default=None),
partition_by=F('distinct_id'),
order_by=F('timestamp').asc()
))\
.annotate(previous_event=Window(
expression=Lag('event', default=None),
partition_by=F('distinct_id'),
order_by=F('timestamp').asc()
))
sessions_sql, sessions_sql_params = sessions.query.sql_with_params()
all_sessions = '\
SELECT *,\
SUM(new_session) OVER (ORDER BY distinct_id, timestamp) AS global_session_id,\
SUM(new_session) OVER (PARTITION BY distinct_id ORDER BY timestamp) AS user_session_id\
FROM (SELECT id, distinct_id, event, elements_hash, timestamp, properties, CASE WHEN EXTRACT(\'EPOCH\' FROM (timestamp - previous_timestamp)) >= (60 * 30)\
OR previous_timestamp IS NULL \
THEN 1 ELSE 0 END AS new_session \
FROM ({}) AS inner_sessions\
) AS outer_sessions'.format(sessions_sql)
result: List = []
if session_type == 'avg':
result = self._session_avg(all_sessions, sessions_sql_params,
date_filter)
elif session_type == 'dist':
result = self._session_dist(all_sessions, sessions_sql_params)
else:
result = self._session_list(all_sessions, sessions_sql_params,
team, date_filter, request)
return result
def extract_recap_documents(
docs: QuerySet,
skip_ocr: bool = False,
order_by: Optional[str] = None,
queue: Optional[str] = None,
) -> None:
"""Loop over RECAPDocuments and extract their contents. Use OCR if requested.
:param docs: A queryset containing the RECAPDocuments to be processed.
:type docs: Django Queryset
:param skip_ocr: Whether OCR should be completed (False) or whether items
should simply be updated to have status OCR_NEEDED.
:type skip_ocr: Bool
:param order_by: An optimization parameter. You may opt to order the
processing by 'small-first' or 'big-first'.
:type order_by: str
:param queue: The celery queue to send the content to.
:type queue: str
"""
docs = docs.exclude(filepath_local="")
if skip_ocr:
# Focus on the items that we don't know if they need OCR.
docs = docs.filter(ocr_status=None)
else:
# We're doing OCR. Only work with those items that require it.
docs = docs.filter(ocr_status=RECAPDocument.OCR_NEEDED)
if order_by is not None:
if order_by == "small-first":
docs = docs.order_by("page_count")
elif order_by == "big-first":
docs = docs.order_by("-page_count")
count = docs.count()
throttle = CeleryThrottle(queue_name=queue)
for i, pk in enumerate(docs.values_list("pk", flat=True)):
throttle.maybe_wait()
extract_recap_pdf.apply_async((pk, skip_ocr), priority=5, queue=queue)
if i % 1000 == 0:
msg = f"Sent {i + 1}/{count} tasks to celery so far."
logger.info(msg)
sys.stdout.write(f"\r{msg}")
sys.stdout.flush()
def sort_queryset(
queryset: QuerySet,
sort_by: SortInputObjectType,
reversed: bool,
channel_slug: Optional[str],
) -> QuerySet:
"""Sort queryset according to given parameters.
rules:
- sorting_field and sorting_attribute cannot be together)
- when sorting_attribute is passed, it is expected that
queryset will have method to sort by attributes
- when sorter has custom sorting method it's name must be like
`prepare_qs_for_sort_{enum_name}` and it must return sorted queryset
Keyword Arguments:
queryset - queryset to be sorted
sort_by - dictionary with sorting field and direction
"""
sorting_direction = sort_by.direction
if reversed:
sorting_direction = REVERSED_DIRECTION[sorting_direction]
sorting_field = sort_by.field
sorting_attribute = getattr(sort_by, "attribute_id", None)
if sorting_field is not None and sorting_attribute is not None:
raise GraphQLError(
"You must provide either `field` or `attributeId` to sort the products."
)
elif sorting_attribute is not None: # empty string as sorting_attribute is valid
return _sort_queryset_by_attribute(queryset, sorting_attribute,
sorting_direction)
sort_enum = sort_by._meta.sort_enum
sorting_fields = sort_enum.get(sorting_field)
sorting_field_name = sorting_fields.name.lower()
custom_sort_by = getattr(sort_enum, f"qs_with_{sorting_field_name}", None)
if custom_sort_by:
queryset = custom_sort_by(queryset, channel_slug=channel_slug)
if sorting_field_name == "rank":
# In rank sorting ID is sorted in opposite direction to rank
if sorting_direction == "-":
sorting_list = ["-rank", "id"]
else:
sorting_list = ["rank", "-id"]
else:
sorting_field_value = sorting_fields.value
sorting_list = [
f"{sorting_direction}{field}" for field in sorting_field_value
]
return queryset.order_by(*sorting_list)
def order_by(self, qs: QuerySet, field_resolver: Callable[[str], str] = None) -> QuerySet:
if self.document_sort_order:
field_resolver = field_resolver if field_resolver else DocumentFilter.default_field_resolver
expressions = self.document_sort_order.strip().split(':')
reverse_order = {'DESC': False, 'ASC': True}.get(expressions[1]) if len(expressions) > 1 else True
if reverse_order is None:
raise SyntaxError('Unknown operator "{0}"'.format(expressions[1]))
sort_order_field = field_resolver(expressions[0])
sort_order_field = sort_order_field if not reverse_order else "-{0}".format(sort_order_field)
qs = qs.order_by(sort_order_field)
return qs
def _get_count(qs: djdm.QuerySet, attr) -> int:
attrs = ['organization', 'department', 'building', 'room']
selectors = [
'seat__room__organization', 'seat__room__department',
'seat__room__building', 'seat__room__room'
]
pos = attrs.index(attr)
count = qs.order_by(*selectors[:pos + 1]).distinct(*selectors[:pos +
1]).count()
# print("get_count", attr, selectors[:pos+1])
return count
def getLastCompetitions(length=6):
result_query = QuerySet(Competition)
for region in Competition.REGIONS:
result_query = result_query | Competition.objects\
.filter(is_creating_finished=True)\
.filter(started_at__gt=django.utils.timezone.now()) \
.filter(region=region[0]) \
.order_by('-started_at') \
.all()[:length]
return result_query.order_by('-started_at')
def dashboard(request):
documents = request.user.document_owner.get_queryset() | request.user.document_set.get_queryset()
documents = documents.distinct().order_by('-last_modified')[:5]
notifications = QuerySet(model=models.ChangeNotification)
for document in documents:
notifications |= document.changenotification_set.get_queryset()[:5]
return render(request, 'dashboard/dashboard.html', {
'documents': documents,
'notifications': notifications.order_by('-modify_time').all()
})
def sort_by_payment(queryset: QuerySet, sort_by: SortInputObjectType) -> QuerySet:
last_payments = (
queryset.exclude(payments__isnull=True)
.annotate(payment_id=Max("payments__pk"))
.values_list("payment_id", flat=True)
)
queryset = queryset.annotate(
last_payment=FilteredRelation(
"payments", condition=Q(payments__pk__in=last_payments)
)
)
return queryset.order_by(f"{sort_by.direction}last_payment__charge_status")
def add_reference_to_field_on_related_model(cls, qs: QuerySet, **kwargs):
cls.assert_is_proxy(qs)
modified_qs = qs.all()
if qs.order_by('related_model_name').distinct('related_model_name').count() == 1:
related_model_name = qs.first().related_model_name
modified_qs = modified_qs.annotate(**{
key: F('{}__{}'.format(related_model_name, value))
for key, value in kwargs.items()
})
return modified_qs
def _get_sorting_annotation(queryset: models.QuerySet, field_name: str,
annotations: dict) -> models.Case:
queryset = queryset.values(field_name)
queryset = queryset.annotate(**annotations)
queryset = queryset.order_by('value')
return models.Case(*[
models.When(**{
"{}".format(field_name): group[field_name],
"then": index
}) for index, group in enumerate(queryset)
],
output_field=models.IntegerField())
def dashboard(request):
documents = request.user.document_owner.get_queryset(
) | request.user.document_set.get_queryset()
documents = documents.distinct().order_by('-last_modified')[:5]
notifications = QuerySet(model=models.ChangeNotification)
for document in documents:
notifications |= document.changenotification_set.get_queryset()[:5]
return render(
request, 'dashboard/dashboard.html', {
'documents': documents,
'notifications': notifications.order_by('-modify_time').all()
})
def _daily_count(subjects_queryset: QuerySet):
dates = []
men_count = []
women_count = []
if subjects_queryset.count():
subjects_queryset = subjects_queryset.order_by('faces__created_at')
oldest_subject = subjects_queryset.first()
newest_subject = subjects_queryset.last()
lower_timestamp = min(
[face.created_at for face in oldest_subject.faces.all()])
upper_timestamp = max(
[face.created_at for face in newest_subject.faces.all()])
lower_date = timezone.localtime(lower_timestamp).date()
upper_date = timezone.localtime(upper_timestamp).date()
curr_date = lower_date
ind = 0
while curr_date <= upper_date:
next_date = curr_date + datetime.timedelta(days=1)
subjects = subjects_queryset.filter(
faces__created_at__date__gte=curr_date,
faces__created_at__date__lt=next_date)
men_count.append(
len(
set([
subject.id for subject in subjects
if subject.pred_sex == Subject.SEX_MAN
])))
women_count.append(
len(
set([
subject.id for subject in subjects
if subject.pred_sex == Subject.SEX_WOMAN
])))
dates.append(str(curr_date))
curr_date = next_date
ind += 1
return {
'dates': dates,
'men_count': men_count,
'women_count': women_count,
}
def add_reference_to_field_on_related_model(cls, qs: QuerySet, **kwargs):
cls.assert_is_proxy(qs)
modified_qs = qs.all()
if qs.order_by('related_model_name').distinct(
'related_model_name').count() == 1:
related_model_name = qs.first().related_model_name
modified_qs = modified_qs.annotate(
**{
key: F('{}__{}'.format(related_model_name, value))
for key, value in kwargs.items()
})
return modified_qs
def get_item_version(queryset: models.QuerySet,
suitable_version: str = None) -> ContentStudyItem:
if suitable_version is None:
return queryset.order_by('-id').first()
current_version = None
for item in queryset:
if compare(item.minimal_plugin_version, suitable_version) <= 0:
if current_version is None:
current_version = item
elif compare(current_version.minimal_plugin_version,
item.minimal_plugin_version) < 0:
current_version = item
return current_version
def sort_queryset_by_default(
queryset: QuerySet, reversed: bool
) -> Tuple[QuerySet, dict]:
"""Sort queryset by it's default ordering."""
queryset_model = queryset.model
default_ordering = ["pk"]
if queryset_model and queryset_model._meta.ordering:
default_ordering = get_model_default_ordering(queryset_model)
ordering_fields = [field.replace("-", "") for field in default_ordering]
direction = "-" if "-" in default_ordering[0] else ""
if reversed:
reversed_direction = REVERSED_DIRECTION[direction]
default_ordering = [f"{reversed_direction}{field}" for field in ordering_fields]
order_by = {"field": ordering_fields, "direction": direction}
return queryset.order_by(*default_ordering), order_by
def build_queryset(self, queryset: models.QuerySet) -> models.QuerySet:
if self.query:
queryset = self.query.apply_to_queryset(queryset)
if self.order_by:
queryset = queryset.order_by(*self.order_by)
if self.distinct:
queryset = queryset.distinct(*self.distinct)
if self.prefetch_trees:
select_related = []
for prefetch_tree in self.prefetch_trees:
tree_select_related = self._flatten_prefetch_tree(
prefetch_tree=prefetch_tree)
if tree_select_related is not None:
select_related += tree_select_related
queryset = queryset.select_related(
*[sr for sr in select_related if sr is not None])
return queryset
def filter_list(field: str, model: Type[Model] = None,
queryset: QuerySet = None, unique=False):
"""
Generate a list with field values to be used with filter lists
:param field: Field name
:param model: Model object
:param queryset: Queryset object
:param unique: Return only unique records
:return: list with values
"""
if model:
queryset = model.objects.all()
if unique:
queryset = queryset.order_by(field).distinct()
values = queryset.values_list(field, flat=True)
return list(values)
def sort_queryset(queryset: QuerySet, sort_by: SortInputObjectType,
sort_enum: graphene.Enum) -> QuerySet:
"""Sort queryset according to given parameters.
Keyword Arguments:
queryset - queryset to be filtered
sort_by - dictionary with sorting field and direction
"""
if sort_by is None or not sort_by.field:
return queryset
direction = sort_by.direction
sorting_field = sort_by.field
custom_sort_by = getattr(sort_enum, f"sort_by_{sorting_field}", None)
if custom_sort_by:
return custom_sort_by(queryset, sort_by)
return queryset.order_by(f"{direction}{sorting_field}")
def __init__(self, blast_type: str, name: str, sequence: str,
gene_codes: QuerySet) -> None:
"""
:param blast_type: new
:param name: name given to query sequence
:param sequence: sequence to blast given by user
:param gene_codes: queryset of Genes to blast against
"""
super(BLASTNew, self).__init__(blast_type, name, sequence, gene_codes)
self.name = name
self.sequence = sequence
log.debug("Will do blastnew of %s", str(gene_codes))
if gene_codes:
self.genes = gene_codes.order_by(
"gene_code").values_list("gene_code", flat=True)
else:
self.genes = []
self.path = os.path.join(self.cwd, 'db', '_'.join(self.genes) + '_seqs.fas.n*')
self.db = os.path.join(self.cwd, 'db', '_'.join(self.genes) + '_seqs.fas')
def assert_is_proxy(qs: QuerySet):
"""Is it proxy for only one related model?"""
count = qs.order_by('related_model_name').distinct('related_model_name').count()
assert count <= 1, 'You should split your model pages by proxy, before register it.'
