Ejemplo n.º 1
0
    def create_from_questions(cls,
                              user: User,
                              questions: QuerySet,
                              max_n=20,
                              randomise_order=True,
                              include_answered=False,
                              save=True) -> "QuizSession":
        cls.check_no_current_session(user=user)

        if include_answered is False:
            questions = questions.exclude(records__in=Record.objects.filter(
                user=user))

        if randomise_order is True:
            questions = questions.order_by("?")

        max_n = max_n if max_n <= questions.count() else questions.count()
        selected_qs = questions[0:max_n + 1]

        if selected_qs.count() < 1:
            raise RuntimeError(
                "Cannot create Quiz Session without any questions!")

        session = cls(user=user)
        session.save()
        session.questions.set(selected_qs)
        session.save()
        return session
Ejemplo n.º 2
0
def summarize_invoice_statistics(modeladmin, request: HttpRequest,
                                 qs: QuerySet):
    invoice_states = list([state for state, name in INVOICE_STATE])

    invoiced_total = {'amount': Decimal('0.00'), 'count': 0}
    unpaid_total = {'amount': Decimal('0.00'), 'count': 0}

    lines = [
        '<pre>',
        _('({total_count} invoices)').format(total_count=qs.count()),
    ]
    for state in invoice_states:
        state_name = choices_label(INVOICE_STATE, state)
        qs2 = qs.filter(state=state)

        invoiced = qs2.filter(state=state).aggregate(amount=Coalesce(
            Sum('amount'), 0),
                                                     count=Count('*'))

        lines.append('{state_name} | x{count} | {amount:.2f}'.format(
            label=_('invoiced'), state_name=state_name, **invoiced))

        for k in ('amount', 'count'):
            invoiced_total[k] += invoiced[k]

    lines.append(
        _('Total') + ' {label} | x{count} | {amount:.2f}'.format(
            label=_('amount'), **invoiced_total))
    lines.append('</pre>')

    lines = align_lines(lines, '|')
    messages.add_message(request,
                         INFO,
                         format_html('<br>'.join(lines)),
                         extra_tags='safe')
Ejemplo n.º 3
0
def check_amount_of_members_ow4_g_suite(
    g_suite_members: List[Dict[str, str]], ow4_users: QuerySet, quiet: bool = False
) -> bool:
    """
    Compare the number of users in an OW4 group to a G Suite group.
    :param g_suite_members: The members of a G Suite group.
    :param ow4_users: The members of an OW4 group.
    :param quiet: Whether to print debug log lines or not.
    :return: Whether or not there are exactly the same amount of users in the two groups.
    """
    g_suite_count = len(g_suite_members)
    ow4_count = ow4_users.count()
    if ow4_count == g_suite_count:
        return True
    elif g_suite_count > ow4_count:
        if not quiet:
            logger.debug(
                f"There are more users in G Suite ({g_suite_count}) than on OW4 ({ow4_count}). "
                "Need to trim inactive users from G Suite."
            )
    else:
        if not quiet:
            logger.debug(
                f"There are more users on OW4 ({ow4_count}) than in G Suite ({g_suite_count}). "
                "Need to update G Suite with new members."
            )
    return False
Ejemplo n.º 4
0
def _hourly_count(subjects_queryset: QuerySet):

    hours = [str(datetime.time(hour=hour)) for hour in range(0, 24)]
    men_count = []
    women_count = []

    if subjects_queryset.count():
        for ind in range(len(hours) - 1):
            hour_lower = hours[ind]
            hour_upper = hours[ind + 1]
            subjects = subjects_queryset.filter(
                faces__created_at__time__gte=hour_lower,
                faces__created_at__time__lt=hour_upper)
            men_count.append(
                len(
                    set([
                        subject.id for subject in subjects
                        if subject.pred_sex == Subject.SEX_MAN
                    ])))
            women_count.append(
                len(
                    set([
                        subject.id for subject in subjects
                        if subject.pred_sex == Subject.SEX_WOMAN
                    ])))

    return {
        'hours': hours,
        'men_count': men_count,
        'women_count': women_count,
    }
Ejemplo n.º 5
0
def deactivate(modeladmin: admin.ModelAdmin, request: WSGIRequest,
               queryset: QuerySet):
    queryset.update(active=False)
    model = queryset.model
    model.objects.disconnect(model.signal, model.receiver, model)
    modeladmin.message_user(request,
                            _('Total deactivated: %s' % queryset.count()))
Ejemplo n.º 6
0
    def paginate_queryset(
        self,
        queryset: QuerySet,
        request: Request,
        view: APIView = None,
    ) -> OptionalList:
        serializer_class = self.get_request_serializer_class()
        serializer = serializer_class(data=request.query_params,
                                      max_per_page=self.max_page_size)
        serializer.is_valid(raise_exception=True)

        self.per_page = serializer.validated_data.get(
            self.page_size_query_param, self.default_page_size)

        after_uuid = serializer.validated_data.get(self.after_query_param)
        before_uuid = serializer.validated_data.get(self.before_query_param)

        order = get_order(self.default_order, serializer.validated_data.keys())
        self.cursor_obj, self.order = get_cursor(queryset, after_uuid,
                                                 before_uuid, order)

        lookup = self.get_lookup()
        if lookup:
            queryset = queryset.filter(**lookup)

        order_sign = '-' if self.order == 'before' else ''
        order_keyword = f'{order_sign}{self.order_by_field}'
        queryset = queryset.order_by(order_keyword)

        base_url = request.build_absolute_uri()
        self.base_url = replace_query_param(base_url,
                                            self.page_size_query_param,
                                            self.per_page)
        self.request = request
        self.has_next = False

        start_offset = 0
        stop_offset = start_offset + self.per_page + 1

        if self.use_count:
            self.total = queryset.count()

        paginated = list(queryset[:stop_offset])

        if len(paginated) > self.per_page:
            self.has_next = True
            del paginated[-1]

        if paginated:
            self.page_boundaries = (
                paginated[0].uuid,
                paginated[-1].uuid,
            )
        elif self.cursor_obj:
            self.page_boundaries = (self.cursor_obj.uuid, self.cursor_obj.uuid)
        else:
            self.page_boundaries = (None, None)

        return paginated
Ejemplo n.º 7
0
def calc_net(queryset: QuerySet):
    total = timedelta()
    queryset = queryset.order_by("-date")
    for i in range(queryset.count()):
        if queryset[i].nature == 1 and i > 0:
            total += queryset[i - 1].date - queryset[i].date

    return format_delta(total)
Ejemplo n.º 8
0
 def saveTextSentiment(sentimentable_classes: models.QuerySet):
     sentiment_comprehender = comprehender()
     sentimentable_list = list(sentimentable_classes)
     text_list = [None] * sentimentable_classes.count()
     processed_sentiment_list = []
     processed_index = 0
     for i in range(sentimentable_classes.count()):
         text_list[i] = sentimentable.__shrinkLongText(
             sentimentable_list[i].text)
     text_list = list(sentimentable.__chunk(text_list=text_list, n=25))
     for i in range(len(text_list)):
         sentiment_list = sentiment_comprehender.comprehendText(
             text=text_list[i])
         for j in range(len(sentiment_list['ResultList'])):
             processed_sentiment_list.append(
                 sentiment_list['ResultList'][j])
     for i in range(sentimentable_classes.count()):
         try:
             sentimentable_classes[i].nlp_processed = True
             sentimentable_list[
                 i].nlp_neutral_sentiment = processed_sentiment_list[i][
                     'SentimentScore']['Neutral']
             sentimentable_list[
                 i].nlp_mixed_sentiment = processed_sentiment_list[i][
                     'SentimentScore']['Mixed']
             sentimentable_list[
                 i].nlp_negative_sentiment = processed_sentiment_list[i][
                     'SentimentScore']['Negative']
             sentimentable_list[
                 i].nlp_positive_sentiment = processed_sentiment_list[i][
                     'SentimentScore']['Positive']
             if sentimentable_list[
                     i].nlp_neutral_sentiment == None or sentimentable_list[
                         i].nlp_mixed_sentiment == None or sentimentable_list[
                             i].nlp_negative_sentiment == None or sentimentable_list[
                                 i].nlp_positive_sentiment == None:
                 sentimentable_list[i].delete()
             else:
                 sentimentable_list[i].save()
             processed_index += 1
         except Exception as ex:
             logging.error(str(ex))
             print(str(ex))
             sentimentable_classes[processed_index].delete()
             processed_index += 1
Ejemplo n.º 9
0
def _make_stats(queryset: QuerySet) -> Any:
    """ Generates stats for the frontend to render """

    # The return value of this function should be kept in sync with
    # the 'PortalStats' interface in assets/src/registry/stats/_view.d.ts.

    dicts = {k: _compute_stats(queryset, v) for (k, v) in stat_dicts.items()}

    return {'total': queryset.count(), **dicts}
Ejemplo n.º 10
0
 def unarchive_selected_bookmarks(self, request, queryset: QuerySet):
     for bookmark in queryset:
         unarchive_bookmark(bookmark)
     bookmarks_count = queryset.count()
     self.message_user(request, ngettext(
         '%d bookmark was successfully unarchived.',
         '%d bookmarks were successfully unarchived.',
         bookmarks_count,
     ) % bookmarks_count, messages.SUCCESS)
Ejemplo n.º 11
0
def create_matches_wanted_galleries_from_providers_internal(
        wanted_galleries: QuerySet,
        provider_filter: str = '', cutoff: float = 0.4, max_matches: int = 20,
        must_be_used: bool = False
) -> None:

    try:
        galleries_title_id = []

        if provider_filter:
            galleries = Gallery.objects.eligible_for_use(provider__contains=provider_filter)
        else:
            galleries = Gallery.objects.eligible_for_use()

        if must_be_used:
            galleries = galleries.filter(
                Q(archive__isnull=False)
                | (Q(gallery_container__isnull=False) & Q(gallery_container__archive__isnull=False))
                | (Q(magazine__isnull=False) & Q(magazine__archive__isnull=False))
            )

        for gallery in galleries:
            if gallery.title:
                galleries_title_id.append(
                    (clean_title(gallery.title), gallery.pk))
            if gallery.title_jpn:
                galleries_title_id.append(
                    (clean_title(gallery.title_jpn), gallery.pk))

        logger.info(
            "Trying to match against gallery database, "
            "{} wanted galleries with no match. Provider filter: {}".format(
                wanted_galleries.count(),
                provider_filter
            )
        )
        for wanted_gallery in wanted_galleries:

            similar_list = get_list_closer_gallery_titles_from_list(
                wanted_gallery.search_title, galleries_title_id, cutoff, max_matches)

            if similar_list is not None:

                logger.info("Found {} matches from title for {}".format(len(similar_list), wanted_gallery.search_title))
                for similar in similar_list:
                    # We filter here instead of the Gallery model, because we use the same list for every WG.
                    if not FoundGallery.objects.filter(wanted_gallery=wanted_gallery, gallery_id=similar[1]):
                        GalleryMatch.objects.get_or_create(
                            wanted_gallery=wanted_gallery,
                            gallery_id=similar[1],
                            defaults={'match_accuracy': similar[2]})

        logger.info("Matching ended")
        return
    except BaseException:
        logger.critical(traceback.format_exc())
Ejemplo n.º 12
0
def swap(model_admin: admin.ModelAdmin, request, queryset: QuerySet):
    if queryset.count() != 2:
        model_admin.message_user(request, _('Swapped can be only directly two matches.'), level=messages.ERROR)
        return

    first, second = queryset  # type: Match, Match
    first.match_term, second.match_term = second.match_term, first.match_term
    first.save(update_fields=['match_term', ])
    second.save(update_fields=['match_term', ])
    model_admin.message_user(request, _('Successfully swapped.'), level=messages.SUCCESS)
Ejemplo n.º 13
0
def paginate_queryset(queryset: QuerySet, request: Request):
    queryset_paginated = queryset

    if request.query_params.get(REST_FRAMEWORK['PAGE_SIZE_QUERY_PARAM']) is not None:
        paginator = api_settings.DEFAULT_PAGINATION_CLASS()
        queryset_paginated = paginator.paginate_queryset(queryset, request)
        count_items = paginator.page.paginator.count
    else:
        count_items = queryset.count()

    return queryset_paginated, count_items
Ejemplo n.º 14
0
    def _check_input_for_variant(cls, cleaned_input: T_INPUT_MAP, qs: QuerySet):
        """Check the cleaned attribute input for a variant.
        An Attribute queryset is supplied.
        - ensure all attributes are passed
        - ensure the values are correct for a variant
        """
        if len(cleaned_input) != qs.count():
            raise ValidationError("All attributes must take a value")

        for attribute, values in cleaned_input:
            validate_attribute_input_for_variant(attribute, values)
Ejemplo n.º 15
0
    def log_pending(
        self,
        existing: QuerySet,
        pending: QuerySet,
        log_level: int = logging.INFO,
    ) -> None:
        """
        Log the number of pending  vs. existing instances.

        Parameters
        ----------
        existing : QuerySet
            Instances with existing runs
        pending : QuerySet
            Instances pending execution
        log_level : int, optional
            Logging level to use, by default 20 (INFO)
        """
        message = self.PENDING_FOUND.format(n_existing=existing.count(),
                                            n_pending=pending.count())
        _LOGGER.log(log_level, message)
Ejemplo n.º 16
0
 def run(self, runs: QuerySet = None) -> pd.DataFrame:
     node = get_cat12_segmentation_node()
     runs = runs or node.get_run_set().filter(status="SUCCESS")
     indices = pd.MultiIndex.from_tuples(
         combinations(range(runs.count()), 2))
     scores = pd.DataFrame(index=indices, columns=self.column_names)
     # scores = from_pandas(scores, chunksize=25000)
     for i_output, output in enumerate(self.COMPARED_OUTPUTS):
         output_name = self._fix_output_name(output)
         print(output_name)
         previous_index_1 = previous_index_2 = None
         data_1 = data_2 = None
         for index, *_ in scores.itertuples():
             index_1, index_2 = index
             run_1, run_2 = runs[index_1], runs[index_2]
             if previous_index_1 != index_1:
                 path_1 = run_1.get_output(output)
                 data_1 = self._read_data(path_1)
             if previous_index_2 != index_2:
                 path_2 = run_2.get_output(output)
                 data_2 = self._read_data(path_2)
             score = self._calculate_mi(data_1, data_2)
             scores.loc[index, output_name] = score
             previous_index_1 = index_1
             previous_index_2 = index_2
             if i_output == 0:
                 scan_1 = self._query_scan(run_1)
                 scan_2 = self._query_scan(run_2)
                 session_1 = scan_1.session
                 session_2 = scan_2.session
                 subject_1 = session_1.subject
                 subject_2 = session_2.subject
                 same_subject = subject_1 == subject_2
                 scores.loc[index, "Subject 1"] = subject_1.id
                 scores.loc[index, "Subject 2"] = subject_2.id
                 scores.loc[index, "Same Subject"] = same_subject
                 scores.loc[index, "Session 1"] = session_1.id
                 scores.loc[index, "Session 2"] = session_2.id
                 if same_subject:
                     same_session = session_1 == session_2
                     scores.loc[index, "Same Session"] = same_session
                 scores.loc[index, "Scan 1"] = scan_1.id
                 scores.loc[index, "Scan 2"] = scan_2.id
                 scores.loc[index,
                            "Scan Description 1"] = scan_1.description
                 scores.loc[index,
                            "Scan Description 2"] = scan_2.description
                 scores.loc[index, "Scan Time 1"] = scan_1.time
                 scores.loc[index, "Scan Time 2"] = scan_2.time
                 if same_subject:
                     delta = self._calculate_days_delta(scan_1, scan_2)
                     scores.loc[index, "Time Delta"] = delta
     return scores
Ejemplo n.º 17
0
def paging_queryset(query_set: QuerySet,
                    page: int = 0,
                    per_page: int = None,
                    **kwarg: dict) -> (QuerySet, dict):
    page_info = {
        'page': 1,
        'per_page': per_page,
        'pages': 1,
        'count': 0,
        'count_e': 0,
    }
    if query_set:
        if page > 0 and per_page:
            if isinstance(query_set, QuerySet) or hasattr(query_set, 'count'):
                paginate_by = int(per_page) if isinstance(
                    per_page, (int, float)) else 10
                page = int(page) if isinstance(page, (int, float)) else 1
                try:
                    paginator = Paginator(query_set, paginate_by)
                    page_info['count_e'] = paginator.count
                    page_info['count'] = query_set.count()
                    page_info['page'] = page
                    page_info['pages'] = paginator.num_pages
                    page_info['per_page'] = paginate_by
                    query_set = paginator.get_page(page)
                except Http404:
                    query_set = ()
                    page_info['count_e'] = 0
                    page_info['count'] = 0
                    page_info['page'] = page
                    page_info['pages'] = 1
                    page_info['per_page'] = paginate_by
        else:
            if isinstance(query_set, QuerySet) or hasattr(query_set, 'count'):
                page_info['count'] = query_set.count()
            elif hasattr(query_set, '__len__') or isinstance(query_set, Sized):
                page_info['count'] = len(query_set)
            else:
                page_info['count'] = 1
    return query_set, page_info
Ejemplo n.º 18
0
def filter_clinics_online(queryset: QuerySet, searcher):
    """

    :param queryset: Clinic QuerySet
    :param searcher: AppointmentClinicSearcher
    :return: Clinic QuerySet
    """
    queryset = queryset.filter(admin__id__in=OnlineService.filter_online(
        User.objects.clinic_admins()))
    if searcher.raise_exception and queryset.count() == 0:
        msg = NO_CLINIC_MSG.format(field='location and treatments')
        raise ValidationError({'location': msg, 'treatments': msg})
    return queryset
Ejemplo n.º 19
0
def paginate(query: QuerySet,
             schema: BaseModel,
             params: Optional[AbstractParams] = None) -> AbstractPage:
    params = resolve_params(params)
    raw_params = params.to_raw_params()

    total = query.count()
    start = raw_params.limit * raw_params.offset
    end = start + raw_params.limit
    rows = query.all()[start:end]

    items = [schema.from_django(row) for row in rows]

    return create_page(items, total, params)
Ejemplo n.º 20
0
def _daily_count(subjects_queryset: QuerySet):

    dates = []
    men_count = []
    women_count = []

    if subjects_queryset.count():
        subjects_queryset = subjects_queryset.order_by('faces__created_at')
        oldest_subject = subjects_queryset.first()
        newest_subject = subjects_queryset.last()

        lower_timestamp = min(
            [face.created_at for face in oldest_subject.faces.all()])
        upper_timestamp = max(
            [face.created_at for face in newest_subject.faces.all()])

        lower_date = timezone.localtime(lower_timestamp).date()
        upper_date = timezone.localtime(upper_timestamp).date()

        curr_date = lower_date
        ind = 0

        while curr_date <= upper_date:
            next_date = curr_date + datetime.timedelta(days=1)

            subjects = subjects_queryset.filter(
                faces__created_at__date__gte=curr_date,
                faces__created_at__date__lt=next_date)
            men_count.append(
                len(
                    set([
                        subject.id for subject in subjects
                        if subject.pred_sex == Subject.SEX_MAN
                    ])))
            women_count.append(
                len(
                    set([
                        subject.id for subject in subjects
                        if subject.pred_sex == Subject.SEX_WOMAN
                    ])))

            dates.append(str(curr_date))
            curr_date = next_date
            ind += 1

    return {
        'dates': dates,
        'men_count': men_count,
        'women_count': women_count,
    }
Ejemplo n.º 21
0
    def paginate_queryset(
        self,
        queryset: QuerySet,
        request: Request,
        view: APIView = None,
    ) -> OptionalList:
        serializer_class = self.get_request_serializer_class()
        serializer = serializer_class(data=request.query_params,
                                      max_per_page=self.max_page_size)
        serializer.is_valid(raise_exception=False)

        self.per_page = serializer.validated_data.get('per_page')
        self.page = serializer.validated_data.get('page', 1)

        if not self.per_page:
            return None

        self.has_next = False
        self.has_prev = (self.page > 1)

        base_url = request.build_absolute_uri()
        self.base_url = replace_query_param(base_url,
                                            self.page_size_query_param,
                                            self.per_page)
        self.request = request

        start_offset = (self.page - 1) * self.per_page
        stop_offset = start_offset + self.per_page + 1

        if self.use_count:
            self.total = len(queryset) if isinstance(
                queryset, list) else queryset.count()
            if self.total:
                self.pages, rem = divmod(self.total, self.per_page)
                if rem:
                    self.pages += 1
            else:
                self.pages = 1
            if self.page > self.pages:
                msg = self.invalid_page_message.format(page_number=self.page)
                raise NotFound(msg)

        paginated = list(queryset[start_offset:stop_offset])

        if len(paginated) > self.per_page:
            self.has_next = True
            del paginated[-1]

        return paginated
Ejemplo n.º 22
0
def page_objects_pagination(
        page: int,
        query_set: QuerySet,
        one_page_max: int = common_constant.DEFAULT_ONE_PAGE_MAX):
    """
        查询对象集根据当前页page分页
    :param page: 当前页
    :param query_set: Django 查询对象集
    :param one_page_max: 单页的最大数量
    :return: (page_query_set, item_sum, *args)  [*args=page_sum, page, one_page_max]
    """

    item_sum = query_set.count()
    item_start, item_end, *args = pagination(item_sum, page, one_page_max)
    page_query_set = query_set[item_start:item_end]
    return page_query_set, item_sum, args
Ejemplo n.º 23
0
 def export_applicants(self, request: HttpRequest, queryset: QuerySet):
     if queryset.count() != 1:
         self.message_user(request,
                           "Please select only one session for export",
                           messages.ERROR)
         return
     session: ApplicationSession = queryset.get()
     csv_resp = HttpResponse(content_type='text/csv')
     csv_resp[
         'Content-Disposition'] = 'attachment; filename="Applicants %s.csv"' % (
             session.move_in_str(), )
     wr = csv.writer(csv_resp)
     fields: List[Field] = Applicant._meta.get_fields(include_parents=False,
                                                      include_hidden=False)
     header = []
     fnames = []
     for f in fields:
         if f.is_relation:
             continue
         header.append(f.verbose_name)
         fnames.append(f.name)
     header.append("Abstain votes")
     header.append("Not suitable votes")  # -2
     header.append("Suitable votes")  # 1
     header.append("Very suitable votes")  # 2
     qs = session.questions()
     for q in qs:
         header.append(q.question_text)
     wr.writerow(header)
     for ap in Applicant.objects.filter(session=session):
         row = []
         for f in fnames:
             row.append(getattr(ap, f))
         for vv in [0, -2, 1, 2]:
             vc = ApplicationVote.objects.filter(applicant=ap,
                                                 points__exact=vv).count()
             row.append(vc)
         for q in qs:
             ans = 'no answer'
             try:
                 ans = ApplicationAnswer.objects.get(applicant=ap,
                                                     question=q).answer
             except:
                 pass
             row.append(ans)
         wr.writerow(row)
     return csv_resp
Ejemplo n.º 24
0
 def paginator(cls, queryset: QuerySet, query_params: dict) -> tuple:
     """
     分页获取页码
     :param queryset: QuerySet orm查询集
     :param query_params: query参数
     :return: tuple
     """
     page: int = int(query_params.get("page") or 1)
     page_size: int = int(query_params.get("page_size") or 10)
     length: int = queryset.count()
     pager: dict = {
         "page": page,
         "page_size": page_size,
         "max_page": math.ceil(length / page_size),
         "total": length
     }
     return queryset[(page - 1) * page_size:page * page_size], pager
Ejemplo n.º 25
0
def batch_qs(qs: QuerySet, batch_size=1000):
    """
    Returns a (start, end, total, queryset) tuple for each batch in the given
    queryset.

    Usage:
        # Make sure to order your querset
        article_qs = Article.objects.order_by('id')
        for start, end, total, qs in batch_qs(article_qs):
            print "Now processing %s - %s of %s" % (start + 1, end, total)
            for article in qs:
                print article.body
    """
    total = qs.count()
    for start in range(0, total, batch_size):
        end = min(start + batch_size, total)
        yield (qs[start:end], start, end, total)
Ejemplo n.º 26
0
def extract_recap_documents(
    docs: QuerySet,
    skip_ocr: bool = False,
    order_by: Optional[str] = None,
    queue: Optional[str] = None,
) -> None:
    """Loop over RECAPDocuments and extract their contents. Use OCR if requested.

    :param docs: A queryset containing the RECAPDocuments to be processed.
    :type docs: Django Queryset
    :param skip_ocr: Whether OCR should be completed (False) or whether items
    should simply be updated to have status OCR_NEEDED.
    :type skip_ocr: Bool
    :param order_by: An optimization parameter. You may opt to order the
    processing by 'small-first' or 'big-first'.
    :type order_by: str
    :param queue: The celery queue to send the content to.
    :type queue: str
    """
    docs = docs.exclude(filepath_local="")
    if skip_ocr:
        # Focus on the items that we don't know if they need OCR.
        docs = docs.filter(ocr_status=None)
    else:
        # We're doing OCR. Only work with those items that require it.
        docs = docs.filter(ocr_status=RECAPDocument.OCR_NEEDED)

    if order_by is not None:
        if order_by == "small-first":
            docs = docs.order_by("page_count")
        elif order_by == "big-first":
            docs = docs.order_by("-page_count")

    count = docs.count()
    throttle = CeleryThrottle(queue_name=queue)
    for i, pk in enumerate(docs.values_list("pk", flat=True)):
        throttle.maybe_wait()
        extract_recap_pdf.apply_async((pk, skip_ocr), priority=5, queue=queue)
        if i % 1000 == 0:
            msg = f"Sent {i + 1}/{count} tasks to celery so far."
            logger.info(msg)
            sys.stdout.write(f"\r{msg}")
            sys.stdout.flush()
Ejemplo n.º 27
0
def reorder_list_by_slugs(queryset: QuerySet, field: str, slugs: List[str]) -> None:
    """Alter objects' `field` value so slugs[0] is at position 0 and so on.

    For instance:
    `remove_gap_from_list(workflow.blocks, 'position', ["block-1", "block-2")`
    will set `.position = 0` for block-1 and `.position = 1` for block-2.

    This must be called within a database transaction.

    This could be a single `UPDATE`. However, Postgres checks uniqueness
    constraints _before the statement ends_, so it leads to a conflict.
    TODO use Django 3.1 with DEFERRABLE INITIALLY DEFFERED constraint to avoid
    this problem. In the meantime, write one row at a time to avoid conflict.
    """
    # Make sure we don't overwrite any positions: make all existing numbers
    # negative.
    assert queryset.count() == len(slugs)
    queryset.update(**{field: -1 - F(field)})
    for position, slug in enumerate(slugs):
        queryset.filter(slug=slug).update(**{field: position})
Ejemplo n.º 28
0
    def _create_search_results_elem(self, all_md: QuerySet, returned_md: list):
        """ Creates the <csw:SearchResults> element

        Returns:
             search_status_elem (_Element): The lxml element
        """
        number_of_records = all_md.count()
        number_of_records_returned = len(returned_md)
        next_record = self.param.start_position + number_of_records_returned
        next_record = next_record if next_record < number_of_records else 0

        attribs = OrderedDict()
        attribs["numberOfRecordsMatched"] = str(number_of_records)
        attribs["numberOfRecordsReturned"] = str(number_of_records_returned)
        attribs["elementSet"] = str(self.param.element_set_name
                                    or ",".join(self.param.element_name))
        attribs["nextRecord"] = str(next_record)

        elem = Element("{}SearchResults".format(self.csw_ns), attrib=attribs)
        return elem
Ejemplo n.º 29
0
def summarize_invoice_statistics(modeladmin, request: HttpRequest,
                                 qs: QuerySet):  # pylint: disable=unused-argument
    invoice_states = list(state for (state, name) in INVOICE_STATE)

    invoiced_total_amount = Decimal("0.00")
    invoiced_total_count = 0

    lines = [
        "<pre>",
        _("({total_count} invoices)").format(total_count=qs.count()),
    ]
    for state in invoice_states:
        state_name = choices_label(INVOICE_STATE, state)
        qs2 = qs.filter(state=state)

        invoiced = qs2.filter(state=state).aggregate(amount=Coalesce(
            Sum("amount"), Decimal("0.00")),
                                                     count=Count("*"))
        invoiced_amount = Decimal(invoiced["amount"])
        invoiced_count = int(invoiced["count"])
        invoiced_total_amount += invoiced_amount
        invoiced_total_count += invoiced_count

        lines.append("{state_name} | x{count} | {amount:.2f}".format(
            state_name=state_name,
            amount=invoiced_amount,
            count=invoiced_count))

    lines.append(
        _("Total") + " {label} | x{count} | {amount:.2f}".format(
            label=_("amount"),
            amount=invoiced_total_amount,
            count=invoiced_total_count))
    lines.append("</pre>")

    lines = align_lines(lines, "|")
    messages.add_message(request,
                         INFO,
                         format_html("<br>".join(lines)),
                         extra_tags="safe")
Ejemplo n.º 30
0
def make_alert_messages(
        d: Docket,
        new_des: QuerySet,
        email_addresses: "ValuesQuerySet[User, Optional[str]]",  # type: ignore
) -> List[EmailMultiAlternatives]:
    """Make docket alert messages that can be sent to users

    :param d: The docket to work on
    :param new_des: The new docket entries
    :param email_addresses: A list of user email addresses to send to
    :return: A list of email messages to send
    """
    case_name = trunc(best_case_name(d), 100, ellipsis="...")
    subject_template = loader.get_template("docket_alert_subject.txt")
    subject = subject_template.render({
        "docket": d,
        "count": new_des.count(),
        "case_name": case_name,
    }).strip()  # Remove newlines that editors can insist on adding.
    email_context = {"new_des": new_des, "docket": d}
    txt_template = loader.get_template("docket_alert_email.txt")
    html_template = loader.get_template("docket_alert_email.html")
    messages = []
    for email_address in email_addresses:
        msg = EmailMultiAlternatives(
            subject=subject,
            body=txt_template.render(email_context),
            from_email=settings.DEFAULT_ALERTS_EMAIL,
            to=[email_address],
            headers={f"X-Entity-Ref-ID": "docket.alert:{d.pk}"},
        )
        html = html_template.render(email_context)
        msg.attach_alternative(html, "text/html")
        messages.append(msg)

    # Add a bcc to the first message in the list so that we get a copy.
    messages[0].bcc = ["*****@*****.**"]
    return messages