def test_foreignkey_reverse(self):
        books = list(Book.objects.all())
        with self.assertNumQueries(1):
            prefetch_related_objects(books, 'first_time_authors')

        with self.assertNumQueries(0):
            [list(book.first_time_authors.all()) for book in books]
Exemplo n.º 2
0
def ensure_caches(user):
    """
    Ensure the cache for a given type.
    """
    from django.db.models import query

    if (not (isinstance(user, Entity))):
        return

    blocked_key = cache_key('blocked', user.pk)
    blocked_entities = cache_manager.cache_get(blocked_key,
                                               sliding_timeout=DEFAULT_TIMEOUT)
    if (blocked_entities is None):
        query.prefetch_related_objects([user], 'blocked_entities')
        cache_manager.cache_set(
            blocked_key, user._prefetched_objects_cache['blocked_entities'])
    else:
        if (not (hasattr(user, '_prefetched_objects_cache'))):
            user._prefetched_objects_cache = {}
        user._prefetched_objects_cache['blocked_entities'] = blocked_entities

    blocking_key = cache_key('blocking', user.pk)
    blocking_entities = cache_manager.cache_get(
        blocking_key, sliding_timeout=DEFAULT_TIMEOUT)
    if (blocking_entities is None):
        query.prefetch_related_objects([user], 'blocking_entities')
        cache_manager.cache_set(
            blocking_key, user._prefetched_objects_cache['blocking_entities'])
    else:
        if (not (hasattr(user, '_prefetched_objects_cache'))):
            user._prefetched_objects_cache = {}
        user._prefetched_objects_cache['blocking_entities'] = blocking_entities
Exemplo n.º 3
0
    def retrieve(self, request, region, agency, pk):
        try:
            stop = Stop.objects.select_related('agency') \
                .get(pk=Stop.create_id(Agency.create_id(region, agency), pk))
        except Stop.DoesNotExist:
            raise Http404('No Route matches the given query.')
        agency = stop.agency
        arrivals = get_provider(stop.agency).arrivals(stop)
        # odd if we have a mixture of units, but we currently don't so...
        # sort the arrivals
        arrivals.sort(key=attrgetter('away'))
        # never return more than 15 arrivals, limit before the prefetch to
        # avoid extra work
        arrivals = arrivals[:15]
        prefetch_related_objects(arrivals, ['direction__route__directions'])

        routes = {}
        for arrival in arrivals:
            if arrival.direction:
                route = arrival.direction.route
                routes[route.id] = route

        self.object = StopAdapter(stop, arrivals=arrivals,
                                  routes=routes.values())
        serializer = self.get_serializer(self.object)
        return Response(serializer.data)
    def test_foreignkey_reverse(self):
        books = list(Book.objects.all())
        with self.assertNumQueries(1):
            prefetch_related_objects(books, 'first_time_authors')

        with self.assertNumQueries(0):
            [list(book.first_time_authors.all()) for book in books]
    def test_foreignkey_forward(self):
        authors = list(Author.objects.all())
        with self.assertNumQueries(1):
            prefetch_related_objects(authors, 'first_book')

        with self.assertNumQueries(0):
            [author.first_book for author in authors]
    def test_m2m_reverse(self):
        author1 = Author.objects.get(id=self.author1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([author1], 'books')

        with self.assertNumQueries(0):
            self.assertEqual(set(author1.books.all()), {self.book1, self.book2})
    def test_prefetch_object(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([book1], Prefetch('authors'))

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()), {self.author1, self.author2, self.author3})
Exemplo n.º 8
0
    def to_container(self, container):
        """Container """
        result = container(self.iterator())
        if self._prefetch_related_lookups:
            prefetch_related_objects(result, *self._prefetch_related_lookups)

        return result
Exemplo n.º 9
0
    def send(self, instances):
        messages = []
        if isinstance(instances, list):
            prefetch_related_objects(instances, ('equipier_set', ))
        elif hasattr(instances, 'prefetch_related'):
            instances.prefetch_related('equipiers')
        for instance in instances:
            dests = set()
            if self.destinataire in ('Organisateur', 'Tous'):
                dests.add(self.course.email_contact)
            if self.destinataire in ('Equipe', 'Tous') and isinstance(instance, Equipe):
                    dests.add(instance.gerant_email)
            if self.destinataire in ('Equipiers', 'Tous'):
                if isinstance(instance, Equipier):
                    dests.add(instance.email)
                if isinstance(instance, Equipe):
                    for equipier in instance.equipier_set.filter(numero__lte=F('equipe__nombre')):
                        dests.add(equipier.email)
            
            context = Context({
                "instance": instance,
                'ROOT_URL': 'http://%s' % Site.objects.get_current(),
            })
            subject = Template(self.sujet).render(context)
            message = Template(self.message).render(context)

            bcc = []
            if self.bcc:
                bcc = re.split('[,; ]+', self.bcc)
            for dest in dests:
                message = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL, [ dest ], bcc, reply_to=[self.course.email_contact,])
                message.content_subtype = "html"
                messages.append(message)
        MailThread(messages).start()
Exemplo n.º 10
0
 def prefetch_related(tests):
     prefetch_related_objects(
         tests,
         'test_run',
         'test_run__environment',
         'test_run__build',
     )
    def test_foreignkey_forward(self):
        authors = list(Author.objects.all())
        with self.assertNumQueries(1):
            prefetch_related_objects(authors, 'first_book')

        with self.assertNumQueries(0):
            [author.first_book for author in authors]
Exemplo n.º 12
0
    def get_queryset(self):
        # Retrieves the latest entry for each `format_requested`, for our content.
        # The `latest_events` sub-request retieves all events for the current content, and
        # and annotates them with their row number if we would select them by requested format,
        # ordered by descending date. The first one by date with the PDF type would be annotated
        # 1, the second with PDF, 2, etc.
        # The outer request selects everything in the inner request, but filtering only the first
        # row for each format, keeping only the latest record for each type for this content.
        # We have to use a sub-request as the SQL spec forbides to filter on a windowed element
        # directly.
        #
        # This uses raw SQL because even if Django supports windowed requests, it does not allow
        # to select _from_ a subrequest (« SELECT * FROM (SELECT … ) WHERE … »).
        exports = PublicationEvent.objects.raw(
            """
            WITH latest_events AS (
                SELECT p.*, ROW_NUMBER() OVER (PARTITION BY format_requested ORDER BY date DESC) AS row
                FROM tutorialv2_publicationevent AS p
                INNER JOIN tutorialv2_publishedcontent published ON (p.published_object_id = published.id)
                WHERE published.content_id = %s
            )
            SELECT * FROM latest_events WHERE row = 1""",
            [int(self.kwargs.get("pk", 0))],
        )

        prefetch_related_objects(exports, "published_object")

        return exports
Exemplo n.º 13
0
        def _fill_obj(obj):
            if included:
                if not hasattr(obj, '_default_translatable_fields'):
                    obj._default_translatable_fields = {
                        field: getattr(obj, field)
                        for field in type(
                            obj)._get_translatable_fields_names()
                    }
                object_id = str(obj.pk)
                instances[object_id] = obj
                nonlocal query
                query |= models.Q(
                    content_type__id=content_type_id,
                    object_id=object_id,
                )

            if hierarchy:
                for (relation, detail) in hierarchy.items():
                    value = getattr(obj, relation, None)

                    if value is not None:
                        if isinstance(value, models.Manager):
                            if not (hasattr(obj, '_prefetched_objects_cache')
                                    and relation
                                    in obj._prefetched_objects_cache):
                                prefetch_related_objects([obj], relation)
                            value = value.all()
                        _fill_entity(
                            entity=value,
                            hierarchy=detail['relations'],
                            included=detail['included'],
                        )
Exemplo n.º 14
0
    def last_actions(self, release, limit, language=None, user=None):
        sqlu = "SELECT pl.pofile_id, max(pl.id) as id FROM pofile_log pl"
        if user:
            sqlu += " LEFT OUTER JOIN pofile_assigns a ON pl.pofile_id = a.pofile_id"
            sqlu += " WHERE a.translate_id=%(userid)s or a.review_id=%(userid)s" % {'userid': user.id}
        else:
            try:
                bot = User.objects.get(username=getattr(settings, 'BOT_USERNAME','bot'))
            except:
                bot = None
            if bot:            
                sqlu += " WHERE pl.user_id <> %d" % bot.pk
        sqlu += " group by pl.pofile_id"
        
        sql = "SELECT p.id, p.pofile_id, p.created, p.user_id, p.action, p.comment FROM pofile_log p INNER JOIN (%s) p2 "\
              "INNER JOIN pofile f ON p.id = p2.id AND f.id = p.pofile_id" % sqlu 
        
        sql += " WHERE f.release_id = %d" % release.id
        if language:
            sql += " AND f.language_id=%d" % language.id 
        sql += " ORDER BY p.id DESC LIMIT %d" % limit

        from django.db.models.query import prefetch_related_objects
        result = list(self.raw(sql))
        prefetch_related_objects(result, ('user',))
        return result
Exemplo n.º 15
0
    def get_attrs(self, item_list, user):
        prefetch_related_objects(item_list, "created_by")

        result = defaultdict(lambda: {"created_by": {}})

        user_serializer = UserSerializer()
        serialized_users = {
            user["id"]: user
            for user in serialize(
                [
                    discover_saved_query.created_by
                    for discover_saved_query in item_list
                    if discover_saved_query.created_by
                ],
                user=user,
                serializer=user_serializer,
            )
        }

        for discover_saved_query in item_list:
            result[discover_saved_query]["created_by"] = serialized_users.get(
                str(discover_saved_query.created_by_id)
            )

        return result
    def test_prefetch_object(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([book1], Prefetch('authors'))

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()),
                             {self.author1, self.author2, self.author3})
    def test_m2m_reverse(self):
        author1 = Author.objects.get(id=self.author1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects([author1], 'books')

        with self.assertNumQueries(0):
            self.assertEqual(set(author1.books.all()),
                             {self.book1, self.book2})
    def test_prefetch_queryset(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects(
                [book1],
                Prefetch('authors', queryset=Author.objects.filter(id__in=[self.author1.id, self.author2.id]))
            )

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()), {self.author1, self.author2})
Exemplo n.º 19
0
 def prefetch_related(tests):
     prefetch_related_objects(
         tests,
         'known_issues',
         'test_run',
         'test_run__environment',
         'test_run__build',
         'test_run__build__project',
         'test_run__build__project__group',
     )
Exemplo n.º 20
0
 def prefetch_related(builds):
     prefetch_related_objects(
         builds,
         'project',
         'project__group',
         'test_runs',
         'test_runs__environment',
         'test_runs__tests',
         'test_runs__tests__suite',
     )
Exemplo n.º 21
0
 def prefetch_related(tests):
     prefetch_related_objects(
         tests,
         'known_issues',
         'test_run',
         'test_run__environment',
         'test_run__build',
         'test_run__build__project',
         'test_run__build__project__group',
     )
Exemplo n.º 22
0
    def _prefetch_related_objects(self):
        things = defaultdict(list)
        for index, thing in enumerate(self._result_cache, start=0):
            things[thing.__class__].append(thing)

        for klass, instances in things.items():
            prefetches = self._prefetch_related_lookups[:]
            if klass in self._our_prefetches:
                prefetches.extend(self._our_prefetches[klass])
            prefetch_related_objects(instances, prefetches)
        return None
Exemplo n.º 23
0
    def get_attrs(self, item_list, user):
        item_dict = {i.id: i for i in item_list}
        prefetch_related_objects(item_list, "created_by")

        widgets = (DashboardWidget.objects.filter(
            dashboard_id__in=item_dict.keys()).order_by("order").values(
                "dashboard_id", "order", "display_type", "detail", "id"))

        result = defaultdict(lambda: {
            "widget_display": [],
            "widget_preview": [],
            "created_by": {}
        })
        for widget in widgets:
            dashboard = item_dict[widget["dashboard_id"]]
            display_type = DashboardWidgetDisplayTypes.get_type_name(
                widget["display_type"])
            result[dashboard]["widget_display"].append(display_type)

        for widget in widgets:
            dashboard = item_dict[widget["dashboard_id"]]
            widget_preview = {
                "displayType":
                DashboardWidgetDisplayTypes.get_type_name(
                    widget["display_type"]),
                "layout":
                None,
            }
            if widget.get("detail"):
                detail = json.loads(widget["detail"])
                if detail.get("layout"):
                    widget_preview["layout"] = detail["layout"]

            result[dashboard]["widget_preview"].append(widget_preview)

        user_serializer = UserSerializer()
        serialized_users = {
            user["id"]: user
            for user in serialize(
                [
                    dashboard.created_by
                    for dashboard in item_list if dashboard.created_by
                ],
                user=user,
                serializer=user_serializer,
            )
        }

        for dashboard in item_dict.values():
            result[dashboard]["created_by"] = serialized_users.get(
                str(dashboard.created_by_id))

        return result
Exemplo n.º 24
0
    def __get_tests__(self, builds, metadata):
        test_runs = []
        for build in builds:
            for test_run in build.test_runs.all():
                test_runs.append(test_run)

        tests = []
        for test_runs_batch in split_list(test_runs, chunk_size=100):
            prefetch_related_objects(test_runs_batch, Prefetch('tests', queryset=Test.objects.filter(metadata=metadata).prefetch_related('metadata').order_by()))
            for test_run in test_runs_batch:
                tests += test_run.tests.all()
        return tests
    def test_prefetch_queryset(self):
        book1 = Book.objects.get(id=self.book1.id)
        with self.assertNumQueries(1):
            prefetch_related_objects(
                [book1],
                Prefetch('authors',
                         queryset=Author.objects.filter(
                             id__in=[self.author1.id, self.author2.id])))

        with self.assertNumQueries(0):
            self.assertEqual(set(book1.authors.all()),
                             {self.author1, self.author2})
Exemplo n.º 26
0
    def _get_sorted_occurrences(self):
        occurrences = []
        if hasattr(self, 'occurrence_pool') and self.occurrence_pool is not None:
            for occurrence in self.occurrence_pool:
                if occurrence.start <= self.utc_end and occurrence.end >= self.utc_start:
                    occurrences.append(occurrence)
            return occurrences

        prefetch_related_objects(self.events, 'occurrence_set')
        for event in self.events:
            event_occurrences = event.get_occurrences(self.start, self.end, clear_prefetch=False)
            occurrences += event_occurrences
        return sorted(occurrences, **self.sorting_options)
Exemplo n.º 27
0
    def retrieve(self, request, region=None, agency=None, *args, **kwargs):
        try:
            lat = float(request.GET['lat'])
            lon = float(request.GET['lon'])
            radius = float(request.GET.get('radius', 500.0))
            if radius < 0 or 5000 < radius:
                # TODO: bad request, invalid param
                raise Exception('invalid radius')
        except KeyError:
            # TODO: bad request, missing param
            raise
        except ValueError:
            # TODO: bad request, bad param
            raise

        # we'll at most return N stops
        # TODO: limiting before filtering is bad
        stops = Stop.objects.nearby(lat, lon, radius)[:20]

        if region and agency:
            agency_id = Agency.create_id(region, agency)

            def desired_agency(stop):
                return stop.agency_id == agency_id

            stops = filter(desired_agency, stops)

        prefetch_related_objects(
            stops,
            ['agency', 'agency__region', 'directions', 'directions__route'])

        routes = {}
        agencies = {}
        regions = {}
        for stop in stops:
            agency = stop.agency
            agencies[agency.id] = agency
            region = agency.region
            regions[region.id] = region
            for direction in stop.directions.all():
                route = direction.route
                routes[route.id] = direction.route

        self.object = Adapter(self,
                              agencies=agencies.values(),
                              regions=regions.values(),
                              routes=routes.values(),
                              stops=stops)

        serializer = self.get_serializer(self.object)
        return Response(serializer.data)
Exemplo n.º 28
0
 def has_permission(self, request):
     if not request.user.is_authenticated():
         return False
     prefetch_related_objects([request.user], ('accreditations__course', ))
     if request.path.endswith('/logout/'):
         return True
     if request.user.is_superuser:
         return True
     if request.path.endswith('/choose/') or request.path.endswith('/ask/') or re.search(r'/ask/[^/]+/$', request.path) or request.path.endswith('/inscriptions/course/add/') or request.path.endswith('/course/jsi18n/'):
         return True
     if 'course_uid' not in request.COOKIES:
         return False
     course_uid = request.COOKIES['course_uid']
     return request.user.accreditations.filter(course__uid=course_uid).exclude(role='').count() > 0
Exemplo n.º 29
0
    def retrieve(self, request, region=None, agency=None, *args, **kwargs):
        try:
            lat = float(request.GET['lat'])
            lon = float(request.GET['lon'])
            radius = float(request.GET.get('radius', 500.0))
            if radius < 0 or 5000 < radius:
                # TODO: bad request, invalid param
                raise Exception('invalid radius')
        except KeyError:
            # TODO: bad request, missing param
            raise
        except ValueError:
            # TODO: bad request, bad param
            raise

        # we'll at most return N stops
        # TODO: limiting before filtering is bad
        stops = Stop.objects.nearby(lat, lon, radius)[:20]

        if region and agency:
            agency_id = Agency.create_id(region, agency)

            def desired_agency(stop):
                return stop.agency_id == agency_id

            stops = filter(desired_agency, stops)

        prefetch_related_objects(stops, ['agency', 'agency__region',
                                         'directions', 'directions__route'])

        routes = {}
        agencies = {}
        regions = {}
        for stop in stops:
            agency = stop.agency
            agencies[agency.id] = agency
            region = agency.region
            regions[region.id] = region
            for direction in stop.directions.all():
                route = direction.route
                routes[route.id] = direction.route

        self.object = Adapter(self, agencies=agencies.values(),
                              regions=regions.values(), routes=routes.values(),
                              stops=stops)

        serializer = self.get_serializer(self.object)
        return Response(serializer.data)
Exemplo n.º 30
0
    def _get_sorted_occurrences(self):
        occurrences = []
        if hasattr(self, 'occurrence_pool') and self.occurrence_pool is not None:
            for occurrence in self.occurrence_pool:
                if occurrence.start <= self.utc_end and occurrence.end >= self.utc_start:
                    occurrences.append(occurrence)
            return occurrences

        try:
            prefetch_related_objects(self.events, 'occurrence_set')
        except AttributeError:
            prefetch_related_objects(self.events, ['occurrence_set'])
        for event in self.events:
            event_occurrences = event.get_occurrences(self.start, self.end, clear_prefetch=False)
            occurrences += event_occurrences
        return sorted(occurrences, **self.sorting_options)
Exemplo n.º 31
0
 def get_context_data(self, *args, **kwargs):
     ret = super().get_context_data(*args, **kwargs)
     user = self.request.user
     membre = user.membre
     if user.is_staff and 'membre_id' in self.kwargs:
         membre = models.Membre.objects.get(id=self.kwargs['membre_id'])
     ret['membre'] = membre
     prefetch_related_objects(
         [membre],
         Prefetch(
             'licences',
             models.Licence.objects.annotate(
                 _montant_paiement=Sum('paiements__montant'))))
     ret['STRIPE_PUBLIC_KEY'] = settings.STRIPE_PUBLIC_KEY
     ret['saisons'] = models.Saison.objects.filter(ouvert=True).exclude(
         membres__in=membre.licences.all())
     return ret
Exemplo n.º 32
0
    def update(self, request, *args, **kwargs):
        partial = kwargs.pop('partial', False)
        instance = self.get_object()
        serializer = self.get_serializer(instance,
                                         data=request.data,
                                         partial=partial)
        serializer.is_valid(raise_exception=True)
        self.perform_update(serializer)

        if getattr(instance, '_prefetched_objects_cache', None):
            # If 'prefetch_related' has been applied to a queryset, we need to
            # forcibly invalidate the prefetch cache on the instance,
            # and then re-prefetch related objects
            instance._prefetched_objects_cache = {}
            prefetch_related_objects([instance], *self.get_prefetch_related())

        return Response(serializer.data)
    def test_m2m_then_m2m(self):
        """
        We can follow a m2m and another m2m.
        """
        authors = list(Author.objects.all())
        with self.assertNumQueries(2):
            prefetch_related_objects(authors, 'books__read_by')

        with self.assertNumQueries(0):
            self.assertEqual(
                [[[str(r) for r in b.read_by.all()] for b in a.books.all()]
                 for a in authors],
                [
                    [['Amy'], ['Belinda']],  # Charlotte - Poems, Jane Eyre
                    [['Amy']],  # Anne - Poems
                    [['Amy'], []],  # Emily - Poems, Wuthering Heights
                    [['Amy', 'Belinda']],  # Jane - Sense and Sense
                ])
Exemplo n.º 34
0
    def list(self, request, agency, lat, lon, *args, **kwargs):
        lat = float(lat)
        lon = float(lon)
        max = float(request.GET.get('max', 0.25))
        if max < 0:
            max = 0.25
        elif 1 < max:
            max = 1

        radius = max * meters_per_mile

        # we'll at most return n stops
        # TODO: limiting before filtering is bad
        stops = Stop.objects.nearby(lat, lon, radius)[:25]

        prefetch_related_objects(stops, ['agency', 'directions',
                                         'directions__route'])

        results = []
        for stop in stops:
            # this is kind of ugly, but should do the trick since we don't know
            # the region
            if not stop.agency_id.endswith(agency):
                continue
            for direction in stop.directions.all():
                results.append({
                    'agency_tag': stop.agency.get_id(),
                    'direction_tag': direction.get_id(),
                    'route_tag': direction.route.get_id(),
                    'stop_tag': stop.get_id(),
                    'stop_id': stop.code,
                    'route_title': direction.route.name,
                    'direction_title': direction.name,
                    'stop_title': stop.name,
                    'distance': stop.distance / meters_per_mile,
                    'lat': stop.lat,
                    'lon': stop.lon
                })

                if len(results) > 25:
                    return Response(results)

        return Response(results)
    def test_m2m_then_m2m(self):
        """
        We can follow a m2m and another m2m.
        """
        authors = list(Author.objects.all())
        with self.assertNumQueries(2):
            prefetch_related_objects(authors, 'books__read_by')

        with self.assertNumQueries(0):
            self.assertEqual(
                [
                    [[str(r) for r in b.read_by.all()] for b in a.books.all()]
                    for a in authors
                ],
                [
                    [['Amy'], ['Belinda']],  # Charlotte - Poems, Jane Eyre
                    [['Amy']],               # Anne - Poems
                    [['Amy'], []],           # Emily - Poems, Wuthering Heights
                    [['Amy', 'Belinda']],    # Jane - Sense and Sense
                ]
            )
Exemplo n.º 36
0
def listar_desafiaveis(request, username):
    jogador = get_object_or_404(Jogador, user__username=username)

    posicao_atual_jogador = jogador.posicao_em(timezone.localtime())

    desafiaveis = buscar_desafiaveis(jogador, timezone.localtime(), False,
                                     False)

    # Buscar usuários de desafiáveis
    prefetch_related_objects(desafiaveis, 'user')

    desafios_pendentes = DesafioLadder.objects.filter(cancelamentodesafioladder__isnull=True, admin_validador__isnull=True,
                                                      score_desafiante__gt=F('score_desafiado'),
                                                      posicao_desafiado__lte=posicao_atual_jogador, posicao_desafiante__gte=posicao_atual_jogador) \
                                                      .select_related('desafiante', 'desafiado')

    return render(
        request, 'jogadores/listar_desafiaveis.html', {
            'desafiaveis': desafiaveis,
            'desafios_pendentes': desafios_pendentes,
            'jogador': jogador
        })
Exemplo n.º 37
0
    def create(self, validated_data):
        """Create instances in bulk efficiently"""

        ModelClass = self.child.Meta.model  # pylint: disable=invalid-name

        info = model_meta.get_field_info(ModelClass)
        m2m_fields = [f for f, i in info.relations.items() if i.to_many]

        instances = []
        m2m_values = []
        for attrs in validated_data:
            m2m_values.append({})
            for field in m2m_fields:
                # m2m fields must be set after the instance is created
                m2m_values[-1][field] = attrs.pop(field, None)
            if hasattr(self.child, "bulk_create_attrs"):
                attrs = self.child.bulk_create_attrs(attrs)
            instances.append(ModelClass(**attrs))

        # create the instances in bulk for efficiency
        try:
            ModelClass.objects.bulk_create(instances)
        except IntegrityError as exc:
            raise ValidationError(exc)

        # set any m2m values
        for instance, m2m_values_ in zip(instances, m2m_values):
            for field_name, value in m2m_values_.items():
                if value:
                    field = getattr(instance, field_name)
                    field.set(value)

        # prefetch any m2m values which are serializer fields to avoid n+1 queries
        prefetch_fields = set(m2m_fields) & set(self.child.Meta.fields)
        prefetch_related_objects(instances, *prefetch_fields)

        return instances
 def test_unknown(self):
     book1 = Book.objects.get(id=self.book1.id)
     with self.assertRaises(AttributeError):
         prefetch_related_objects([book1], 'unknown_attribute')
Exemplo n.º 39
0
def get_person_as_version_data(person, new_person=False):
    """
    If new_person is True then skip some DB checks that we know will be
    empty. This should reduce the number of queries needed.
    """

    # Prefetch to reduce the number of queries
    prefetch_related_objects([person], "tmp_person_identifiers")

    result = {}
    result["id"] = str(person.id)

    # Add PersonIdentifier fields in to the global version namespace
    # TODO: move these to dict to make it easier to merge/split/revert
    existing_identifiers = {
        identifier.value_type: identifier.value
        for identifier in person.tmp_person_identifiers.all()
    }
    for field in PersonIdentifierFields:
        result[field.name] = existing_identifiers.get(field.name, "")

    for field in settings.SIMPLE_POPOLO_FIELDS:
        result[field.name] = getattr(person, field.name) or ""

    # Add legacy identifiers
    # TODO: these should use the PersonIdenfitiers model and value types,
    # but this code emulates the legacy way of adding IDs.
    if person.get_single_identifier_of_type("theyworkforyou"):
        result["identifiers"] = []
        new_id = person.get_single_identifier_of_type(
            "theyworkforyou").internal_identifier
        if "publicwhip" not in new_id:
            new_id = "uk.org.publicwhip/person/{}".format(new_id)

        result["identifiers"].append({
            "identifier": new_id,
            "scheme": "uk.org.publicwhip"
        })
    if person.get_single_identifier_of_type("twitter_username"):
        result["identifiers"] = result.get("identifiers", [])
        result["identifiers"].append({
            "identifier":
            person.get_single_identifier_of_type(
                "twitter_username").internal_identifier,
            "scheme":
            "twitter",
        })

    result["other_names"] = []
    candidacies = {}
    not_standing = []

    if not new_person:
        result["other_names"] = [{
            "name": on.name,
            "note": on.note,
            "start_date": on.start_date,
            "end_date": on.end_date,
        }
                                 for on in person.other_names.order_by(
                                     "name", "start_date", "end_date")]

        for membership in person.memberships.all():
            ballot = membership.ballot
            candidacy = {"party": membership.party.ec_id}
            if membership.elected is not None:
                candidacy["elected"] = membership.elected
            if membership.party_list_position is not None:
                candidacy[
                    "party_list_position"] = membership.party_list_position
            candidacies[ballot.ballot_paper_id] = candidacy

        for not_standing_in_election in person.not_standing.all():
            not_standing.append(not_standing_in_election.slug)

    # Add `favourite_biscuits` to an `extra_fields` key
    # to re-produce the previous ExtraField model.
    # This is done like this to save changing the version diff
    # for exery edit to move to key to the parent object.
    # In the future we will have to run a script to move all the
    # keys to wherever we want them.
    extra_fields = {"favourite_biscuits": person.favourite_biscuit or ""}

    result["extra_fields"] = extra_fields

    result["candidacies"] = candidacies
    if not_standing:
        result["not_standing"] = not_standing
    return result
Exemplo n.º 40
0
 def prefetch(self, *related):
     prefetch_related_objects([self], *related)
Exemplo n.º 41
0
    def get_nodes(self, request):
        from cms.models import Title

        site = self.renderer.site
        lang = self.renderer.request_language
        pages = get_page_queryset(
            site,
            draft=self.renderer.draft_mode_active,
            published=not self.renderer.draft_mode_active,
        )

        if is_valid_site_language(lang, site_id=site.pk):
            _valid_language = True
            _hide_untranslated = hide_untranslated(lang, site.pk)
        else:
            _valid_language = False
            _hide_untranslated = False

        if _valid_language:
            # The request language has been explicitly configured
            # for the current site.
            if _hide_untranslated:
                fallbacks = []
            else:
                fallbacks = get_fallback_languages(lang, site_id=site.pk)
            languages = [lang
                         ] + [_lang for _lang in fallbacks if _lang != lang]
        else:
            # The request language is not configured for the current site.
            # Fallback to all configured public languages for the current site.
            languages = get_public_languages(site.pk)
            fallbacks = languages

        pages = (pages.filter(title_set__language__in=languages).
                 select_related('node').order_by('node__path').distinct())

        if not self.renderer.draft_mode_active:
            # we're dealing with public pages.
            # prefetch the draft versions.
            pages = pages.select_related('publisher_public__node')
        pages = get_visible_nodes(request, pages, site)

        if not pages:
            return []

        try:
            homepage = [page for page in pages if page.is_home][0]
        except IndexError:
            homepage = None

        titles = Title.objects.filter(
            language__in=languages,
            publisher_is_draft=self.renderer.draft_mode_active,
        )

        lookup = Prefetch(
            'title_set',
            to_attr='filtered_translations',
            queryset=titles,
        )
        prefetch_related_objects(pages, lookup)
        # Build the blank title instances only once
        blank_title_cache = {
            language: EmptyTitle(language=language)
            for language in languages
        }

        if lang not in blank_title_cache:
            blank_title_cache[lang] = EmptyTitle(language=lang)

        # Maps a node id to its page id
        node_id_to_page = {}

        def _page_to_node(page):
            # EmptyTitle is used to prevent the cms from trying
            # to find a translation in the database
            page.title_cache = blank_title_cache.copy()

            for trans in page.filtered_translations:
                page.title_cache[trans.language] = trans
            menu_node = get_menu_node_for_page(
                self.renderer,
                page,
                language=lang,
                fallbacks=fallbacks,
            )
            return menu_node

        menu_nodes = []

        for page in pages:
            node = page.node
            parent_id = node_id_to_page.get(node.parent_id)

            if node.parent_id and not parent_id:
                # If the parent page is not available (unpublished, etc..)
                # don't bother creating menu nodes for its descendants.
                continue

            menu_node = _page_to_node(page)
            cut_homepage = homepage and not homepage.in_navigation

            if cut_homepage and parent_id == homepage.pk:
                # When the homepage is hidden from navigation,
                # we need to cut all its direct children from it.
                menu_node.parent_id = None
            else:
                menu_node.parent_id = parent_id
            node_id_to_page[node.pk] = page.pk
            menu_nodes.append(menu_node)
        return menu_nodes
Exemplo n.º 42
0
def load_chunks(submit_milestone, user_map, django_user):
    chunks = []
    chunk_map = {}
    submissions = {}

    django_submissions = submit_milestone.submissions.exclude(authors=django_user).prefetch_related("authors")

    # the exclude() in the call below failed because of a Django bug.  Try submit_milestone=55, django_user=1016 on the production db.
    # django_chunks = Chunk.objects \
    #         .filter(file__submission__milestone=submit_milestone) \
    #         .exclude(file__submission__authors=django_user) \
    #         .prefetch_related('file__submission__authors') \
    #         .values('id', 'name', 'cluster_id', 'file__submission', 'class_type', 'student_lines')

    # using a correct raw SQL query instead..
    django_chunks = Chunk.objects \
        .raw("""
SELECT files.submission_id, `chunks`.`id`, `chunks`.`file_id`, `chunks`.`name`, `chunks`.`start`, `chunks`.`end`, `chunks`.`cluster_id`, `chunks`.`created`, `chunks`.`modified`, `chunks`.`class_type`, `chunks`.`staff_portion`, `chunks`.`student_lines`, `chunks`.`chunk_info`
FROM `chunks`
join files on chunks.file_id=files.id
join submissions on files.submission_id=submissions.id
join submissions_authors on submissions.id=submissions_authors.submission_id
where submissions.milestone_id=%s and not(user_id=%s)
            """, [submit_milestone.id, django_user.id])
    # preload the file.submission according to http://python.6.x6.nabble.com/Prefetch-related-data-when-using-raw-td4983196.html
    django_chunks = list(django_chunks)
    prefetch_related_objects(django_chunks, ['file__submission'])

    django_tasks = Task.objects.filter(
            submission__milestone=submit_milestone) \
            .exclude(submission__authors=django_user) \
                    .select_related('reviewer__user', 'chunk') \

    # load all submissions and chunks into lightweight internal objects
    django_submission_chunks = defaultdict(list)
    for chunk in django_chunks:
        django_submission_chunks[chunk.file.submission_id].append(chunk)

    for django_submission in django_submissions:
        submission = SubmissionForReview(
                id=django_submission.id,
                authors=[user_map[author.id] for author in django_submission.authors.all() if author.id in user_map],
                chunks=django_submission_chunks[django_submission.id])
        #logging.debug(django_submission.authors.all())
        #logging.debug(submission.authors)
        if not submission.chunks:
            # toss out any submissions without chunks
            continue
        submissions[submission.id] = submission
        chunks.extend(submission.chunks)
    #logging.debug(submissions)

    # load existing reviewing assignments
    for chunk in chunks:
        chunk_map[chunk.id] = chunk
    #logging.debug(chunk_map)

    for django_task in django_tasks:
        if django_task.chunk:
            chunk = chunk_map[django_task.chunk_id] # looks like dead code
            reviewer = user_map[django_task.reviewer_id]
            chunk_map[django_task.chunk_id].assign_reviewer(reviewer)

    return chunks
Exemplo n.º 43
0
def load_chunks(submit_milestone, user_map, django_user):
    chunks = []
    chunk_map = {}
    submissions = {}

    django_submissions = submit_milestone.submissions.exclude(
        authors=django_user).prefetch_related("authors")

    # the exclude() in the call below failed because of a Django bug.  Try submit_milestone=55, django_user=1016 on the production db.
    # django_chunks = Chunk.objects \
    #         .filter(file__submission__milestone=submit_milestone) \
    #         .exclude(file__submission__authors=django_user) \
    #         .prefetch_related('file__submission__authors') \
    #         .values('id', 'name', 'cluster_id', 'file__submission', 'class_type', 'student_lines')

    # using a correct raw SQL query instead..
    django_chunks = Chunk.objects \
        .raw("""
SELECT files.submission_id, `chunks`.`id`, `chunks`.`file_id`, `chunks`.`name`, `chunks`.`start`, `chunks`.`end`, `chunks`.`cluster_id`, `chunks`.`created`, `chunks`.`modified`, `chunks`.`class_type`, `chunks`.`staff_portion`, `chunks`.`student_lines`, `chunks`.`chunk_info`
FROM `chunks` 
join files on chunks.file_id=files.id
join submissions on files.submission_id=submissions.id
join submissions_authors on submissions.id=submissions_authors.submission_id
where submissions.milestone_id=%s and not(user_id=%s)
            """, [submit_milestone.id, django_user.id])
    # preload the file.submission according to http://python.6.x6.nabble.com/Prefetch-related-data-when-using-raw-td4983196.html
    django_chunks = list(django_chunks)
    prefetch_related_objects(django_chunks, ['file__submission'])

    django_tasks = Task.objects.filter(
            submission__milestone=submit_milestone) \
            .exclude(submission__authors=django_user) \
                    .select_related('reviewer', 'chunk') \

    # load all submissions and chunks into lightweight internal objects
    django_submission_chunks = defaultdict(list)
    for chunk in django_chunks:
        django_submission_chunks[chunk.file.submission_id].append(chunk)

    for django_submission in django_submissions:
        submission = SubmissionForReview(
            id=django_submission.id,
            authors=[
                user_map[author.id]
                for author in django_submission.authors.all()
                if author.id in user_map
            ],
            chunks=django_submission_chunks[django_submission.id])
        #logging.debug(django_submission.authors.all())
        #logging.debug(submission.authors)
        if not submission.chunks:
            # toss out any submissions without chunks
            continue
        submissions[submission.id] = submission
        chunks.extend(submission.chunks)
    logging.debug(submissions)

    # load existing reviewing assignments
    for chunk in chunks:
        chunk_map[chunk.id] = chunk
    #logging.debug(chunk_map)

    for django_task in django_tasks:
        if django_task.chunk:
            chunk = chunk_map[django_task.chunk_id]  # looks like dead code
            reviewer = user_map[django_task.reviewer_id]
            chunk_map[django_task.chunk_id].assign_reviewer(reviewer)

    return chunks
Exemplo n.º 44
0
 def prefetch(self, *related):
     prefetch_related_objects([self], *related)
 def test_unknown(self):
     book1 = Book.objects.get(id=self.book1.id)
     with self.assertRaises(AttributeError):
         prefetch_related_objects([book1], 'unknown_attribute')
Exemplo n.º 46
0
    def get_context_data(self, **kwargs):
        u = User.objects.none()
        if 'user' in kwargs:
            try:
                u = User.objects.get(pk=int(kwargs['user']))
            except:
                u = User.objects.get(username=kwargs['user'])

        elif self.request.GET.get('user', False):
            try:
                u = User.objects.get(pk=int(self.request.GET['user']))
            except:
                u = User.objects.get(username=self.request.GET['user'])

        elif not self.request.user.is_anonymous():
            # if the user is logged in and no user is specified, show logged in user
            u = User.objects.get(pk=int(self.request.user.id))

        # get all resources the profile user owns
        resources = u.uaccess.owned_resources
        # get a list of groupmembershiprequests
        group_membership_requests = GroupMembershipRequest.objects.filter(
            invitation_to=u).all()

        # if requesting user is not the profile user, then show only resources that the
        # requesting user has access
        if self.request.user != u:
            if self.request.user.is_authenticated():
                if self.request.user.is_superuser:
                    # admin can see all resources owned by profile user
                    pass
                else:
                    # filter out any resources the requesting user doesn't have access
                    resources = resources.filter(
                        Q(pk__in=self.request.user.uaccess.view_resources)
                        | Q(raccess__public=True)
                        | Q(raccess__discoverable=True))
            else:
                # for anonymous requesting user show only resources that are either public or
                # discoverable
                resources = resources.filter(
                    Q(raccess__public=True) | Q(raccess__discoverable=True))

        # get resource attributes used in profile page
        resources = resources.only('title', 'resource_type', 'created')
        # prefetch resource metadata elements
        meta_contenttypes = get_metadata_contenttypes()
        for ct in meta_contenttypes:
            # get a list of resources having metadata that is an instance of a specific
            # metadata class (e.g., CoreMetaData)
            res_list = [res for res in resources if res.content_type == ct]
            prefetch_related_objects(res_list,
                                     Prefetch('content_object__creators'),
                                     Prefetch('content_object___description'),
                                     Prefetch('content_object___title'))
        return {
            'profile_user': u,
            'resources': resources,
            'quota_message': get_quota_message(u),
            'group_membership_requests': group_membership_requests,
        }
Exemplo n.º 47
0
    def get_context_data(self, **kwargs):
        u = User.objects.none()
        if 'user' in kwargs:
            try:
                u = User.objects.get(pk=int(kwargs['user']))
            except:
                u = User.objects.get(username=kwargs['user'])

        elif self.request.GET.get('user', False):
            try:
                u = User.objects.get(pk=int(self.request.GET['user']))
            except:
                u = User.objects.get(username=self.request.GET['user'])

        elif not self.request.user.is_anonymous():
            # if the user is logged in and no user is specified, show logged in user
            u = User.objects.get(pk=int(self.request.user.id))

        # get all resources the profile user owns
        resources = u.uaccess.owned_resources
        # get a list of groupmembershiprequests
        group_membership_requests = GroupMembershipRequest.objects.filter(invitation_to=u).all()

        # if requesting user is not the profile user, then show only resources that the
        # requesting user has access
        if self.request.user != u:
            if self.request.user.is_authenticated():
                if self.request.user.is_superuser:
                    # admin can see all resources owned by profile user
                    pass
                else:
                    # filter out any resources the requesting user doesn't have access
                    resources = resources.filter(
                        Q(pk__in=self.request.user.uaccess.view_resources) |
                        Q(raccess__public=True) |
                        Q(raccess__discoverable=True))
            else:
                # for anonymous requesting user show only resources that are either public or
                # discoverable
                resources = resources.filter(Q(raccess__public=True) |
                                             Q(raccess__discoverable=True))

        # get resource attributes used in profile page
        resources = resources.only('title', 'resource_type', 'created')
        # prefetch resource metadata elements
        meta_contenttypes = get_metadata_contenttypes()
        for ct in meta_contenttypes:
            # get a list of resources having metadata that is an instance of a specific
            # metadata class (e.g., CoreMetaData)
            res_list = [res for res in resources if res.content_type == ct]
            prefetch_related_objects(res_list,
                                     Prefetch('content_object__creators'),
                                     Prefetch('content_object___description'),
                                     Prefetch('content_object___title')
                                     )
        return {
            'profile_user': u,
            'resources': resources,
            'quota_message': get_quota_message(u),
            'group_membership_requests': group_membership_requests,
        }