def test_foreignkey_reverse(self): books = list(Book.objects.all()) with self.assertNumQueries(1): prefetch_related_objects(books, 'first_time_authors') with self.assertNumQueries(0): [list(book.first_time_authors.all()) for book in books]
def ensure_caches(user): """ Ensure the cache for a given type. """ from django.db.models import query if (not (isinstance(user, Entity))): return blocked_key = cache_key('blocked', user.pk) blocked_entities = cache_manager.cache_get(blocked_key, sliding_timeout=DEFAULT_TIMEOUT) if (blocked_entities is None): query.prefetch_related_objects([user], 'blocked_entities') cache_manager.cache_set( blocked_key, user._prefetched_objects_cache['blocked_entities']) else: if (not (hasattr(user, '_prefetched_objects_cache'))): user._prefetched_objects_cache = {} user._prefetched_objects_cache['blocked_entities'] = blocked_entities blocking_key = cache_key('blocking', user.pk) blocking_entities = cache_manager.cache_get( blocking_key, sliding_timeout=DEFAULT_TIMEOUT) if (blocking_entities is None): query.prefetch_related_objects([user], 'blocking_entities') cache_manager.cache_set( blocking_key, user._prefetched_objects_cache['blocking_entities']) else: if (not (hasattr(user, '_prefetched_objects_cache'))): user._prefetched_objects_cache = {} user._prefetched_objects_cache['blocking_entities'] = blocking_entities
def retrieve(self, request, region, agency, pk): try: stop = Stop.objects.select_related('agency') \ .get(pk=Stop.create_id(Agency.create_id(region, agency), pk)) except Stop.DoesNotExist: raise Http404('No Route matches the given query.') agency = stop.agency arrivals = get_provider(stop.agency).arrivals(stop) # odd if we have a mixture of units, but we currently don't so... # sort the arrivals arrivals.sort(key=attrgetter('away')) # never return more than 15 arrivals, limit before the prefetch to # avoid extra work arrivals = arrivals[:15] prefetch_related_objects(arrivals, ['direction__route__directions']) routes = {} for arrival in arrivals: if arrival.direction: route = arrival.direction.route routes[route.id] = route self.object = StopAdapter(stop, arrivals=arrivals, routes=routes.values()) serializer = self.get_serializer(self.object) return Response(serializer.data)
def test_foreignkey_forward(self): authors = list(Author.objects.all()) with self.assertNumQueries(1): prefetch_related_objects(authors, 'first_book') with self.assertNumQueries(0): [author.first_book for author in authors]
def test_m2m_reverse(self): author1 = Author.objects.get(id=self.author1.id) with self.assertNumQueries(1): prefetch_related_objects([author1], 'books') with self.assertNumQueries(0): self.assertEqual(set(author1.books.all()), {self.book1, self.book2})
def test_prefetch_object(self): book1 = Book.objects.get(id=self.book1.id) with self.assertNumQueries(1): prefetch_related_objects([book1], Prefetch('authors')) with self.assertNumQueries(0): self.assertEqual(set(book1.authors.all()), {self.author1, self.author2, self.author3})
def to_container(self, container): """Container """ result = container(self.iterator()) if self._prefetch_related_lookups: prefetch_related_objects(result, *self._prefetch_related_lookups) return result
def send(self, instances): messages = [] if isinstance(instances, list): prefetch_related_objects(instances, ('equipier_set', )) elif hasattr(instances, 'prefetch_related'): instances.prefetch_related('equipiers') for instance in instances: dests = set() if self.destinataire in ('Organisateur', 'Tous'): dests.add(self.course.email_contact) if self.destinataire in ('Equipe', 'Tous') and isinstance(instance, Equipe): dests.add(instance.gerant_email) if self.destinataire in ('Equipiers', 'Tous'): if isinstance(instance, Equipier): dests.add(instance.email) if isinstance(instance, Equipe): for equipier in instance.equipier_set.filter(numero__lte=F('equipe__nombre')): dests.add(equipier.email) context = Context({ "instance": instance, 'ROOT_URL': 'http://%s' % Site.objects.get_current(), }) subject = Template(self.sujet).render(context) message = Template(self.message).render(context) bcc = [] if self.bcc: bcc = re.split('[,; ]+', self.bcc) for dest in dests: message = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL, [ dest ], bcc, reply_to=[self.course.email_contact,]) message.content_subtype = "html" messages.append(message) MailThread(messages).start()
def prefetch_related(tests): prefetch_related_objects( tests, 'test_run', 'test_run__environment', 'test_run__build', )
def get_queryset(self): # Retrieves the latest entry for each `format_requested`, for our content. # The `latest_events` sub-request retieves all events for the current content, and # and annotates them with their row number if we would select them by requested format, # ordered by descending date. The first one by date with the PDF type would be annotated # 1, the second with PDF, 2, etc. # The outer request selects everything in the inner request, but filtering only the first # row for each format, keeping only the latest record for each type for this content. # We have to use a sub-request as the SQL spec forbides to filter on a windowed element # directly. # # This uses raw SQL because even if Django supports windowed requests, it does not allow # to select _from_ a subrequest (« SELECT * FROM (SELECT … ) WHERE … »). exports = PublicationEvent.objects.raw( """ WITH latest_events AS ( SELECT p.*, ROW_NUMBER() OVER (PARTITION BY format_requested ORDER BY date DESC) AS row FROM tutorialv2_publicationevent AS p INNER JOIN tutorialv2_publishedcontent published ON (p.published_object_id = published.id) WHERE published.content_id = %s ) SELECT * FROM latest_events WHERE row = 1""", [int(self.kwargs.get("pk", 0))], ) prefetch_related_objects(exports, "published_object") return exports
def _fill_obj(obj): if included: if not hasattr(obj, '_default_translatable_fields'): obj._default_translatable_fields = { field: getattr(obj, field) for field in type( obj)._get_translatable_fields_names() } object_id = str(obj.pk) instances[object_id] = obj nonlocal query query |= models.Q( content_type__id=content_type_id, object_id=object_id, ) if hierarchy: for (relation, detail) in hierarchy.items(): value = getattr(obj, relation, None) if value is not None: if isinstance(value, models.Manager): if not (hasattr(obj, '_prefetched_objects_cache') and relation in obj._prefetched_objects_cache): prefetch_related_objects([obj], relation) value = value.all() _fill_entity( entity=value, hierarchy=detail['relations'], included=detail['included'], )
def last_actions(self, release, limit, language=None, user=None): sqlu = "SELECT pl.pofile_id, max(pl.id) as id FROM pofile_log pl" if user: sqlu += " LEFT OUTER JOIN pofile_assigns a ON pl.pofile_id = a.pofile_id" sqlu += " WHERE a.translate_id=%(userid)s or a.review_id=%(userid)s" % {'userid': user.id} else: try: bot = User.objects.get(username=getattr(settings, 'BOT_USERNAME','bot')) except: bot = None if bot: sqlu += " WHERE pl.user_id <> %d" % bot.pk sqlu += " group by pl.pofile_id" sql = "SELECT p.id, p.pofile_id, p.created, p.user_id, p.action, p.comment FROM pofile_log p INNER JOIN (%s) p2 "\ "INNER JOIN pofile f ON p.id = p2.id AND f.id = p.pofile_id" % sqlu sql += " WHERE f.release_id = %d" % release.id if language: sql += " AND f.language_id=%d" % language.id sql += " ORDER BY p.id DESC LIMIT %d" % limit from django.db.models.query import prefetch_related_objects result = list(self.raw(sql)) prefetch_related_objects(result, ('user',)) return result
def get_attrs(self, item_list, user): prefetch_related_objects(item_list, "created_by") result = defaultdict(lambda: {"created_by": {}}) user_serializer = UserSerializer() serialized_users = { user["id"]: user for user in serialize( [ discover_saved_query.created_by for discover_saved_query in item_list if discover_saved_query.created_by ], user=user, serializer=user_serializer, ) } for discover_saved_query in item_list: result[discover_saved_query]["created_by"] = serialized_users.get( str(discover_saved_query.created_by_id) ) return result
def test_prefetch_queryset(self): book1 = Book.objects.get(id=self.book1.id) with self.assertNumQueries(1): prefetch_related_objects( [book1], Prefetch('authors', queryset=Author.objects.filter(id__in=[self.author1.id, self.author2.id])) ) with self.assertNumQueries(0): self.assertEqual(set(book1.authors.all()), {self.author1, self.author2})
def prefetch_related(tests): prefetch_related_objects( tests, 'known_issues', 'test_run', 'test_run__environment', 'test_run__build', 'test_run__build__project', 'test_run__build__project__group', )
def prefetch_related(builds): prefetch_related_objects( builds, 'project', 'project__group', 'test_runs', 'test_runs__environment', 'test_runs__tests', 'test_runs__tests__suite', )
def _prefetch_related_objects(self): things = defaultdict(list) for index, thing in enumerate(self._result_cache, start=0): things[thing.__class__].append(thing) for klass, instances in things.items(): prefetches = self._prefetch_related_lookups[:] if klass in self._our_prefetches: prefetches.extend(self._our_prefetches[klass]) prefetch_related_objects(instances, prefetches) return None
def get_attrs(self, item_list, user): item_dict = {i.id: i for i in item_list} prefetch_related_objects(item_list, "created_by") widgets = (DashboardWidget.objects.filter( dashboard_id__in=item_dict.keys()).order_by("order").values( "dashboard_id", "order", "display_type", "detail", "id")) result = defaultdict(lambda: { "widget_display": [], "widget_preview": [], "created_by": {} }) for widget in widgets: dashboard = item_dict[widget["dashboard_id"]] display_type = DashboardWidgetDisplayTypes.get_type_name( widget["display_type"]) result[dashboard]["widget_display"].append(display_type) for widget in widgets: dashboard = item_dict[widget["dashboard_id"]] widget_preview = { "displayType": DashboardWidgetDisplayTypes.get_type_name( widget["display_type"]), "layout": None, } if widget.get("detail"): detail = json.loads(widget["detail"]) if detail.get("layout"): widget_preview["layout"] = detail["layout"] result[dashboard]["widget_preview"].append(widget_preview) user_serializer = UserSerializer() serialized_users = { user["id"]: user for user in serialize( [ dashboard.created_by for dashboard in item_list if dashboard.created_by ], user=user, serializer=user_serializer, ) } for dashboard in item_dict.values(): result[dashboard]["created_by"] = serialized_users.get( str(dashboard.created_by_id)) return result
def __get_tests__(self, builds, metadata): test_runs = [] for build in builds: for test_run in build.test_runs.all(): test_runs.append(test_run) tests = [] for test_runs_batch in split_list(test_runs, chunk_size=100): prefetch_related_objects(test_runs_batch, Prefetch('tests', queryset=Test.objects.filter(metadata=metadata).prefetch_related('metadata').order_by())) for test_run in test_runs_batch: tests += test_run.tests.all() return tests
def test_prefetch_queryset(self): book1 = Book.objects.get(id=self.book1.id) with self.assertNumQueries(1): prefetch_related_objects( [book1], Prefetch('authors', queryset=Author.objects.filter( id__in=[self.author1.id, self.author2.id]))) with self.assertNumQueries(0): self.assertEqual(set(book1.authors.all()), {self.author1, self.author2})
def _get_sorted_occurrences(self): occurrences = [] if hasattr(self, 'occurrence_pool') and self.occurrence_pool is not None: for occurrence in self.occurrence_pool: if occurrence.start <= self.utc_end and occurrence.end >= self.utc_start: occurrences.append(occurrence) return occurrences prefetch_related_objects(self.events, 'occurrence_set') for event in self.events: event_occurrences = event.get_occurrences(self.start, self.end, clear_prefetch=False) occurrences += event_occurrences return sorted(occurrences, **self.sorting_options)
def retrieve(self, request, region=None, agency=None, *args, **kwargs): try: lat = float(request.GET['lat']) lon = float(request.GET['lon']) radius = float(request.GET.get('radius', 500.0)) if radius < 0 or 5000 < radius: # TODO: bad request, invalid param raise Exception('invalid radius') except KeyError: # TODO: bad request, missing param raise except ValueError: # TODO: bad request, bad param raise # we'll at most return N stops # TODO: limiting before filtering is bad stops = Stop.objects.nearby(lat, lon, radius)[:20] if region and agency: agency_id = Agency.create_id(region, agency) def desired_agency(stop): return stop.agency_id == agency_id stops = filter(desired_agency, stops) prefetch_related_objects( stops, ['agency', 'agency__region', 'directions', 'directions__route']) routes = {} agencies = {} regions = {} for stop in stops: agency = stop.agency agencies[agency.id] = agency region = agency.region regions[region.id] = region for direction in stop.directions.all(): route = direction.route routes[route.id] = direction.route self.object = Adapter(self, agencies=agencies.values(), regions=regions.values(), routes=routes.values(), stops=stops) serializer = self.get_serializer(self.object) return Response(serializer.data)
def has_permission(self, request): if not request.user.is_authenticated(): return False prefetch_related_objects([request.user], ('accreditations__course', )) if request.path.endswith('/logout/'): return True if request.user.is_superuser: return True if request.path.endswith('/choose/') or request.path.endswith('/ask/') or re.search(r'/ask/[^/]+/$', request.path) or request.path.endswith('/inscriptions/course/add/') or request.path.endswith('/course/jsi18n/'): return True if 'course_uid' not in request.COOKIES: return False course_uid = request.COOKIES['course_uid'] return request.user.accreditations.filter(course__uid=course_uid).exclude(role='').count() > 0
def retrieve(self, request, region=None, agency=None, *args, **kwargs): try: lat = float(request.GET['lat']) lon = float(request.GET['lon']) radius = float(request.GET.get('radius', 500.0)) if radius < 0 or 5000 < radius: # TODO: bad request, invalid param raise Exception('invalid radius') except KeyError: # TODO: bad request, missing param raise except ValueError: # TODO: bad request, bad param raise # we'll at most return N stops # TODO: limiting before filtering is bad stops = Stop.objects.nearby(lat, lon, radius)[:20] if region and agency: agency_id = Agency.create_id(region, agency) def desired_agency(stop): return stop.agency_id == agency_id stops = filter(desired_agency, stops) prefetch_related_objects(stops, ['agency', 'agency__region', 'directions', 'directions__route']) routes = {} agencies = {} regions = {} for stop in stops: agency = stop.agency agencies[agency.id] = agency region = agency.region regions[region.id] = region for direction in stop.directions.all(): route = direction.route routes[route.id] = direction.route self.object = Adapter(self, agencies=agencies.values(), regions=regions.values(), routes=routes.values(), stops=stops) serializer = self.get_serializer(self.object) return Response(serializer.data)
def _get_sorted_occurrences(self): occurrences = [] if hasattr(self, 'occurrence_pool') and self.occurrence_pool is not None: for occurrence in self.occurrence_pool: if occurrence.start <= self.utc_end and occurrence.end >= self.utc_start: occurrences.append(occurrence) return occurrences try: prefetch_related_objects(self.events, 'occurrence_set') except AttributeError: prefetch_related_objects(self.events, ['occurrence_set']) for event in self.events: event_occurrences = event.get_occurrences(self.start, self.end, clear_prefetch=False) occurrences += event_occurrences return sorted(occurrences, **self.sorting_options)
def get_context_data(self, *args, **kwargs): ret = super().get_context_data(*args, **kwargs) user = self.request.user membre = user.membre if user.is_staff and 'membre_id' in self.kwargs: membre = models.Membre.objects.get(id=self.kwargs['membre_id']) ret['membre'] = membre prefetch_related_objects( [membre], Prefetch( 'licences', models.Licence.objects.annotate( _montant_paiement=Sum('paiements__montant')))) ret['STRIPE_PUBLIC_KEY'] = settings.STRIPE_PUBLIC_KEY ret['saisons'] = models.Saison.objects.filter(ouvert=True).exclude( membres__in=membre.licences.all()) return ret
def update(self, request, *args, **kwargs): partial = kwargs.pop('partial', False) instance = self.get_object() serializer = self.get_serializer(instance, data=request.data, partial=partial) serializer.is_valid(raise_exception=True) self.perform_update(serializer) if getattr(instance, '_prefetched_objects_cache', None): # If 'prefetch_related' has been applied to a queryset, we need to # forcibly invalidate the prefetch cache on the instance, # and then re-prefetch related objects instance._prefetched_objects_cache = {} prefetch_related_objects([instance], *self.get_prefetch_related()) return Response(serializer.data)
def test_m2m_then_m2m(self): """ We can follow a m2m and another m2m. """ authors = list(Author.objects.all()) with self.assertNumQueries(2): prefetch_related_objects(authors, 'books__read_by') with self.assertNumQueries(0): self.assertEqual( [[[str(r) for r in b.read_by.all()] for b in a.books.all()] for a in authors], [ [['Amy'], ['Belinda']], # Charlotte - Poems, Jane Eyre [['Amy']], # Anne - Poems [['Amy'], []], # Emily - Poems, Wuthering Heights [['Amy', 'Belinda']], # Jane - Sense and Sense ])
def list(self, request, agency, lat, lon, *args, **kwargs): lat = float(lat) lon = float(lon) max = float(request.GET.get('max', 0.25)) if max < 0: max = 0.25 elif 1 < max: max = 1 radius = max * meters_per_mile # we'll at most return n stops # TODO: limiting before filtering is bad stops = Stop.objects.nearby(lat, lon, radius)[:25] prefetch_related_objects(stops, ['agency', 'directions', 'directions__route']) results = [] for stop in stops: # this is kind of ugly, but should do the trick since we don't know # the region if not stop.agency_id.endswith(agency): continue for direction in stop.directions.all(): results.append({ 'agency_tag': stop.agency.get_id(), 'direction_tag': direction.get_id(), 'route_tag': direction.route.get_id(), 'stop_tag': stop.get_id(), 'stop_id': stop.code, 'route_title': direction.route.name, 'direction_title': direction.name, 'stop_title': stop.name, 'distance': stop.distance / meters_per_mile, 'lat': stop.lat, 'lon': stop.lon }) if len(results) > 25: return Response(results) return Response(results)
def test_m2m_then_m2m(self): """ We can follow a m2m and another m2m. """ authors = list(Author.objects.all()) with self.assertNumQueries(2): prefetch_related_objects(authors, 'books__read_by') with self.assertNumQueries(0): self.assertEqual( [ [[str(r) for r in b.read_by.all()] for b in a.books.all()] for a in authors ], [ [['Amy'], ['Belinda']], # Charlotte - Poems, Jane Eyre [['Amy']], # Anne - Poems [['Amy'], []], # Emily - Poems, Wuthering Heights [['Amy', 'Belinda']], # Jane - Sense and Sense ] )
def listar_desafiaveis(request, username): jogador = get_object_or_404(Jogador, user__username=username) posicao_atual_jogador = jogador.posicao_em(timezone.localtime()) desafiaveis = buscar_desafiaveis(jogador, timezone.localtime(), False, False) # Buscar usuários de desafiáveis prefetch_related_objects(desafiaveis, 'user') desafios_pendentes = DesafioLadder.objects.filter(cancelamentodesafioladder__isnull=True, admin_validador__isnull=True, score_desafiante__gt=F('score_desafiado'), posicao_desafiado__lte=posicao_atual_jogador, posicao_desafiante__gte=posicao_atual_jogador) \ .select_related('desafiante', 'desafiado') return render( request, 'jogadores/listar_desafiaveis.html', { 'desafiaveis': desafiaveis, 'desafios_pendentes': desafios_pendentes, 'jogador': jogador })
def create(self, validated_data): """Create instances in bulk efficiently""" ModelClass = self.child.Meta.model # pylint: disable=invalid-name info = model_meta.get_field_info(ModelClass) m2m_fields = [f for f, i in info.relations.items() if i.to_many] instances = [] m2m_values = [] for attrs in validated_data: m2m_values.append({}) for field in m2m_fields: # m2m fields must be set after the instance is created m2m_values[-1][field] = attrs.pop(field, None) if hasattr(self.child, "bulk_create_attrs"): attrs = self.child.bulk_create_attrs(attrs) instances.append(ModelClass(**attrs)) # create the instances in bulk for efficiency try: ModelClass.objects.bulk_create(instances) except IntegrityError as exc: raise ValidationError(exc) # set any m2m values for instance, m2m_values_ in zip(instances, m2m_values): for field_name, value in m2m_values_.items(): if value: field = getattr(instance, field_name) field.set(value) # prefetch any m2m values which are serializer fields to avoid n+1 queries prefetch_fields = set(m2m_fields) & set(self.child.Meta.fields) prefetch_related_objects(instances, *prefetch_fields) return instances
def test_unknown(self): book1 = Book.objects.get(id=self.book1.id) with self.assertRaises(AttributeError): prefetch_related_objects([book1], 'unknown_attribute')
def get_person_as_version_data(person, new_person=False): """ If new_person is True then skip some DB checks that we know will be empty. This should reduce the number of queries needed. """ # Prefetch to reduce the number of queries prefetch_related_objects([person], "tmp_person_identifiers") result = {} result["id"] = str(person.id) # Add PersonIdentifier fields in to the global version namespace # TODO: move these to dict to make it easier to merge/split/revert existing_identifiers = { identifier.value_type: identifier.value for identifier in person.tmp_person_identifiers.all() } for field in PersonIdentifierFields: result[field.name] = existing_identifiers.get(field.name, "") for field in settings.SIMPLE_POPOLO_FIELDS: result[field.name] = getattr(person, field.name) or "" # Add legacy identifiers # TODO: these should use the PersonIdenfitiers model and value types, # but this code emulates the legacy way of adding IDs. if person.get_single_identifier_of_type("theyworkforyou"): result["identifiers"] = [] new_id = person.get_single_identifier_of_type( "theyworkforyou").internal_identifier if "publicwhip" not in new_id: new_id = "uk.org.publicwhip/person/{}".format(new_id) result["identifiers"].append({ "identifier": new_id, "scheme": "uk.org.publicwhip" }) if person.get_single_identifier_of_type("twitter_username"): result["identifiers"] = result.get("identifiers", []) result["identifiers"].append({ "identifier": person.get_single_identifier_of_type( "twitter_username").internal_identifier, "scheme": "twitter", }) result["other_names"] = [] candidacies = {} not_standing = [] if not new_person: result["other_names"] = [{ "name": on.name, "note": on.note, "start_date": on.start_date, "end_date": on.end_date, } for on in person.other_names.order_by( "name", "start_date", "end_date")] for membership in person.memberships.all(): ballot = membership.ballot candidacy = {"party": membership.party.ec_id} if membership.elected is not None: candidacy["elected"] = membership.elected if membership.party_list_position is not None: candidacy[ "party_list_position"] = membership.party_list_position candidacies[ballot.ballot_paper_id] = candidacy for not_standing_in_election in person.not_standing.all(): not_standing.append(not_standing_in_election.slug) # Add `favourite_biscuits` to an `extra_fields` key # to re-produce the previous ExtraField model. # This is done like this to save changing the version diff # for exery edit to move to key to the parent object. # In the future we will have to run a script to move all the # keys to wherever we want them. extra_fields = {"favourite_biscuits": person.favourite_biscuit or ""} result["extra_fields"] = extra_fields result["candidacies"] = candidacies if not_standing: result["not_standing"] = not_standing return result
def prefetch(self, *related): prefetch_related_objects([self], *related)
def get_nodes(self, request): from cms.models import Title site = self.renderer.site lang = self.renderer.request_language pages = get_page_queryset( site, draft=self.renderer.draft_mode_active, published=not self.renderer.draft_mode_active, ) if is_valid_site_language(lang, site_id=site.pk): _valid_language = True _hide_untranslated = hide_untranslated(lang, site.pk) else: _valid_language = False _hide_untranslated = False if _valid_language: # The request language has been explicitly configured # for the current site. if _hide_untranslated: fallbacks = [] else: fallbacks = get_fallback_languages(lang, site_id=site.pk) languages = [lang ] + [_lang for _lang in fallbacks if _lang != lang] else: # The request language is not configured for the current site. # Fallback to all configured public languages for the current site. languages = get_public_languages(site.pk) fallbacks = languages pages = (pages.filter(title_set__language__in=languages). select_related('node').order_by('node__path').distinct()) if not self.renderer.draft_mode_active: # we're dealing with public pages. # prefetch the draft versions. pages = pages.select_related('publisher_public__node') pages = get_visible_nodes(request, pages, site) if not pages: return [] try: homepage = [page for page in pages if page.is_home][0] except IndexError: homepage = None titles = Title.objects.filter( language__in=languages, publisher_is_draft=self.renderer.draft_mode_active, ) lookup = Prefetch( 'title_set', to_attr='filtered_translations', queryset=titles, ) prefetch_related_objects(pages, lookup) # Build the blank title instances only once blank_title_cache = { language: EmptyTitle(language=language) for language in languages } if lang not in blank_title_cache: blank_title_cache[lang] = EmptyTitle(language=lang) # Maps a node id to its page id node_id_to_page = {} def _page_to_node(page): # EmptyTitle is used to prevent the cms from trying # to find a translation in the database page.title_cache = blank_title_cache.copy() for trans in page.filtered_translations: page.title_cache[trans.language] = trans menu_node = get_menu_node_for_page( self.renderer, page, language=lang, fallbacks=fallbacks, ) return menu_node menu_nodes = [] for page in pages: node = page.node parent_id = node_id_to_page.get(node.parent_id) if node.parent_id and not parent_id: # If the parent page is not available (unpublished, etc..) # don't bother creating menu nodes for its descendants. continue menu_node = _page_to_node(page) cut_homepage = homepage and not homepage.in_navigation if cut_homepage and parent_id == homepage.pk: # When the homepage is hidden from navigation, # we need to cut all its direct children from it. menu_node.parent_id = None else: menu_node.parent_id = parent_id node_id_to_page[node.pk] = page.pk menu_nodes.append(menu_node) return menu_nodes
def load_chunks(submit_milestone, user_map, django_user): chunks = [] chunk_map = {} submissions = {} django_submissions = submit_milestone.submissions.exclude(authors=django_user).prefetch_related("authors") # the exclude() in the call below failed because of a Django bug. Try submit_milestone=55, django_user=1016 on the production db. # django_chunks = Chunk.objects \ # .filter(file__submission__milestone=submit_milestone) \ # .exclude(file__submission__authors=django_user) \ # .prefetch_related('file__submission__authors') \ # .values('id', 'name', 'cluster_id', 'file__submission', 'class_type', 'student_lines') # using a correct raw SQL query instead.. django_chunks = Chunk.objects \ .raw(""" SELECT files.submission_id, `chunks`.`id`, `chunks`.`file_id`, `chunks`.`name`, `chunks`.`start`, `chunks`.`end`, `chunks`.`cluster_id`, `chunks`.`created`, `chunks`.`modified`, `chunks`.`class_type`, `chunks`.`staff_portion`, `chunks`.`student_lines`, `chunks`.`chunk_info` FROM `chunks` join files on chunks.file_id=files.id join submissions on files.submission_id=submissions.id join submissions_authors on submissions.id=submissions_authors.submission_id where submissions.milestone_id=%s and not(user_id=%s) """, [submit_milestone.id, django_user.id]) # preload the file.submission according to http://python.6.x6.nabble.com/Prefetch-related-data-when-using-raw-td4983196.html django_chunks = list(django_chunks) prefetch_related_objects(django_chunks, ['file__submission']) django_tasks = Task.objects.filter( submission__milestone=submit_milestone) \ .exclude(submission__authors=django_user) \ .select_related('reviewer__user', 'chunk') \ # load all submissions and chunks into lightweight internal objects django_submission_chunks = defaultdict(list) for chunk in django_chunks: django_submission_chunks[chunk.file.submission_id].append(chunk) for django_submission in django_submissions: submission = SubmissionForReview( id=django_submission.id, authors=[user_map[author.id] for author in django_submission.authors.all() if author.id in user_map], chunks=django_submission_chunks[django_submission.id]) #logging.debug(django_submission.authors.all()) #logging.debug(submission.authors) if not submission.chunks: # toss out any submissions without chunks continue submissions[submission.id] = submission chunks.extend(submission.chunks) #logging.debug(submissions) # load existing reviewing assignments for chunk in chunks: chunk_map[chunk.id] = chunk #logging.debug(chunk_map) for django_task in django_tasks: if django_task.chunk: chunk = chunk_map[django_task.chunk_id] # looks like dead code reviewer = user_map[django_task.reviewer_id] chunk_map[django_task.chunk_id].assign_reviewer(reviewer) return chunks
def load_chunks(submit_milestone, user_map, django_user): chunks = [] chunk_map = {} submissions = {} django_submissions = submit_milestone.submissions.exclude( authors=django_user).prefetch_related("authors") # the exclude() in the call below failed because of a Django bug. Try submit_milestone=55, django_user=1016 on the production db. # django_chunks = Chunk.objects \ # .filter(file__submission__milestone=submit_milestone) \ # .exclude(file__submission__authors=django_user) \ # .prefetch_related('file__submission__authors') \ # .values('id', 'name', 'cluster_id', 'file__submission', 'class_type', 'student_lines') # using a correct raw SQL query instead.. django_chunks = Chunk.objects \ .raw(""" SELECT files.submission_id, `chunks`.`id`, `chunks`.`file_id`, `chunks`.`name`, `chunks`.`start`, `chunks`.`end`, `chunks`.`cluster_id`, `chunks`.`created`, `chunks`.`modified`, `chunks`.`class_type`, `chunks`.`staff_portion`, `chunks`.`student_lines`, `chunks`.`chunk_info` FROM `chunks` join files on chunks.file_id=files.id join submissions on files.submission_id=submissions.id join submissions_authors on submissions.id=submissions_authors.submission_id where submissions.milestone_id=%s and not(user_id=%s) """, [submit_milestone.id, django_user.id]) # preload the file.submission according to http://python.6.x6.nabble.com/Prefetch-related-data-when-using-raw-td4983196.html django_chunks = list(django_chunks) prefetch_related_objects(django_chunks, ['file__submission']) django_tasks = Task.objects.filter( submission__milestone=submit_milestone) \ .exclude(submission__authors=django_user) \ .select_related('reviewer', 'chunk') \ # load all submissions and chunks into lightweight internal objects django_submission_chunks = defaultdict(list) for chunk in django_chunks: django_submission_chunks[chunk.file.submission_id].append(chunk) for django_submission in django_submissions: submission = SubmissionForReview( id=django_submission.id, authors=[ user_map[author.id] for author in django_submission.authors.all() if author.id in user_map ], chunks=django_submission_chunks[django_submission.id]) #logging.debug(django_submission.authors.all()) #logging.debug(submission.authors) if not submission.chunks: # toss out any submissions without chunks continue submissions[submission.id] = submission chunks.extend(submission.chunks) logging.debug(submissions) # load existing reviewing assignments for chunk in chunks: chunk_map[chunk.id] = chunk #logging.debug(chunk_map) for django_task in django_tasks: if django_task.chunk: chunk = chunk_map[django_task.chunk_id] # looks like dead code reviewer = user_map[django_task.reviewer_id] chunk_map[django_task.chunk_id].assign_reviewer(reviewer) return chunks
def get_context_data(self, **kwargs): u = User.objects.none() if 'user' in kwargs: try: u = User.objects.get(pk=int(kwargs['user'])) except: u = User.objects.get(username=kwargs['user']) elif self.request.GET.get('user', False): try: u = User.objects.get(pk=int(self.request.GET['user'])) except: u = User.objects.get(username=self.request.GET['user']) elif not self.request.user.is_anonymous(): # if the user is logged in and no user is specified, show logged in user u = User.objects.get(pk=int(self.request.user.id)) # get all resources the profile user owns resources = u.uaccess.owned_resources # get a list of groupmembershiprequests group_membership_requests = GroupMembershipRequest.objects.filter( invitation_to=u).all() # if requesting user is not the profile user, then show only resources that the # requesting user has access if self.request.user != u: if self.request.user.is_authenticated(): if self.request.user.is_superuser: # admin can see all resources owned by profile user pass else: # filter out any resources the requesting user doesn't have access resources = resources.filter( Q(pk__in=self.request.user.uaccess.view_resources) | Q(raccess__public=True) | Q(raccess__discoverable=True)) else: # for anonymous requesting user show only resources that are either public or # discoverable resources = resources.filter( Q(raccess__public=True) | Q(raccess__discoverable=True)) # get resource attributes used in profile page resources = resources.only('title', 'resource_type', 'created') # prefetch resource metadata elements meta_contenttypes = get_metadata_contenttypes() for ct in meta_contenttypes: # get a list of resources having metadata that is an instance of a specific # metadata class (e.g., CoreMetaData) res_list = [res for res in resources if res.content_type == ct] prefetch_related_objects(res_list, Prefetch('content_object__creators'), Prefetch('content_object___description'), Prefetch('content_object___title')) return { 'profile_user': u, 'resources': resources, 'quota_message': get_quota_message(u), 'group_membership_requests': group_membership_requests, }
def get_context_data(self, **kwargs): u = User.objects.none() if 'user' in kwargs: try: u = User.objects.get(pk=int(kwargs['user'])) except: u = User.objects.get(username=kwargs['user']) elif self.request.GET.get('user', False): try: u = User.objects.get(pk=int(self.request.GET['user'])) except: u = User.objects.get(username=self.request.GET['user']) elif not self.request.user.is_anonymous(): # if the user is logged in and no user is specified, show logged in user u = User.objects.get(pk=int(self.request.user.id)) # get all resources the profile user owns resources = u.uaccess.owned_resources # get a list of groupmembershiprequests group_membership_requests = GroupMembershipRequest.objects.filter(invitation_to=u).all() # if requesting user is not the profile user, then show only resources that the # requesting user has access if self.request.user != u: if self.request.user.is_authenticated(): if self.request.user.is_superuser: # admin can see all resources owned by profile user pass else: # filter out any resources the requesting user doesn't have access resources = resources.filter( Q(pk__in=self.request.user.uaccess.view_resources) | Q(raccess__public=True) | Q(raccess__discoverable=True)) else: # for anonymous requesting user show only resources that are either public or # discoverable resources = resources.filter(Q(raccess__public=True) | Q(raccess__discoverable=True)) # get resource attributes used in profile page resources = resources.only('title', 'resource_type', 'created') # prefetch resource metadata elements meta_contenttypes = get_metadata_contenttypes() for ct in meta_contenttypes: # get a list of resources having metadata that is an instance of a specific # metadata class (e.g., CoreMetaData) res_list = [res for res in resources if res.content_type == ct] prefetch_related_objects(res_list, Prefetch('content_object__creators'), Prefetch('content_object___description'), Prefetch('content_object___title') ) return { 'profile_user': u, 'resources': resources, 'quota_message': get_quota_message(u), 'group_membership_requests': group_membership_requests, }