def test_foreignkey_reverse(self):
books = list(Book.objects.all())
with self.assertNumQueries(1):
prefetch_related_objects(books, 'first_time_authors')
with self.assertNumQueries(0):
[list(book.first_time_authors.all()) for book in books]
def ensure_caches(user):
"""
Ensure the cache for a given type.
"""
from django.db.models import query
if (not (isinstance(user, Entity))):
return
blocked_key = cache_key('blocked', user.pk)
blocked_entities = cache_manager.cache_get(blocked_key,
sliding_timeout=DEFAULT_TIMEOUT)
if (blocked_entities is None):
query.prefetch_related_objects([user], 'blocked_entities')
cache_manager.cache_set(
blocked_key, user._prefetched_objects_cache['blocked_entities'])
else:
if (not (hasattr(user, '_prefetched_objects_cache'))):
user._prefetched_objects_cache = {}
user._prefetched_objects_cache['blocked_entities'] = blocked_entities
blocking_key = cache_key('blocking', user.pk)
blocking_entities = cache_manager.cache_get(
blocking_key, sliding_timeout=DEFAULT_TIMEOUT)
if (blocking_entities is None):
query.prefetch_related_objects([user], 'blocking_entities')
cache_manager.cache_set(
blocking_key, user._prefetched_objects_cache['blocking_entities'])
else:
if (not (hasattr(user, '_prefetched_objects_cache'))):
user._prefetched_objects_cache = {}
user._prefetched_objects_cache['blocking_entities'] = blocking_entities
def retrieve(self, request, region, agency, pk):
try:
stop = Stop.objects.select_related('agency') \
.get(pk=Stop.create_id(Agency.create_id(region, agency), pk))
except Stop.DoesNotExist:
raise Http404('No Route matches the given query.')
agency = stop.agency
arrivals = get_provider(stop.agency).arrivals(stop)
# odd if we have a mixture of units, but we currently don't so...
# sort the arrivals
arrivals.sort(key=attrgetter('away'))
# never return more than 15 arrivals, limit before the prefetch to
# avoid extra work
arrivals = arrivals[:15]
prefetch_related_objects(arrivals, ['direction__route__directions'])
routes = {}
for arrival in arrivals:
if arrival.direction:
route = arrival.direction.route
routes[route.id] = route
self.object = StopAdapter(stop, arrivals=arrivals,
routes=routes.values())
serializer = self.get_serializer(self.object)
return Response(serializer.data)
def test_foreignkey_reverse(self):
books = list(Book.objects.all())
with self.assertNumQueries(1):
prefetch_related_objects(books, 'first_time_authors')
with self.assertNumQueries(0):
[list(book.first_time_authors.all()) for book in books]
def test_foreignkey_forward(self):
authors = list(Author.objects.all())
with self.assertNumQueries(1):
prefetch_related_objects(authors, 'first_book')
with self.assertNumQueries(0):
[author.first_book for author in authors]
def test_m2m_reverse(self):
author1 = Author.objects.get(id=self.author1.id)
with self.assertNumQueries(1):
prefetch_related_objects([author1], 'books')
with self.assertNumQueries(0):
self.assertEqual(set(author1.books.all()), {self.book1, self.book2})
def test_prefetch_object(self):
book1 = Book.objects.get(id=self.book1.id)
with self.assertNumQueries(1):
prefetch_related_objects([book1], Prefetch('authors'))
with self.assertNumQueries(0):
self.assertEqual(set(book1.authors.all()), {self.author1, self.author2, self.author3})
def to_container(self, container):
"""Container """
result = container(self.iterator())
if self._prefetch_related_lookups:
prefetch_related_objects(result, *self._prefetch_related_lookups)
return result
def send(self, instances):
messages = []
if isinstance(instances, list):
prefetch_related_objects(instances, ('equipier_set', ))
elif hasattr(instances, 'prefetch_related'):
instances.prefetch_related('equipiers')
for instance in instances:
dests = set()
if self.destinataire in ('Organisateur', 'Tous'):
dests.add(self.course.email_contact)
if self.destinataire in ('Equipe', 'Tous') and isinstance(instance, Equipe):
dests.add(instance.gerant_email)
if self.destinataire in ('Equipiers', 'Tous'):
if isinstance(instance, Equipier):
dests.add(instance.email)
if isinstance(instance, Equipe):
for equipier in instance.equipier_set.filter(numero__lte=F('equipe__nombre')):
dests.add(equipier.email)
context = Context({
"instance": instance,
'ROOT_URL': 'http://%s' % Site.objects.get_current(),
})
subject = Template(self.sujet).render(context)
message = Template(self.message).render(context)
bcc = []
if self.bcc:
bcc = re.split('[,; ]+', self.bcc)
for dest in dests:
message = EmailMessage(subject, message, settings.DEFAULT_FROM_EMAIL, [ dest ], bcc, reply_to=[self.course.email_contact,])
message.content_subtype = "html"
messages.append(message)
MailThread(messages).start()
def prefetch_related(tests):
prefetch_related_objects(
tests,
'test_run',
'test_run__environment',
'test_run__build',
)
def test_foreignkey_forward(self):
authors = list(Author.objects.all())
with self.assertNumQueries(1):
prefetch_related_objects(authors, 'first_book')
with self.assertNumQueries(0):
[author.first_book for author in authors]
def get_queryset(self):
# Retrieves the latest entry for each `format_requested`, for our content.
# The `latest_events` sub-request retieves all events for the current content, and
# and annotates them with their row number if we would select them by requested format,
# ordered by descending date. The first one by date with the PDF type would be annotated
# 1, the second with PDF, 2, etc.
# The outer request selects everything in the inner request, but filtering only the first
# row for each format, keeping only the latest record for each type for this content.
# We have to use a sub-request as the SQL spec forbides to filter on a windowed element
# directly.
#
# This uses raw SQL because even if Django supports windowed requests, it does not allow
# to select _from_ a subrequest (« SELECT * FROM (SELECT … ) WHERE … »).
exports = PublicationEvent.objects.raw(
"""
WITH latest_events AS (
SELECT p.*, ROW_NUMBER() OVER (PARTITION BY format_requested ORDER BY date DESC) AS row
FROM tutorialv2_publicationevent AS p
INNER JOIN tutorialv2_publishedcontent published ON (p.published_object_id = published.id)
WHERE published.content_id = %s
)
SELECT * FROM latest_events WHERE row = 1""",
[int(self.kwargs.get("pk", 0))],
)
prefetch_related_objects(exports, "published_object")
return exports
def _fill_obj(obj):
if included:
if not hasattr(obj, '_default_translatable_fields'):
obj._default_translatable_fields = {
field: getattr(obj, field)
for field in type(
obj)._get_translatable_fields_names()
}
object_id = str(obj.pk)
instances[object_id] = obj
nonlocal query
query |= models.Q(
content_type__id=content_type_id,
object_id=object_id,
)
if hierarchy:
for (relation, detail) in hierarchy.items():
value = getattr(obj, relation, None)
if value is not None:
if isinstance(value, models.Manager):
if not (hasattr(obj, '_prefetched_objects_cache')
and relation
in obj._prefetched_objects_cache):
prefetch_related_objects([obj], relation)
value = value.all()
_fill_entity(
entity=value,
hierarchy=detail['relations'],
included=detail['included'],
)
def last_actions(self, release, limit, language=None, user=None):
sqlu = "SELECT pl.pofile_id, max(pl.id) as id FROM pofile_log pl"
if user:
sqlu += " LEFT OUTER JOIN pofile_assigns a ON pl.pofile_id = a.pofile_id"
sqlu += " WHERE a.translate_id=%(userid)s or a.review_id=%(userid)s" % {'userid': user.id}
else:
try:
bot = User.objects.get(username=getattr(settings, 'BOT_USERNAME','bot'))
except:
bot = None
if bot:
sqlu += " WHERE pl.user_id <> %d" % bot.pk
sqlu += " group by pl.pofile_id"
sql = "SELECT p.id, p.pofile_id, p.created, p.user_id, p.action, p.comment FROM pofile_log p INNER JOIN (%s) p2 "\
"INNER JOIN pofile f ON p.id = p2.id AND f.id = p.pofile_id" % sqlu
sql += " WHERE f.release_id = %d" % release.id
if language:
sql += " AND f.language_id=%d" % language.id
sql += " ORDER BY p.id DESC LIMIT %d" % limit
from django.db.models.query import prefetch_related_objects
result = list(self.raw(sql))
prefetch_related_objects(result, ('user',))
return result
def get_attrs(self, item_list, user):
prefetch_related_objects(item_list, "created_by")
result = defaultdict(lambda: {"created_by": {}})
user_serializer = UserSerializer()
serialized_users = {
user["id"]: user
for user in serialize(
[
discover_saved_query.created_by
for discover_saved_query in item_list
if discover_saved_query.created_by
],
user=user,
serializer=user_serializer,
)
}
for discover_saved_query in item_list:
result[discover_saved_query]["created_by"] = serialized_users.get(
str(discover_saved_query.created_by_id)
)
return result
def test_prefetch_object(self):
book1 = Book.objects.get(id=self.book1.id)
with self.assertNumQueries(1):
prefetch_related_objects([book1], Prefetch('authors'))
with self.assertNumQueries(0):
self.assertEqual(set(book1.authors.all()),
{self.author1, self.author2, self.author3})
def test_m2m_reverse(self):
author1 = Author.objects.get(id=self.author1.id)
with self.assertNumQueries(1):
prefetch_related_objects([author1], 'books')
with self.assertNumQueries(0):
self.assertEqual(set(author1.books.all()),
{self.book1, self.book2})
def test_prefetch_queryset(self):
book1 = Book.objects.get(id=self.book1.id)
with self.assertNumQueries(1):
prefetch_related_objects(
[book1],
Prefetch('authors', queryset=Author.objects.filter(id__in=[self.author1.id, self.author2.id]))
)
with self.assertNumQueries(0):
self.assertEqual(set(book1.authors.all()), {self.author1, self.author2})
def prefetch_related(tests):
prefetch_related_objects(
tests,
'known_issues',
'test_run',
'test_run__environment',
'test_run__build',
'test_run__build__project',
'test_run__build__project__group',
)
def prefetch_related(builds):
prefetch_related_objects(
builds,
'project',
'project__group',
'test_runs',
'test_runs__environment',
'test_runs__tests',
'test_runs__tests__suite',
)
def prefetch_related(tests):
prefetch_related_objects(
tests,
'known_issues',
'test_run',
'test_run__environment',
'test_run__build',
'test_run__build__project',
'test_run__build__project__group',
)
def _prefetch_related_objects(self):
things = defaultdict(list)
for index, thing in enumerate(self._result_cache, start=0):
things[thing.__class__].append(thing)
for klass, instances in things.items():
prefetches = self._prefetch_related_lookups[:]
if klass in self._our_prefetches:
prefetches.extend(self._our_prefetches[klass])
prefetch_related_objects(instances, prefetches)
return None
def get_attrs(self, item_list, user):
item_dict = {i.id: i for i in item_list}
prefetch_related_objects(item_list, "created_by")
widgets = (DashboardWidget.objects.filter(
dashboard_id__in=item_dict.keys()).order_by("order").values(
"dashboard_id", "order", "display_type", "detail", "id"))
result = defaultdict(lambda: {
"widget_display": [],
"widget_preview": [],
"created_by": {}
})
for widget in widgets:
dashboard = item_dict[widget["dashboard_id"]]
display_type = DashboardWidgetDisplayTypes.get_type_name(
widget["display_type"])
result[dashboard]["widget_display"].append(display_type)
for widget in widgets:
dashboard = item_dict[widget["dashboard_id"]]
widget_preview = {
"displayType":
DashboardWidgetDisplayTypes.get_type_name(
widget["display_type"]),
"layout":
None,
}
if widget.get("detail"):
detail = json.loads(widget["detail"])
if detail.get("layout"):
widget_preview["layout"] = detail["layout"]
result[dashboard]["widget_preview"].append(widget_preview)
user_serializer = UserSerializer()
serialized_users = {
user["id"]: user
for user in serialize(
[
dashboard.created_by
for dashboard in item_list if dashboard.created_by
],
user=user,
serializer=user_serializer,
)
}
for dashboard in item_dict.values():
result[dashboard]["created_by"] = serialized_users.get(
str(dashboard.created_by_id))
return result
def __get_tests__(self, builds, metadata):
test_runs = []
for build in builds:
for test_run in build.test_runs.all():
test_runs.append(test_run)
tests = []
for test_runs_batch in split_list(test_runs, chunk_size=100):
prefetch_related_objects(test_runs_batch, Prefetch('tests', queryset=Test.objects.filter(metadata=metadata).prefetch_related('metadata').order_by()))
for test_run in test_runs_batch:
tests += test_run.tests.all()
return tests
def test_prefetch_queryset(self):
book1 = Book.objects.get(id=self.book1.id)
with self.assertNumQueries(1):
prefetch_related_objects(
[book1],
Prefetch('authors',
queryset=Author.objects.filter(
id__in=[self.author1.id, self.author2.id])))
with self.assertNumQueries(0):
self.assertEqual(set(book1.authors.all()),
{self.author1, self.author2})
def _get_sorted_occurrences(self):
occurrences = []
if hasattr(self, 'occurrence_pool') and self.occurrence_pool is not None:
for occurrence in self.occurrence_pool:
if occurrence.start <= self.utc_end and occurrence.end >= self.utc_start:
occurrences.append(occurrence)
return occurrences
prefetch_related_objects(self.events, 'occurrence_set')
for event in self.events:
event_occurrences = event.get_occurrences(self.start, self.end, clear_prefetch=False)
occurrences += event_occurrences
return sorted(occurrences, **self.sorting_options)
def retrieve(self, request, region=None, agency=None, *args, **kwargs):
try:
lat = float(request.GET['lat'])
lon = float(request.GET['lon'])
radius = float(request.GET.get('radius', 500.0))
if radius < 0 or 5000 < radius:
# TODO: bad request, invalid param
raise Exception('invalid radius')
except KeyError:
# TODO: bad request, missing param
raise
except ValueError:
# TODO: bad request, bad param
raise
# we'll at most return N stops
# TODO: limiting before filtering is bad
stops = Stop.objects.nearby(lat, lon, radius)[:20]
if region and agency:
agency_id = Agency.create_id(region, agency)
def desired_agency(stop):
return stop.agency_id == agency_id
stops = filter(desired_agency, stops)
prefetch_related_objects(
stops,
['agency', 'agency__region', 'directions', 'directions__route'])
routes = {}
agencies = {}
regions = {}
for stop in stops:
agency = stop.agency
agencies[agency.id] = agency
region = agency.region
regions[region.id] = region
for direction in stop.directions.all():
route = direction.route
routes[route.id] = direction.route
self.object = Adapter(self,
agencies=agencies.values(),
regions=regions.values(),
routes=routes.values(),
stops=stops)
serializer = self.get_serializer(self.object)
return Response(serializer.data)
def has_permission(self, request):
if not request.user.is_authenticated():
return False
prefetch_related_objects([request.user], ('accreditations__course', ))
if request.path.endswith('/logout/'):
return True
if request.user.is_superuser:
return True
if request.path.endswith('/choose/') or request.path.endswith('/ask/') or re.search(r'/ask/[^/]+/$', request.path) or request.path.endswith('/inscriptions/course/add/') or request.path.endswith('/course/jsi18n/'):
return True
if 'course_uid' not in request.COOKIES:
return False
course_uid = request.COOKIES['course_uid']
return request.user.accreditations.filter(course__uid=course_uid).exclude(role='').count() > 0
def retrieve(self, request, region=None, agency=None, *args, **kwargs):
try:
lat = float(request.GET['lat'])
lon = float(request.GET['lon'])
radius = float(request.GET.get('radius', 500.0))
if radius < 0 or 5000 < radius:
# TODO: bad request, invalid param
raise Exception('invalid radius')
except KeyError:
# TODO: bad request, missing param
raise
except ValueError:
# TODO: bad request, bad param
raise
# we'll at most return N stops
# TODO: limiting before filtering is bad
stops = Stop.objects.nearby(lat, lon, radius)[:20]
if region and agency:
agency_id = Agency.create_id(region, agency)
def desired_agency(stop):
return stop.agency_id == agency_id
stops = filter(desired_agency, stops)
prefetch_related_objects(stops, ['agency', 'agency__region',
'directions', 'directions__route'])
routes = {}
agencies = {}
regions = {}
for stop in stops:
agency = stop.agency
agencies[agency.id] = agency
region = agency.region
regions[region.id] = region
for direction in stop.directions.all():
route = direction.route
routes[route.id] = direction.route
self.object = Adapter(self, agencies=agencies.values(),
regions=regions.values(), routes=routes.values(),
stops=stops)
serializer = self.get_serializer(self.object)
return Response(serializer.data)
def _get_sorted_occurrences(self):
occurrences = []
if hasattr(self, 'occurrence_pool') and self.occurrence_pool is not None:
for occurrence in self.occurrence_pool:
if occurrence.start <= self.utc_end and occurrence.end >= self.utc_start:
occurrences.append(occurrence)
return occurrences
try:
prefetch_related_objects(self.events, 'occurrence_set')
except AttributeError:
prefetch_related_objects(self.events, ['occurrence_set'])
for event in self.events:
event_occurrences = event.get_occurrences(self.start, self.end, clear_prefetch=False)
occurrences += event_occurrences
return sorted(occurrences, **self.sorting_options)
def get_context_data(self, *args, **kwargs):
ret = super().get_context_data(*args, **kwargs)
user = self.request.user
membre = user.membre
if user.is_staff and 'membre_id' in self.kwargs:
membre = models.Membre.objects.get(id=self.kwargs['membre_id'])
ret['membre'] = membre
prefetch_related_objects(
[membre],
Prefetch(
'licences',
models.Licence.objects.annotate(
_montant_paiement=Sum('paiements__montant'))))
ret['STRIPE_PUBLIC_KEY'] = settings.STRIPE_PUBLIC_KEY
ret['saisons'] = models.Saison.objects.filter(ouvert=True).exclude(
membres__in=membre.licences.all())
return ret
def update(self, request, *args, **kwargs):
partial = kwargs.pop('partial', False)
instance = self.get_object()
serializer = self.get_serializer(instance,
data=request.data,
partial=partial)
serializer.is_valid(raise_exception=True)
self.perform_update(serializer)
if getattr(instance, '_prefetched_objects_cache', None):
# If 'prefetch_related' has been applied to a queryset, we need to
# forcibly invalidate the prefetch cache on the instance,
# and then re-prefetch related objects
instance._prefetched_objects_cache = {}
prefetch_related_objects([instance], *self.get_prefetch_related())
return Response(serializer.data)
def test_m2m_then_m2m(self):
"""
We can follow a m2m and another m2m.
"""
authors = list(Author.objects.all())
with self.assertNumQueries(2):
prefetch_related_objects(authors, 'books__read_by')
with self.assertNumQueries(0):
self.assertEqual(
[[[str(r) for r in b.read_by.all()] for b in a.books.all()]
for a in authors],
[
[['Amy'], ['Belinda']], # Charlotte - Poems, Jane Eyre
[['Amy']], # Anne - Poems
[['Amy'], []], # Emily - Poems, Wuthering Heights
[['Amy', 'Belinda']], # Jane - Sense and Sense
])
def list(self, request, agency, lat, lon, *args, **kwargs):
lat = float(lat)
lon = float(lon)
max = float(request.GET.get('max', 0.25))
if max < 0:
max = 0.25
elif 1 < max:
max = 1
radius = max * meters_per_mile
# we'll at most return n stops
# TODO: limiting before filtering is bad
stops = Stop.objects.nearby(lat, lon, radius)[:25]
prefetch_related_objects(stops, ['agency', 'directions',
'directions__route'])
results = []
for stop in stops:
# this is kind of ugly, but should do the trick since we don't know
# the region
if not stop.agency_id.endswith(agency):
continue
for direction in stop.directions.all():
results.append({
'agency_tag': stop.agency.get_id(),
'direction_tag': direction.get_id(),
'route_tag': direction.route.get_id(),
'stop_tag': stop.get_id(),
'stop_id': stop.code,
'route_title': direction.route.name,
'direction_title': direction.name,
'stop_title': stop.name,
'distance': stop.distance / meters_per_mile,
'lat': stop.lat,
'lon': stop.lon
})
if len(results) > 25:
return Response(results)
return Response(results)
def test_m2m_then_m2m(self):
"""
We can follow a m2m and another m2m.
"""
authors = list(Author.objects.all())
with self.assertNumQueries(2):
prefetch_related_objects(authors, 'books__read_by')
with self.assertNumQueries(0):
self.assertEqual(
[
[[str(r) for r in b.read_by.all()] for b in a.books.all()]
for a in authors
],
[
[['Amy'], ['Belinda']], # Charlotte - Poems, Jane Eyre
[['Amy']], # Anne - Poems
[['Amy'], []], # Emily - Poems, Wuthering Heights
[['Amy', 'Belinda']], # Jane - Sense and Sense
]
)
def listar_desafiaveis(request, username):
jogador = get_object_or_404(Jogador, user__username=username)
posicao_atual_jogador = jogador.posicao_em(timezone.localtime())
desafiaveis = buscar_desafiaveis(jogador, timezone.localtime(), False,
False)
# Buscar usuários de desafiáveis
prefetch_related_objects(desafiaveis, 'user')
desafios_pendentes = DesafioLadder.objects.filter(cancelamentodesafioladder__isnull=True, admin_validador__isnull=True,
score_desafiante__gt=F('score_desafiado'),
posicao_desafiado__lte=posicao_atual_jogador, posicao_desafiante__gte=posicao_atual_jogador) \
.select_related('desafiante', 'desafiado')
return render(
request, 'jogadores/listar_desafiaveis.html', {
'desafiaveis': desafiaveis,
'desafios_pendentes': desafios_pendentes,
'jogador': jogador
})
def create(self, validated_data):
"""Create instances in bulk efficiently"""
ModelClass = self.child.Meta.model # pylint: disable=invalid-name
info = model_meta.get_field_info(ModelClass)
m2m_fields = [f for f, i in info.relations.items() if i.to_many]
instances = []
m2m_values = []
for attrs in validated_data:
m2m_values.append({})
for field in m2m_fields:
# m2m fields must be set after the instance is created
m2m_values[-1][field] = attrs.pop(field, None)
if hasattr(self.child, "bulk_create_attrs"):
attrs = self.child.bulk_create_attrs(attrs)
instances.append(ModelClass(**attrs))
# create the instances in bulk for efficiency
try:
ModelClass.objects.bulk_create(instances)
except IntegrityError as exc:
raise ValidationError(exc)
# set any m2m values
for instance, m2m_values_ in zip(instances, m2m_values):
for field_name, value in m2m_values_.items():
if value:
field = getattr(instance, field_name)
field.set(value)
# prefetch any m2m values which are serializer fields to avoid n+1 queries
prefetch_fields = set(m2m_fields) & set(self.child.Meta.fields)
prefetch_related_objects(instances, *prefetch_fields)
return instances
def test_unknown(self):
book1 = Book.objects.get(id=self.book1.id)
with self.assertRaises(AttributeError):
prefetch_related_objects([book1], 'unknown_attribute')
def get_person_as_version_data(person, new_person=False):
"""
If new_person is True then skip some DB checks that we know will be
empty. This should reduce the number of queries needed.
"""
# Prefetch to reduce the number of queries
prefetch_related_objects([person], "tmp_person_identifiers")
result = {}
result["id"] = str(person.id)
# Add PersonIdentifier fields in to the global version namespace
# TODO: move these to dict to make it easier to merge/split/revert
existing_identifiers = {
identifier.value_type: identifier.value
for identifier in person.tmp_person_identifiers.all()
}
for field in PersonIdentifierFields:
result[field.name] = existing_identifiers.get(field.name, "")
for field in settings.SIMPLE_POPOLO_FIELDS:
result[field.name] = getattr(person, field.name) or ""
# Add legacy identifiers
# TODO: these should use the PersonIdenfitiers model and value types,
# but this code emulates the legacy way of adding IDs.
if person.get_single_identifier_of_type("theyworkforyou"):
result["identifiers"] = []
new_id = person.get_single_identifier_of_type(
"theyworkforyou").internal_identifier
if "publicwhip" not in new_id:
new_id = "uk.org.publicwhip/person/{}".format(new_id)
result["identifiers"].append({
"identifier": new_id,
"scheme": "uk.org.publicwhip"
})
if person.get_single_identifier_of_type("twitter_username"):
result["identifiers"] = result.get("identifiers", [])
result["identifiers"].append({
"identifier":
person.get_single_identifier_of_type(
"twitter_username").internal_identifier,
"scheme":
"twitter",
})
result["other_names"] = []
candidacies = {}
not_standing = []
if not new_person:
result["other_names"] = [{
"name": on.name,
"note": on.note,
"start_date": on.start_date,
"end_date": on.end_date,
}
for on in person.other_names.order_by(
"name", "start_date", "end_date")]
for membership in person.memberships.all():
ballot = membership.ballot
candidacy = {"party": membership.party.ec_id}
if membership.elected is not None:
candidacy["elected"] = membership.elected
if membership.party_list_position is not None:
candidacy[
"party_list_position"] = membership.party_list_position
candidacies[ballot.ballot_paper_id] = candidacy
for not_standing_in_election in person.not_standing.all():
not_standing.append(not_standing_in_election.slug)
# Add `favourite_biscuits` to an `extra_fields` key
# to re-produce the previous ExtraField model.
# This is done like this to save changing the version diff
# for exery edit to move to key to the parent object.
# In the future we will have to run a script to move all the
# keys to wherever we want them.
extra_fields = {"favourite_biscuits": person.favourite_biscuit or ""}
result["extra_fields"] = extra_fields
result["candidacies"] = candidacies
if not_standing:
result["not_standing"] = not_standing
return result
def prefetch(self, *related):
prefetch_related_objects([self], *related)
def get_nodes(self, request):
from cms.models import Title
site = self.renderer.site
lang = self.renderer.request_language
pages = get_page_queryset(
site,
draft=self.renderer.draft_mode_active,
published=not self.renderer.draft_mode_active,
)
if is_valid_site_language(lang, site_id=site.pk):
_valid_language = True
_hide_untranslated = hide_untranslated(lang, site.pk)
else:
_valid_language = False
_hide_untranslated = False
if _valid_language:
# The request language has been explicitly configured
# for the current site.
if _hide_untranslated:
fallbacks = []
else:
fallbacks = get_fallback_languages(lang, site_id=site.pk)
languages = [lang
] + [_lang for _lang in fallbacks if _lang != lang]
else:
# The request language is not configured for the current site.
# Fallback to all configured public languages for the current site.
languages = get_public_languages(site.pk)
fallbacks = languages
pages = (pages.filter(title_set__language__in=languages).
select_related('node').order_by('node__path').distinct())
if not self.renderer.draft_mode_active:
# we're dealing with public pages.
# prefetch the draft versions.
pages = pages.select_related('publisher_public__node')
pages = get_visible_nodes(request, pages, site)
if not pages:
return []
try:
homepage = [page for page in pages if page.is_home][0]
except IndexError:
homepage = None
titles = Title.objects.filter(
language__in=languages,
publisher_is_draft=self.renderer.draft_mode_active,
)
lookup = Prefetch(
'title_set',
to_attr='filtered_translations',
queryset=titles,
)
prefetch_related_objects(pages, lookup)
# Build the blank title instances only once
blank_title_cache = {
language: EmptyTitle(language=language)
for language in languages
}
if lang not in blank_title_cache:
blank_title_cache[lang] = EmptyTitle(language=lang)
# Maps a node id to its page id
node_id_to_page = {}
def _page_to_node(page):
# EmptyTitle is used to prevent the cms from trying
# to find a translation in the database
page.title_cache = blank_title_cache.copy()
for trans in page.filtered_translations:
page.title_cache[trans.language] = trans
menu_node = get_menu_node_for_page(
self.renderer,
page,
language=lang,
fallbacks=fallbacks,
)
return menu_node
menu_nodes = []
for page in pages:
node = page.node
parent_id = node_id_to_page.get(node.parent_id)
if node.parent_id and not parent_id:
# If the parent page is not available (unpublished, etc..)
# don't bother creating menu nodes for its descendants.
continue
menu_node = _page_to_node(page)
cut_homepage = homepage and not homepage.in_navigation
if cut_homepage and parent_id == homepage.pk:
# When the homepage is hidden from navigation,
# we need to cut all its direct children from it.
menu_node.parent_id = None
else:
menu_node.parent_id = parent_id
node_id_to_page[node.pk] = page.pk
menu_nodes.append(menu_node)
return menu_nodes
def load_chunks(submit_milestone, user_map, django_user):
chunks = []
chunk_map = {}
submissions = {}
django_submissions = submit_milestone.submissions.exclude(authors=django_user).prefetch_related("authors")
# the exclude() in the call below failed because of a Django bug. Try submit_milestone=55, django_user=1016 on the production db.
# django_chunks = Chunk.objects \
# .filter(file__submission__milestone=submit_milestone) \
# .exclude(file__submission__authors=django_user) \
# .prefetch_related('file__submission__authors') \
# .values('id', 'name', 'cluster_id', 'file__submission', 'class_type', 'student_lines')
# using a correct raw SQL query instead..
django_chunks = Chunk.objects \
.raw("""
SELECT files.submission_id, `chunks`.`id`, `chunks`.`file_id`, `chunks`.`name`, `chunks`.`start`, `chunks`.`end`, `chunks`.`cluster_id`, `chunks`.`created`, `chunks`.`modified`, `chunks`.`class_type`, `chunks`.`staff_portion`, `chunks`.`student_lines`, `chunks`.`chunk_info`
FROM `chunks`
join files on chunks.file_id=files.id
join submissions on files.submission_id=submissions.id
join submissions_authors on submissions.id=submissions_authors.submission_id
where submissions.milestone_id=%s and not(user_id=%s)
""", [submit_milestone.id, django_user.id])
# preload the file.submission according to http://python.6.x6.nabble.com/Prefetch-related-data-when-using-raw-td4983196.html
django_chunks = list(django_chunks)
prefetch_related_objects(django_chunks, ['file__submission'])
django_tasks = Task.objects.filter(
submission__milestone=submit_milestone) \
.exclude(submission__authors=django_user) \
.select_related('reviewer__user', 'chunk') \
# load all submissions and chunks into lightweight internal objects
django_submission_chunks = defaultdict(list)
for chunk in django_chunks:
django_submission_chunks[chunk.file.submission_id].append(chunk)
for django_submission in django_submissions:
submission = SubmissionForReview(
id=django_submission.id,
authors=[user_map[author.id] for author in django_submission.authors.all() if author.id in user_map],
chunks=django_submission_chunks[django_submission.id])
#logging.debug(django_submission.authors.all())
#logging.debug(submission.authors)
if not submission.chunks:
# toss out any submissions without chunks
continue
submissions[submission.id] = submission
chunks.extend(submission.chunks)
#logging.debug(submissions)
# load existing reviewing assignments
for chunk in chunks:
chunk_map[chunk.id] = chunk
#logging.debug(chunk_map)
for django_task in django_tasks:
if django_task.chunk:
chunk = chunk_map[django_task.chunk_id] # looks like dead code
reviewer = user_map[django_task.reviewer_id]
chunk_map[django_task.chunk_id].assign_reviewer(reviewer)
return chunks
def load_chunks(submit_milestone, user_map, django_user):
chunks = []
chunk_map = {}
submissions = {}
django_submissions = submit_milestone.submissions.exclude(
authors=django_user).prefetch_related("authors")
# the exclude() in the call below failed because of a Django bug. Try submit_milestone=55, django_user=1016 on the production db.
# django_chunks = Chunk.objects \
# .filter(file__submission__milestone=submit_milestone) \
# .exclude(file__submission__authors=django_user) \
# .prefetch_related('file__submission__authors') \
# .values('id', 'name', 'cluster_id', 'file__submission', 'class_type', 'student_lines')
# using a correct raw SQL query instead..
django_chunks = Chunk.objects \
.raw("""
SELECT files.submission_id, `chunks`.`id`, `chunks`.`file_id`, `chunks`.`name`, `chunks`.`start`, `chunks`.`end`, `chunks`.`cluster_id`, `chunks`.`created`, `chunks`.`modified`, `chunks`.`class_type`, `chunks`.`staff_portion`, `chunks`.`student_lines`, `chunks`.`chunk_info`
FROM `chunks`
join files on chunks.file_id=files.id
join submissions on files.submission_id=submissions.id
join submissions_authors on submissions.id=submissions_authors.submission_id
where submissions.milestone_id=%s and not(user_id=%s)
""", [submit_milestone.id, django_user.id])
# preload the file.submission according to http://python.6.x6.nabble.com/Prefetch-related-data-when-using-raw-td4983196.html
django_chunks = list(django_chunks)
prefetch_related_objects(django_chunks, ['file__submission'])
django_tasks = Task.objects.filter(
submission__milestone=submit_milestone) \
.exclude(submission__authors=django_user) \
.select_related('reviewer', 'chunk') \
# load all submissions and chunks into lightweight internal objects
django_submission_chunks = defaultdict(list)
for chunk in django_chunks:
django_submission_chunks[chunk.file.submission_id].append(chunk)
for django_submission in django_submissions:
submission = SubmissionForReview(
id=django_submission.id,
authors=[
user_map[author.id]
for author in django_submission.authors.all()
if author.id in user_map
],
chunks=django_submission_chunks[django_submission.id])
#logging.debug(django_submission.authors.all())
#logging.debug(submission.authors)
if not submission.chunks:
# toss out any submissions without chunks
continue
submissions[submission.id] = submission
chunks.extend(submission.chunks)
logging.debug(submissions)
# load existing reviewing assignments
for chunk in chunks:
chunk_map[chunk.id] = chunk
#logging.debug(chunk_map)
for django_task in django_tasks:
if django_task.chunk:
chunk = chunk_map[django_task.chunk_id] # looks like dead code
reviewer = user_map[django_task.reviewer_id]
chunk_map[django_task.chunk_id].assign_reviewer(reviewer)
return chunks
def prefetch(self, *related):
prefetch_related_objects([self], *related)
def test_unknown(self):
book1 = Book.objects.get(id=self.book1.id)
with self.assertRaises(AttributeError):
prefetch_related_objects([book1], 'unknown_attribute')
def get_context_data(self, **kwargs):
u = User.objects.none()
if 'user' in kwargs:
try:
u = User.objects.get(pk=int(kwargs['user']))
except:
u = User.objects.get(username=kwargs['user'])
elif self.request.GET.get('user', False):
try:
u = User.objects.get(pk=int(self.request.GET['user']))
except:
u = User.objects.get(username=self.request.GET['user'])
elif not self.request.user.is_anonymous():
# if the user is logged in and no user is specified, show logged in user
u = User.objects.get(pk=int(self.request.user.id))
# get all resources the profile user owns
resources = u.uaccess.owned_resources
# get a list of groupmembershiprequests
group_membership_requests = GroupMembershipRequest.objects.filter(
invitation_to=u).all()
# if requesting user is not the profile user, then show only resources that the
# requesting user has access
if self.request.user != u:
if self.request.user.is_authenticated():
if self.request.user.is_superuser:
# admin can see all resources owned by profile user
pass
else:
# filter out any resources the requesting user doesn't have access
resources = resources.filter(
Q(pk__in=self.request.user.uaccess.view_resources)
| Q(raccess__public=True)
| Q(raccess__discoverable=True))
else:
# for anonymous requesting user show only resources that are either public or
# discoverable
resources = resources.filter(
Q(raccess__public=True) | Q(raccess__discoverable=True))
# get resource attributes used in profile page
resources = resources.only('title', 'resource_type', 'created')
# prefetch resource metadata elements
meta_contenttypes = get_metadata_contenttypes()
for ct in meta_contenttypes:
# get a list of resources having metadata that is an instance of a specific
# metadata class (e.g., CoreMetaData)
res_list = [res for res in resources if res.content_type == ct]
prefetch_related_objects(res_list,
Prefetch('content_object__creators'),
Prefetch('content_object___description'),
Prefetch('content_object___title'))
return {
'profile_user': u,
'resources': resources,
'quota_message': get_quota_message(u),
'group_membership_requests': group_membership_requests,
}
def get_context_data(self, **kwargs):
u = User.objects.none()
if 'user' in kwargs:
try:
u = User.objects.get(pk=int(kwargs['user']))
except:
u = User.objects.get(username=kwargs['user'])
elif self.request.GET.get('user', False):
try:
u = User.objects.get(pk=int(self.request.GET['user']))
except:
u = User.objects.get(username=self.request.GET['user'])
elif not self.request.user.is_anonymous():
# if the user is logged in and no user is specified, show logged in user
u = User.objects.get(pk=int(self.request.user.id))
# get all resources the profile user owns
resources = u.uaccess.owned_resources
# get a list of groupmembershiprequests
group_membership_requests = GroupMembershipRequest.objects.filter(invitation_to=u).all()
# if requesting user is not the profile user, then show only resources that the
# requesting user has access
if self.request.user != u:
if self.request.user.is_authenticated():
if self.request.user.is_superuser:
# admin can see all resources owned by profile user
pass
else:
# filter out any resources the requesting user doesn't have access
resources = resources.filter(
Q(pk__in=self.request.user.uaccess.view_resources) |
Q(raccess__public=True) |
Q(raccess__discoverable=True))
else:
# for anonymous requesting user show only resources that are either public or
# discoverable
resources = resources.filter(Q(raccess__public=True) |
Q(raccess__discoverable=True))
# get resource attributes used in profile page
resources = resources.only('title', 'resource_type', 'created')
# prefetch resource metadata elements
meta_contenttypes = get_metadata_contenttypes()
for ct in meta_contenttypes:
# get a list of resources having metadata that is an instance of a specific
# metadata class (e.g., CoreMetaData)
res_list = [res for res in resources if res.content_type == ct]
prefetch_related_objects(res_list,
Prefetch('content_object__creators'),
Prefetch('content_object___description'),
Prefetch('content_object___title')
)
return {
'profile_user': u,
'resources': resources,
'quota_message': get_quota_message(u),
'group_membership_requests': group_membership_requests,
}