def polymorphic_prefetch_related(self, *lookups):
clone = self._clone()
for lookup in lookups:
if not isinstance(lookup, Prefetch):
lookup = Prefetch(lookup)
if lookup.queryset is None:
rel = self.model._meta.get_field(lookup.prefetch_through)
lookup.queryset = rel.related_model.objects.all()
lookup.queryset = lookup.queryset.polymorphic()
clone = clone.prefetch_related(lookup)
return clone
class RunApiViewSet(mixins.ListModelMixin, mixins.CreateModelMixin,
mixins.RetrieveModelMixin, mixins.UpdateModelMixin,
GenericViewSet):
queryset = Run.objects.prefetch_related(
Prefetch('port_set', queryset=Port.objects.select_related(
'run'))).order_by('-created_date').all()
def get_serializer_class(self):
if self.action == 'list':
return RunApiListSerializer
else:
return RunSerializerFull
def query_from_dict(self, query_filter, queryset, input_list):
for single_input in input_list:
key, val = single_input.split(':')
query = {query_filter % key: val}
queryset = queryset.filter(**query).all()
return queryset
@swagger_auto_schema(query_serializer=RunApiListSerializer)
def list(self, request, *args, **kwargs):
query_list_types = [
'job_groups', 'request_ids', 'inputs', 'tags', 'jira_ids',
'run_ids', 'apps', 'run', 'values_run', 'ports'
]
fixed_query_params = fix_query_list(request.query_params,
query_list_types)
serializer = RunApiListSerializer(data=fixed_query_params)
if serializer.is_valid():
queryset = time_filter(Run, fixed_query_params)
queryset = time_filter(Run,
request.query_params,
time_modal='modified_date',
previous_queryset=queryset)
job_groups = fixed_query_params.get('job_groups')
jira_ids = fixed_query_params.get('jira_ids')
run_ids = fixed_query_params.get('run_ids')
status_param = fixed_query_params.get('status')
ports = fixed_query_params.get('ports')
tags = fixed_query_params.get('tags')
request_ids = fixed_query_params.get('request_ids')
apps = fixed_query_params.get('apps')
values_run = fixed_query_params.get('values_run')
run = fixed_query_params.get('run')
run_distribution = fixed_query_params.get('run_distribution')
count = fixed_query_params.get('count')
full = fixed_query_params.get('full')
if full:
full = bool(strtobool(full))
if job_groups:
queryset = queryset.filter(job_group__in=job_groups)
if jira_ids:
queryset = queryset.filter(job_group__jira_id__in=jira_ids)
if run_ids:
queryset = queryset.filter(id__in=run_ids)
if status_param:
queryset = queryset.filter(
status=RunStatus[status_param].value)
if ports:
queryset = self.query_from_dict("port__%s__exact", queryset,
ports)
if tags:
queryset = self.query_from_dict("tags__%s__exact", queryset,
tags)
if request_ids:
queryset = queryset.filter(tags__requestId__in=request_ids)
if apps:
queryset = queryset.filter(app__in=apps)
if run:
filter_query = dict()
for single_run in run:
key, value = single_run.split(':')
if value == 'True' or value == 'true':
value = True
if value == 'False' or value == 'false':
value = False
filter_query[key] = value
if filter_query:
queryset = queryset.filter(**filter_query)
if values_run:
if len(values_run) == 1:
ret_str = values_run[0]
queryset = queryset.values_list(
ret_str, flat=True).order_by(ret_str).distinct(ret_str)
else:
values_run_query_list = [
single_run for single_run in values_run
]
values_run_query_set = set(values_run_query_list)
queryset = queryset.values_list(
*values_run_query_set).distinct()
if run_distribution:
distribution_dict = {}
run_query = run_distribution
run_ids = queryset.values_list('id', flat=True)
queryset = Run.objects.all()
queryset = queryset.filter(
id__in=run_ids).values(run_query).order_by().annotate(
Count(run_query))
for single_arg in queryset:
single_arg_name = None
single_arg_count = 0
for single_key, single_value in single_arg.items():
if 'count' in single_key:
single_arg_count = single_value
else:
single_arg_name = single_value
if single_arg_name is not None:
distribution_dict[single_arg_name] = single_arg_count
return Response(distribution_dict, status=status.HTTP_200_OK)
if count:
count = bool(strtobool(count))
if count:
return Response(queryset.count(),
status=status.HTTP_200_OK)
try:
page = self.paginate_queryset(queryset.all())
except ValidationError as e:
return Response(e, status=status.HTTP_400_BAD_REQUEST)
if page is not None:
if values_run:
return self.get_paginated_response(page)
if full:
serializer = RunSerializerFull(page, many=True)
else:
serializer = RunSerializerPartial(page, many=True)
return self.get_paginated_response(serializer.data)
return Response([], status=status.HTTP_200_OK)
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def create(self, request, *args, **kwargs):
serializer = APIRunCreateSerializer(data=request.data,
context={'request': request})
if serializer.is_valid():
run = serializer.save()
response = RunSerializerFull(run)
create_run_task.delay(response.data['id'], request.data['inputs'])
job_group_notifier_id = str(run.job_group_notifier_id)
self._send_notifications(job_group_notifier_id, run)
return Response(response.data, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def _send_notifications(self, job_group_notifier_id, run):
pipeline_name = run.app.name
pipeline_version = run.app.version
pipeline_link = run.app.pipeline_link
pipeline_description_event = AddPipelineToDescriptionEvent(
job_group_notifier_id, pipeline_name, pipeline_version,
pipeline_link).to_dict()
send_notification.delay(pipeline_description_event)
run_event = RunStartedEvent(job_group_notifier_id, str(run.id),
run.app.name, run.app.pipeline_link,
run.output_directory, run.tags).to_dict()
send_notification.delay(run_event)
def find_root_by_name(text_label_name):
return TextLabel.objects.prefetch_related(
Prefetch('translatedtextlabel_set',
to_attr="translated_text_labels")).get(label=text_label_name,
parent__isnull=True)
def generateCustomSiteReport(self, pk, base_url, fs_ids, startdate,
enddate, removeNullField):
self.base_url = base_url
self.removeNullField = removeNullField
# Our container for 'Flowable' objects
elements = []
toc = TableOfContents()
toc.levelStyles = [
PS(fontName='arialuni',
fontSize=12,
name='TOCHeading1',
leftIndent=20,
firstLineIndent=-20,
spaceBefore=5,
leading=10),
PS(fontName='arialuni',
fontSize=10,
name='TOCHeading2',
leftIndent=40,
firstLineIndent=-20,
spaceBefore=3,
leading=10),
PS(fontName='arialuni',
fontSize=9,
name='TOCHeading3',
leftIndent=40,
firstLineIndent=-20,
spaceBefore=3,
leading=10),
]
elements.append(
Paragraph('Custom Responses Report for Site', self.centered))
elements.append(PageBreak())
elements.append(Paragraph('Table of contents', self.centered))
elements.append(toc)
elements.append(PageBreak())
# A large collection of style sheets pre-made for us
styles = getSampleStyleSheet()
styles.add(ParagraphStyle(name='centered', alignment=TA_CENTER))
site = Site.objects.select_related('project').get(pk=pk)
self.project_name = site.project.name
self.project_logo = site.project.logo.url
elements.append(Paragraph(site.name, self.h1))
elements.append(Paragraph(site.identifier, styles['Normal']))
if site.address:
elements.append(Paragraph(site.address, styles['Normal']))
if site.phone:
elements.append(Paragraph(site.phone, styles['Normal']))
if site.region:
elements.append(Paragraph(site.region.name, styles['Normal']))
elements.append(Spacer(0, 10))
elements.append(Paragraph("Site Information", styles['Normal']))
metas = generateSiteMetaAttribs(pk)
styBackground = ParagraphStyle('background',
parent=self.bodystyle,
backColor=colors.white)
meta_data = []
if metas:
for meta in metas:
row = [
Paragraph(meta['question_text'], styBackground),
Paragraph(
str(meta['answer']) if isinstance(meta['answer'], int)
else meta['answer'], styBackground)
]
meta_data.append(row)
metat1 = Table(meta_data, colWidths=(60 * mm, None))
metat1.setStyle(self.ts1)
elements.append(metat1)
elements.append(PageBreak())
elements.append(Paragraph('Responses', self.h2))
split_startdate = startdate.split('-')
split_enddate = enddate.split('-')
new_startdate = date(int(split_startdate[0]), int(split_startdate[1]),
int(split_startdate[2]))
end = date(int(split_enddate[0]), int(split_enddate[1]),
int(split_enddate[2]))
new_enddate = end + datetime.timedelta(days=1)
forms = FieldSightXF.objects.select_related('xf').filter(
pk__in=fs_ids, is_survey=False, is_deleted=False).order_by(
'project', 'is_staged', 'is_scheduled', 'stage__stage__order',
'stage__order', 'stage__date_created',
'date_created').prefetch_related(
Prefetch(
'site_form_instances',
queryset=FInstance.objects.select_related('instance').
filter(date__range=[new_startdate, new_enddate])),
Prefetch('project_form_instances',
queryset=FInstance.objects.select_related(
'instance').filter(
site_id=site.id,
date__range=[new_startdate,
new_enddate])))
if not forms:
elements.append(
Paragraph("No Any Responses Yet.", styles['Heading5']))
#a=FieldSightXF.objects.select_related('xf').filter(site_id=291).prefetch_related(Prefetch('site_form_instances', queryset=FInstance.objects.select_related('instance')))
styNormal = styleSheet['Normal']
styBackground = ParagraphStyle('background',
parent=styNormal,
backColor=colors.white)
for form in forms:
elements.append(Spacer(0, 10))
elements.append(Paragraph(form.xf.title, self.h3))
elements.append(
Paragraph(form.form_type() + " Form", styles['Heading4']))
if form.stage:
if form.stage.stage:
elements.append(
Paragraph("Stage Id: " + str(form.stage.stage.order),
self.paragraphstyle))
elements.append(
Paragraph("Sub Stage Id: " + str(form.stage.order),
self.paragraphstyle))
else:
elements.append(
Paragraph("Stage Id: " + str(form.stage.order),
self.paragraphstyle))
json_question = form.xf.json
form_user_name = form.xf.user.username
self.media_folder = form_user_name
#cursor = get_instaces_for_site_individual_form(form.id)
sub_count = 0
if not form.from_project and form.site_form_instances.all():
for instance in form.site_form_instances.all():
self.instance_id = instance.instance_id
new_elements = self.append_answers(json_question, instance,
sub_count)
elements += new_elements
elif form.project_form_instances.all():
for instance in form.project_form_instances.all():
self.instance_id = instance.instance_id
new_elements = self.append_answers(json_question, instance,
sub_count)
elements += new_elements
else:
elements.append(
Paragraph("No Submisions Yet. ", styles['Heading5']))
elements.append(Spacer(0, 10))
self.doc.multiBuild(elements, onLaterPages=self._header_footer)
def get_grouped_items(event, subevent=None, voucher=None, channel='web'):
items = event.items.filter_available(
channel=channel, voucher=voucher).select_related(
'category',
'tax_rule', # for re-grouping
).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(subevent=subevent)),
Prefetch(
'bundles',
queryset=ItemBundle.objects.prefetch_related(
Prefetch('bundled_item',
queryset=event.items.select_related(
'tax_rule').prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(
subevent=subevent)), )),
Prefetch('bundled_variation',
queryset=ItemVariation.objects.select_related(
'item', 'item__tax_rule').filter(
item__event=event).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(
subevent=subevent)), )),
)),
Prefetch('variations',
to_attr='available_variations',
queryset=ItemVariation.objects.filter(
active=True, quotas__isnull=False).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(
subevent=subevent))).distinct()),
).annotate(quotac=Count('quotas'),
has_variations=Count('variations')).filter(
quotac__gt=0).order_by('category__position',
'category_id', 'position',
'name')
display_add_to_cart = False
external_quota_cache = event.cache.get('item_quota_cache')
quota_cache = external_quota_cache or {}
if subevent:
item_price_override = subevent.item_price_overrides
var_price_override = subevent.var_price_overrides
else:
item_price_override = {}
var_price_override = {}
restrict_vars = set()
if voucher and voucher.quota_id:
# If a voucher is set to a specific quota, we need to filter out on that level
restrict_vars = set(voucher.quota.variations.all())
for item in items:
if voucher and voucher.item_id and voucher.variation_id:
# Restrict variations if the voucher only allows one
item.available_variations = [
v for v in item.available_variations
if v.pk == voucher.variation_id
]
max_per_order = item.max_per_order or int(
event.settings.max_items_per_order)
if not item.has_variations:
item._remove = not bool(item._subevent_quotas)
if voucher and (voucher.allow_ignore_quota or voucher.block_quota):
item.cached_availability = (Quota.AVAILABILITY_OK,
voucher.max_usages -
voucher.redeemed)
else:
item.cached_availability = list(
item.check_quotas(subevent=subevent,
_cache=quota_cache,
include_bundled=True))
item.order_max = min(
item.cached_availability[1] if item.cached_availability[1]
is not None else sys.maxsize, max_per_order)
original_price = item_price_override.get(item.pk,
item.default_price)
if voucher:
price = voucher.calculate_price(original_price)
else:
price = original_price
item.display_price = item.tax(price,
currency=event.currency,
include_bundled=True)
if price != original_price:
item.original_price = original_price
display_add_to_cart = display_add_to_cart or item.order_max > 0
else:
for var in item.available_variations:
if voucher and (voucher.allow_ignore_quota
or voucher.block_quota):
var.cached_availability = (Quota.AVAILABILITY_OK,
voucher.max_usages -
voucher.redeemed)
else:
var.cached_availability = list(
var.check_quotas(subevent=subevent,
_cache=quota_cache,
include_bundled=True))
var.order_max = min(
var.cached_availability[1] if var.cached_availability[1]
is not None else sys.maxsize, max_per_order)
original_price = var_price_override.get(var.pk, var.price)
if voucher:
price = voucher.calculate_price(original_price)
else:
price = original_price
var.display_price = var.tax(price,
currency=event.currency,
include_bundled=True)
if price != original_price:
var.original_price = original_price
display_add_to_cart = display_add_to_cart or var.order_max > 0
item.available_variations = [
v for v in item.available_variations
if v._subevent_quotas and (
not voucher or not voucher.quota_id or v in restrict_vars)
]
if voucher and voucher.variation_id:
item.available_variations = [
v for v in item.available_variations
if v.pk == voucher.variation_id
]
if len(item.available_variations) > 0:
item.min_price = min([
v.display_price.net if event.settings.display_net_prices
else v.display_price.gross
for v in item.available_variations
])
item.max_price = max([
v.display_price.net if event.settings.display_net_prices
else v.display_price.gross
for v in item.available_variations
])
item._remove = not bool(item.available_variations)
if not external_quota_cache:
event.cache.set('item_quota_cache', quota_cache, 5)
items = [
item for item in items
if (len(item.available_variations) > 0 or not item.has_variations)
and not item._remove
]
return items, display_add_to_cart
class RunApiViewSet(mixins.ListModelMixin, mixins.CreateModelMixin,
mixins.RetrieveModelMixin, mixins.UpdateModelMixin,
GenericViewSet):
queryset = (Run.objects.prefetch_related(
Prefetch("port_set", queryset=Port.objects.select_related(
"run"))).order_by("-created_date").all())
def get_serializer_class(self):
if self.action == "list":
return RunApiListSerializer
else:
return RunSerializerFull
@swagger_auto_schema(query_serializer=RunApiListSerializer)
def list(self, request, *args, **kwargs):
query_list_types = [
"job_groups",
"request_ids",
"inputs",
"tags",
"jira_ids",
"run_ids",
"apps",
"run",
"values_run",
"ports",
]
fixed_query_params = fix_query_list(request.query_params,
query_list_types)
serializer = RunApiListSerializer(data=fixed_query_params)
if serializer.is_valid():
queryset = time_filter(Run, fixed_query_params)
queryset = time_filter(Run,
request.query_params,
time_modal="modified_date",
previous_queryset=queryset)
job_groups = fixed_query_params.get("job_groups")
jira_ids = fixed_query_params.get("jira_ids")
run_ids = fixed_query_params.get("run_ids")
status_param = fixed_query_params.get("status")
ports = fixed_query_params.get("ports")
tags = fixed_query_params.get("tags")
operator_run = fixed_query_params.get("operator_run")
request_ids = fixed_query_params.get("request_ids")
apps = fixed_query_params.get("apps")
values_run = fixed_query_params.get("values_run")
run = fixed_query_params.get("run")
run_distribution = fixed_query_params.get("run_distribution")
count = fixed_query_params.get("count")
full = fixed_query_params.get("full")
if full:
full = bool(strtobool(full))
if operator_run:
queryset = queryset.filter(operator_run_id=operator_run)
if job_groups:
queryset = queryset.filter(job_group__in=job_groups)
if jira_ids:
queryset = queryset.filter(job_group__jira_id__in=jira_ids)
if run_ids:
queryset = queryset.filter(id__in=run_ids)
if status_param:
queryset = queryset.filter(
status=RunStatus[status_param].value)
if ports:
queryset = query_from_dict("port__%s__exact", queryset, ports)
if tags:
queryset = query_from_dict("tags__%s__contains", queryset,
tags)
if request_ids:
queryset = queryset.filter(tags__requestId__in=request_ids)
if apps:
queryset = queryset.filter(app__in=apps)
if run:
filter_query = dict()
for single_run in run:
key, value = single_run.split(":")
if value == "True" or value == "true":
value = True
if value == "False" or value == "false":
value = False
filter_query[key] = value
if filter_query:
queryset = queryset.filter(**filter_query)
if values_run:
if len(values_run) == 1:
ret_str = values_run[0]
queryset = queryset.values_list(
ret_str, flat=True).order_by(ret_str).distinct(ret_str)
else:
values_run_query_list = [
single_run for single_run in values_run
]
values_run_query_set = set(values_run_query_list)
sorted_query_list = sorted(values_run_query_set)
queryset = queryset.values_list(
*sorted_query_list).distinct()
if run_distribution:
distribution_dict = {}
run_query = run_distribution
run_ids = queryset.values_list("id", flat=True)
queryset = Run.objects.all()
queryset = queryset.filter(
id__in=run_ids).values(run_query).order_by().annotate(
Count(run_query))
for single_arg in queryset:
single_arg_name = None
single_arg_count = 0
for single_key, single_value in single_arg.items():
if "count" in single_key:
single_arg_count = single_value
else:
single_arg_name = single_value
if single_arg_name is not None:
distribution_dict[single_arg_name] = single_arg_count
return Response(distribution_dict, status=status.HTTP_200_OK)
if count:
count = bool(strtobool(count))
if count:
return Response(queryset.count(),
status=status.HTTP_200_OK)
try:
page = self.paginate_queryset(queryset.all())
except ValidationError as e:
return Response(e, status=status.HTTP_400_BAD_REQUEST)
if page is not None:
if values_run:
return self.get_paginated_response(page)
if full:
serializer = RunSerializerFull(page, many=True)
else:
serializer = RunSerializerPartial(page, many=True)
return self.get_paginated_response(serializer.data)
return Response([], status=status.HTTP_200_OK)
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def create(self, request, *args, **kwargs):
run_creator = RunCreator(**request.data)
if run_creator.is_valid():
run = run_creator.create()
response = RunSerializerFull(run)
create_run_task.delay(response.data["id"], request.data["inputs"])
job_group_notifier_id = str(run.job_group_notifier_id)
if job_group_notifier_id:
self._send_notifications(job_group_notifier_id, run)
return Response(response.data, status=status.HTTP_201_CREATED)
return Response("Error", status=status.HTTP_400_BAD_REQUEST)
@swagger_auto_schema(request_body=AbortRunSerializer)
@action(detail=False, methods=["post"])
def abort(self, request, *args, **kwargs):
serializer = AbortRunSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
job_group_id = request.data.get("job_group_id", None)
runs = request.data.get("runs", [])
abort_job_task.delay(job_group_id, runs)
return Response("Abort task submitted",
status=status.HTTP_202_ACCEPTED)
def _send_notifications(self, job_group_notifier_id, run):
pipeline_name = run.app.name
pipeline_version = run.app.version
pipeline_link = run.app.pipeline_link
pipeline_description_event = AddPipelineToDescriptionEvent(
job_group_notifier_id, pipeline_name, pipeline_version,
pipeline_link).to_dict()
send_notification.delay(pipeline_description_event)
run_event = RunStartedEvent(job_group_notifier_id, str(run.id),
run.app.name, run.app.pipeline_link,
run.output_directory, run.tags).to_dict()
send_notification.delay(run_event)
def listar_modulos_com_aulas():
aulas_ordenadas = Aula.objects.order_by('order')
return Modulo.objects.order_by('order').prefetch_related(
Prefetch('aula_set', queryset=aulas_ordenadas, to_attr='aulas')).all()
def with_orders(self):
qs = (Order.objects.with_total_value().with_payment().select_related(
'user').all())
return self.prefetch_related(
Prefetch('order_set', queryset=qs, to_attr='orders'))
class CourseViewSet(NonDestroyableModelViewSet):
product_attribute_value_prefetch = Prefetch(
'products__attribute_values',
queryset=ProductAttributeValue.objects.select_related(
'attribute').all())
products_prefetch = Prefetch(
'products',
queryset=Product.objects.select_related('parent__product_class').all())
lookup_value_regex = COURSE_ID_REGEX
serializer_class = serializers.CourseSerializer
permission_classes = (
IsAuthenticated,
IsAdminUser,
)
def get_queryset(self):
return Course.objects.filter(site=self.request.site).prefetch_related(
self.products_prefetch, self.product_attribute_value_prefetch,
'products__stockrecords')
def list(self, request, *args, **kwargs):
"""
List all courses.
---
parameters:
- name: include_products
description: Indicates if the related products should be included in the response.
required: false
type: boolean
paramType: query
multiple: false
"""
return super(CourseViewSet, self).list(request, *args, **kwargs)
def create(self, request, *args, **kwargs):
course = Course.objects.create(id=request.data['id'],
name=request.data['name'],
site=request.site)
data = serializers.CourseSerializer(course,
context={
'request': request
}).data
return Response(data, status=status.HTTP_201_CREATED)
def retrieve(self, request, *args, **kwargs):
"""
Retrieve details for a course.
---
parameters:
- name: include_products
description: Indicates if the related products should be included in the response.
required: false
type: boolean
paramType: query
multiple: false
"""
return super(CourseViewSet, self).retrieve(request, *args, **kwargs)
def get_serializer_context(self):
context = super(CourseViewSet, self).get_serializer_context()
context['include_products'] = bool(
self.request.GET.get('include_products', False))
return context
@detail_route(methods=['post'])
def publish(self, request, pk=None): # pylint: disable=unused-argument
""" Publish the course to LMS. """
course = self.get_object()
published = False
msg = 'Course [{course_id}] was not published to LMS ' \
'because the switch [publish_course_modes_to_lms] is disabled.'
if waffle.switch_is_active('publish_course_modes_to_lms'):
published = course.publish_to_lms()
if published:
msg = 'Course [{course_id}] was successfully published to LMS.'
else:
msg = 'An error occurred while publishing [{course_id}] to LMS.'
return Response({'status': msg.format(course_id=course.id)},
status=status.HTTP_200_OK if published else
status.HTTP_500_INTERNAL_SERVER_ERROR)
def get_data_for_es(cls, index, action):
"""
Load all the course runs from the Course model and format them for the ElasticSearch index
"""
for course in (Course.objects.filter(
extended_object__publisher_is_draft=False,
extended_object__title_set__published=True,
).prefetch_related(
Prefetch(
"extended_object__title_set",
to_attr="published_titles",
queryset=Title.objects.filter(published=True),
)).distinct()):
# Prepare published titles
titles = {
t.language: t.title
for t in course.extended_object.published_titles
}
# Prepare cover images
cover_images = {}
for cover_image in Picture.objects.filter(
cmsplugin_ptr__placeholder__page=course.extended_object,
cmsplugin_ptr__placeholder__slot="course_cover",
):
# Force the image format before computing it
cover_image.use_no_cropping = False
cover_image.width = COURSES_COVER_IMAGE_WIDTH
cover_image.height = COURSES_COVER_IMAGE_HEIGHT
cover_images[
cover_image.cmsplugin_ptr.language] = cover_image.img_src
# Prepare syllabus texts
syllabus_texts = defaultdict(list)
for simple_text in SimpleText.objects.filter(
cmsplugin_ptr__placeholder__page=course.extended_object,
cmsplugin_ptr__placeholder__slot="course_syllabus",
):
syllabus_texts[simple_text.cmsplugin_ptr.language].append(
simple_text.body)
# Make sure we get title information for categories in the same request
category_pages = (
course.get_root_to_leaf_category_pages().prefetch_related(
Prefetch(
"title_set",
to_attr="published_titles",
queryset=Title.objects.filter(published=True),
)).only("pk"))
# Make sure we get title information for organizations in the same request
organizations = (course.get_organizations().prefetch_related(
Prefetch(
"extended_object__title_set",
to_attr="published_titles",
queryset=Title.objects.filter(published=True),
)).only("extended_object").distinct())
course_runs = course.get_course_runs()
for course_run in course_runs:
yield {
"_id":
str(course_run.extended_object_id),
"_index":
index,
"_op_type":
action,
"_type":
cls.document_type,
"start":
course_run.start,
"end":
course_run.end,
"enrollment_start":
course_run.enrollment_start,
"enrollment_end":
course_run.enrollment_end,
"absolute_url": {
language:
course_run.extended_object.get_absolute_url(language)
for language in titles.keys()
},
"categories": [str(page.pk) for page in category_pages],
# Index the names of categories to surface them in full text searches
"categories_names":
reduce(
lambda acc, title: {
**acc,
title.language:
acc[title.language] + [title.title]
if acc.get(title.language) else [title.title],
},
[
title for page in category_pages
for title in page.published_titles
],
{},
),
"complete": {
language: slice_string_for_completion(title)
for language, title in titles.items()
},
"cover_image":
cover_images,
"description":
{l: " ".join(st)
for l, st in syllabus_texts.items()},
"is_new":
len(course_runs) == 1,
"languages":
course_run.languages,
"organizations": [
str(id)
for id in course.get_organizations().values_list(
"public_extension__extended_object", flat=True)
if id is not None
],
# Index the names of organizations to surface them in full text searches
"organizations_names":
reduce(
lambda acc, title: {
**acc,
title.language:
acc[title.language] + [title.title]
if acc.get(title.language) else [title.title],
},
[
title for organization in organizations for title
in organization.extended_object.published_titles
],
{},
),
"title":
titles,
}
def pokemonFullItemContext(context):
request = context['request']
context['stats'] = [
OrderedDict([
('hit_points', {
'value':
context['item'].hit_points,
'name':
_('Hit Points'),
'percent':
(context['item'].hit_points /
django_settings.POKEMONS_MAX_STATS['hit_points']) * 100,
}),
('attack', {
'value':
context['item'].attack,
'name':
_('Attack'),
'percent':
(context['item'].attack /
django_settings.POKEMONS_MAX_STATS['attack']) * 100,
}),
('defense', {
'value':
context['item'].defense,
'name':
_('Defense'),
'percent':
(context['item'].defense /
django_settings.POKEMONS_MAX_STATS['defense']) * 100,
}),
]),
OrderedDict([
('max_cp', {
'value':
context['item'].max_cp,
'name':
_('Max CP'),
'percent':
(context['item'].max_cp /
django_settings.POKEMONS_MAX_STATS['max_cp']) * 100,
}),
('catch_rate', {
'value':
context['item'].catch_rate,
'name':
_('Catch Rate'),
'percent':
(context['item'].catch_rate /
django_settings.POKEMONS_MAX_STATS['catch_rate']) * 100,
'suffix':
'%',
}),
('flee_rate', {
'value':
context['item'].flee_rate,
'name':
_('Flee Rate'),
'percent':
(context['item'].flee_rate /
django_settings.POKEMONS_MAX_STATS['flee_rate']) * 100,
'suffix':
'%',
}),
]),
]
context['tab'] = 'collection'
if 'tab' in request.GET and request.GET['tab'] in _full_pokemon_tabs:
context['tab'] = request.GET['tab']
if context['tab'] == 'collection':
if request.user.is_authenticated():
request.user.all_accounts = request.user.accounts.all(
).prefetch_related(
Prefetch('pokedex',
queryset=models.Pokedex.objects.filter(
pokemon_id=context['item'].id),
to_attr='in_pokedex'),
Prefetch('pokemons',
queryset=models.OwnedPokemon.objects.filter(
pokemon_id=context['item'].id).order_by('-cp'),
to_attr='all_pokemons'),
)
for account in request.user.all_accounts:
if len(account.in_pokedex):
account.in_pokedex = account.in_pokedex[0]
else:
account.in_pokedex = models.Pokedex.objects.create(
account=account, pokemon=context['item'])
for op in account.all_pokemons:
op.progress_percent = (op.cp / op.max_cp) * 100 if op.cp else 0
elif context['tab'] == 'evolutions':
pass
elif context['tab'] == 'attacks':
context['item'].all_attacks = context['item'].attacks.all().order_by(
'is_special')
def actions(self) -> List:
last_updated_action_ts = Action.objects.filter(team_id=self.team_id).aggregate(models.Max("updated_at"))[
"updated_at__max"
]
actions = (
Action.objects.filter(
team_id=self.team_id, steps__event=self.event, deleted=False, # filter by event name to narrow down
)
.distinct("id")
.prefetch_related(Prefetch("steps", queryset=ActionStep.objects.order_by("id")))
)
if not self._can_use_cached_query(last_updated_action_ts):
TEAM_ACTION_QUERY_CACHE[self.team_id], _ = actions.query.sql_with_params()
if len(actions) == 0:
return []
events: models.QuerySet[Any] = Event.objects.filter(pk=self.pk)
for action in actions:
events = events.annotate(
**{
f"action_{action.pk}": Event.objects.filter(pk=self.pk)
.query_db_by_action(action)
.values("id")[:1]
}
)
# This block is a little cryptic so bear with me
# We grab the query and the params from the ORM here
q, p = events.query.sql_with_params()
# We then take the parameters and replace the event id's with a placeholder
# We use this later to sub back in future event id's
# The rest of the parameters are shared between action types
qp = tuple(["%s" if i == self.pk else i for i in p])
# Create a cache item and add it to the cache keyed on team_id and event id
qcache = {self.event: (q, qp)}
TEAM_EVENT_ACTION_QUERY_CACHE[self.team_id].update(qcache)
# Update the last updated team action timestamp for future reference
LAST_UPDATED_TEAM_ACTION[self.team_id] = last_updated_action_ts
else:
# If we have reached this block we are about to use the sql query cache
# Grab the actions using the cached action query
actions.raw(TEAM_ACTION_QUERY_CACHE[self.team_id])
# Grab the cached query and query params, we will need to replace some params
q, p = TEAM_EVENT_ACTION_QUERY_CACHE[self.team_id][self.event]
# Replace the query param placeholders with the event id (opposite of what we did above)
qp = tuple([self.pk if i == "%s" else i for i in p])
with connection.cursor() as cursor:
# Format and execute the cached query using the mostly cached params
qstring = cursor.mogrify(q, qp)
cursor.execute(qstring)
events = namedtuplefetchall(cursor)
event = [event for event in events][0]
filtered_actions = [action for action in actions if getattr(event, f"action_{action.pk}", None)]
return filtered_actions
def get_grouped_items(event, subevent=None, voucher=None):
items = event.items.all().filter(
Q(active=True)
& Q(Q(available_from__isnull=True) | Q(available_from__lte=now()))
& Q(Q(available_until__isnull=True) | Q(available_until__gte=now()))
& ~Q(category__is_addon=True))
vouchq = Q(hide_without_voucher=False)
if voucher:
if voucher.item_id:
vouchq |= Q(pk=voucher.item_id)
items = items.filter(pk=voucher.item_id)
elif voucher.quota_id:
items = items.filter(quotas__in=[voucher.quota_id])
items = items.filter(vouchq).select_related(
'category',
'tax_rule', # for re-grouping
).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(subevent=subevent)),
Prefetch('variations',
to_attr='available_variations',
queryset=ItemVariation.objects.filter(
active=True, quotas__isnull=False).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(
subevent=subevent))).distinct()),
).annotate(quotac=Count('quotas'),
has_variations=Count('variations')).filter(
quotac__gt=0).order_by('category__position', 'category_id',
'position', 'name')
display_add_to_cart = False
external_quota_cache = event.cache.get('item_quota_cache')
quota_cache = external_quota_cache or {}
if subevent:
item_price_override = subevent.item_price_overrides
var_price_override = subevent.var_price_overrides
else:
item_price_override = {}
var_price_override = {}
for item in items:
if voucher and voucher.item_id and voucher.variation_id:
# Restrict variations if the voucher only allows one
item.available_variations = [
v for v in item.available_variations
if v.pk == voucher.variation_id
]
max_per_order = item.max_per_order or int(
event.settings.max_items_per_order)
if not item.has_variations:
item._remove = not bool(item._subevent_quotas)
if voucher and (voucher.allow_ignore_quota or voucher.block_quota):
item.cached_availability = (Quota.AVAILABILITY_OK,
voucher.max_usages -
voucher.redeemed)
else:
item.cached_availability = list(
item.check_quotas(subevent=subevent, _cache=quota_cache))
item.order_max = min(
item.cached_availability[1] if item.cached_availability[1]
is not None else sys.maxsize, max_per_order)
price = item_price_override.get(item.pk, item.default_price)
if voucher:
price = voucher.calculate_price(price)
item.display_price = item.tax(price)
display_add_to_cart = display_add_to_cart or item.order_max > 0
else:
for var in item.available_variations:
if voucher and (voucher.allow_ignore_quota
or voucher.block_quota):
var.cached_availability = (Quota.AVAILABILITY_OK,
voucher.max_usages -
voucher.redeemed)
else:
var.cached_availability = list(
var.check_quotas(subevent=subevent,
_cache=quota_cache))
var.order_max = min(
var.cached_availability[1] if var.cached_availability[1]
is not None else sys.maxsize, max_per_order)
price = var_price_override.get(var.pk, var.price)
if voucher:
price = voucher.calculate_price(price)
var.display_price = var.tax(price)
display_add_to_cart = display_add_to_cart or var.order_max > 0
item.available_variations = [
v for v in item.available_variations if v._subevent_quotas
]
if voucher and voucher.variation_id:
item.available_variations = [
v for v in item.available_variations
if v.pk == voucher.variation_id
]
if len(item.available_variations) > 0:
item.min_price = min([
v.display_price.net if event.settings.display_net_prices
else v.display_price.gross
for v in item.available_variations
])
item.max_price = max([
v.display_price.net if event.settings.display_net_prices
else v.display_price.gross
for v in item.available_variations
])
item._remove = not bool(item.available_variations)
if not external_quota_cache:
event.cache.set('item_quota_cache', quota_cache, 5)
items = [
item for item in items
if (len(item.available_variations) > 0 or not item.has_variations)
and not item._remove
]
return items, display_add_to_cart
class ChannelViewSet(
ChannelsMixin,
MultipleLookupDetailMixin,
mixins.CreateModelMixin,
mixins.RetrieveModelMixin,
mixins.UpdateModelMixin,
mixins.ListModelMixin,
mixins.DestroyModelMixin,
viewsets.GenericViewSet,
):
url_lookups = [
{
"lookup_field": "uuid",
"validator": serializers.serializers.UUIDField().to_internal_value,
},
{
"lookup_field":
"username",
"validator":
federation_utils.get_actor_data_from_username,
"get_query":
lambda v: Q(
actor__domain=v["domain"],
actor__preferred_username__iexact=v["username"],
),
},
]
filterset_class = filters.ChannelFilter
serializer_class = serializers.ChannelSerializer
queryset = (models.Channel.objects.all().prefetch_related(
"library",
"attributed_to",
"actor",
Prefetch("artist", queryset=ARTIST_PREFETCH_QS),
).order_by("-creation_date"))
permission_classes = [
oauth_permissions.ScopePermission,
permissions.OwnerPermission,
]
required_scope = "libraries"
anonymous_policy = "setting"
owner_checks = ["write"]
owner_field = "attributed_to.user"
owner_exception = exceptions.PermissionDenied
def get_serializer_class(self):
if self.request.method.lower() in ["head", "get", "options"]:
return serializers.ChannelSerializer
elif self.action in ["update", "partial_update"]:
return serializers.ChannelUpdateSerializer
return serializers.ChannelCreateSerializer
def perform_create(self, serializer):
return serializer.save(attributed_to=self.request.user.actor)
def list(self, request, *args, **kwargs):
if self.request.GET.get("output") == "opml":
queryset = self.filter_queryset(self.get_queryset())[:500]
opml = serializers.get_opml(
channels=queryset,
date=timezone.now(),
title="Funkwhale channels OPML export",
)
xml_body = renderers.render_xml(
renderers.dict_to_xml_tree("opml", opml))
return http.HttpResponse(xml_body, content_type="application/xml")
else:
return super().list(request, *args, **kwargs)
def get_object(self):
obj = super().get_object()
if (self.action == "retrieve"
and self.request.GET.get("refresh", "").lower() == "true"):
obj = music_views.refetch_obj(obj, self.get_queryset())
return obj
@decorators.action(
detail=True,
methods=["post"],
permission_classes=[rest_permissions.IsAuthenticated],
)
def subscribe(self, request, *args, **kwargs):
object = self.get_object()
subscription = federation_models.Follow(actor=request.user.actor)
subscription.fid = subscription.get_federation_id()
subscription, created = SubscriptionsViewSet.queryset.get_or_create(
target=object.actor,
actor=request.user.actor,
defaults={
"approved": True,
"fid": subscription.fid,
"uuid": subscription.uuid,
},
)
# prefetch stuff
subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
if not object.actor.is_local:
routes.outbox.dispatch({"type": "Follow"},
context={"follow": subscription})
data = serializers.SubscriptionSerializer(subscription).data
return response.Response(data, status=201)
@decorators.action(
detail=True,
methods=["post", "delete"],
permission_classes=[rest_permissions.IsAuthenticated],
)
def unsubscribe(self, request, *args, **kwargs):
object = self.get_object()
follow_qs = request.user.actor.emitted_follows.filter(
target=object.actor)
follow = follow_qs.first()
if follow:
if not object.actor.is_local:
routes.outbox.dispatch(
{
"type": "Undo",
"object": {
"type": "Follow"
}
},
context={"follow": follow},
)
follow_qs.delete()
return response.Response(status=204)
@decorators.action(
detail=True,
methods=["get"],
content_negotiation_class=renderers.PodcastRSSContentNegociation,
)
def rss(self, request, *args, **kwargs):
object = self.get_object()
if not object.attributed_to.is_local:
return response.Response({"detail": "Not found"}, status=404)
if object.attributed_to == actors.get_service_actor():
# external feed, we redirect to the canonical one
return http.HttpResponseRedirect(object.rss_url)
uploads = (object.library.uploads.playable_by(None).prefetch_related(
Prefetch(
"track",
queryset=music_models.Track.objects.select_related(
"attachment_cover", "description").prefetch_related(
music_views.TAG_PREFETCH, ),
), ).select_related(
"track__attachment_cover",
"track__description").order_by("-creation_date"))[:50]
data = serializers.rss_serialize_channel_full(channel=object,
uploads=uploads)
return response.Response(data, status=200)
@decorators.action(
methods=["get"],
detail=False,
url_path="metadata-choices",
url_name="metadata_choices",
permission_classes=[],
)
def metedata_choices(self, request, *args, **kwargs):
data = {
"language": [{
"value": code,
"label": name
} for code, name in locales.ISO_639_CHOICES],
"itunes_category": [{
"value": code,
"label": code,
"children": children
} for code, children in categories.ITUNES_CATEGORIES.items()],
}
return response.Response(data)
@decorators.action(
methods=["post"],
detail=False,
url_path="rss-subscribe",
url_name="rss_subscribe",
)
@transaction.atomic
def rss_subscribe(self, request, *args, **kwargs):
serializer = serializers.RssSubscribeSerializer(data=request.data)
if not serializer.is_valid():
return response.Response(serializer.errors, status=400)
channel = (models.Channel.objects.filter(
rss_url=serializer.validated_data["url"], ).order_by("id").first())
if not channel:
# try to retrieve the channel via its URL and create it
try:
channel, uploads = serializers.get_channel_from_rss_url(
serializer.validated_data["url"])
except serializers.FeedFetchException as e:
return response.Response(
{"detail": str(e)},
status=400,
)
subscription = federation_models.Follow(actor=request.user.actor)
subscription.fid = subscription.get_federation_id()
subscription, created = SubscriptionsViewSet.queryset.get_or_create(
target=channel.actor,
actor=request.user.actor,
defaults={
"approved": True,
"fid": subscription.fid,
"uuid": subscription.uuid,
},
)
# prefetch stuff
subscription = SubscriptionsViewSet.queryset.get(pk=subscription.pk)
return response.Response(
serializers.SubscriptionSerializer(subscription).data, status=201)
def get_serializer_context(self):
context = super().get_serializer_context()
context["subscriptions_count"] = self.action in [
"retrieve",
"create",
"update",
"partial_update",
]
if self.request.user.is_authenticated:
context["actor"] = self.request.user.actor
return context
@transaction.atomic
def perform_destroy(self, instance):
instance.__class__.objects.filter(pk=instance.pk).delete()
common_utils.on_commit(federation_tasks.remove_actor.delay,
actor_id=instance.actor.pk)
def get_grouped_items(event, subevent=None, voucher=None, channel='web', require_seat=0):
items = event.items.using(settings.DATABASE_REPLICA).filter_available(channel=channel, voucher=voucher).select_related(
'category', 'tax_rule', # for re-grouping
'hidden_if_available',
).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.using(settings.DATABASE_REPLICA).filter(subevent=subevent)),
Prefetch('bundles',
queryset=ItemBundle.objects.using(settings.DATABASE_REPLICA).prefetch_related(
Prefetch('bundled_item',
queryset=event.items.using(settings.DATABASE_REPLICA).select_related('tax_rule').prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.using(settings.DATABASE_REPLICA).filter(subevent=subevent)),
)),
Prefetch('bundled_variation',
queryset=ItemVariation.objects.using(
settings.DATABASE_REPLICA
).select_related('item', 'item__tax_rule').filter(item__event=event).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.using(settings.DATABASE_REPLICA).filter(subevent=subevent)),
)),
)),
Prefetch('variations', to_attr='available_variations',
queryset=ItemVariation.objects.using(settings.DATABASE_REPLICA).filter(active=True, quotas__isnull=False).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.using(settings.DATABASE_REPLICA).filter(subevent=subevent))
).distinct()),
).annotate(
quotac=Count('quotas'),
has_variations=Count('variations'),
requires_seat=Exists(
SeatCategoryMapping.objects.filter(
product_id=OuterRef('pk'),
subevent=subevent
)
)
).filter(
quotac__gt=0,
).order_by('category__position', 'category_id', 'position', 'name')
if require_seat:
items = items.filter(requires_seat__gt=0)
else:
items = items.filter(requires_seat=0)
display_add_to_cart = False
external_quota_cache = event.cache.get('item_quota_cache')
quota_cache = external_quota_cache or {}
if subevent:
item_price_override = subevent.item_price_overrides
var_price_override = subevent.var_price_overrides
else:
item_price_override = {}
var_price_override = {}
restrict_vars = set()
if voucher and voucher.quota_id:
# If a voucher is set to a specific quota, we need to filter out on that level
restrict_vars = set(voucher.quota.variations.all())
for item in items:
if voucher and voucher.item_id and voucher.variation_id:
# Restrict variations if the voucher only allows one
item.available_variations = [v for v in item.available_variations
if v.pk == voucher.variation_id]
if get_all_sales_channels()[channel].unlimited_items_per_order:
max_per_order = sys.maxsize
else:
max_per_order = item.max_per_order or int(event.settings.max_items_per_order)
if item.hidden_if_available:
q = item.hidden_if_available.availability(_cache=quota_cache)
if q[0] == Quota.AVAILABILITY_OK:
item._remove = True
continue
item.description = str(item.description)
for recv, resp in item_description.send(sender=event, item=item, variation=None):
if resp:
item.description += ("<br/>" if item.description else "") + resp
if not item.has_variations:
item._remove = False
if not bool(item._subevent_quotas):
item._remove = True
continue
if voucher and (voucher.allow_ignore_quota or voucher.block_quota):
item.cached_availability = (
Quota.AVAILABILITY_OK, voucher.max_usages - voucher.redeemed
)
else:
item.cached_availability = list(
item.check_quotas(subevent=subevent, _cache=quota_cache, include_bundled=True)
)
if event.settings.hide_sold_out and item.cached_availability[0] < Quota.AVAILABILITY_RESERVED:
item._remove = True
continue
item.order_max = min(
item.cached_availability[1]
if item.cached_availability[1] is not None else sys.maxsize,
max_per_order
)
original_price = item_price_override.get(item.pk, item.default_price)
if voucher:
price = voucher.calculate_price(original_price)
else:
price = original_price
item.display_price = item.tax(price, currency=event.currency, include_bundled=True)
if price != original_price:
item.original_price = item.tax(original_price, currency=event.currency, include_bundled=True)
else:
item.original_price = (
item.tax(item.original_price, currency=event.currency, include_bundled=True,
base_price_is='net' if event.settings.display_net_prices else 'gross') # backwards-compat
if item.original_price else None
)
display_add_to_cart = display_add_to_cart or item.order_max > 0
else:
for var in item.available_variations:
var.description = str(var.description)
for recv, resp in item_description.send(sender=event, item=item, variation=var):
if resp:
var.description += ("<br/>" if var.description else "") + resp
if voucher and (voucher.allow_ignore_quota or voucher.block_quota):
var.cached_availability = (
Quota.AVAILABILITY_OK, voucher.max_usages - voucher.redeemed
)
else:
var.cached_availability = list(
var.check_quotas(subevent=subevent, _cache=quota_cache, include_bundled=True)
)
var.order_max = min(
var.cached_availability[1]
if var.cached_availability[1] is not None else sys.maxsize,
max_per_order
)
original_price = var_price_override.get(var.pk, var.price)
if voucher:
price = voucher.calculate_price(original_price)
else:
price = original_price
var.display_price = var.tax(price, currency=event.currency, include_bundled=True)
if price != original_price:
var.original_price = var.tax(original_price, currency=event.currency, include_bundled=True)
else:
var.original_price = (
var.tax(var.original_price or item.original_price, currency=event.currency,
include_bundled=True,
base_price_is='net' if event.settings.display_net_prices else 'gross') # backwards-compat
) if var.original_price or item.original_price else None
display_add_to_cart = display_add_to_cart or var.order_max > 0
item.original_price = (
item.tax(item.original_price, currency=event.currency, include_bundled=True,
base_price_is='net' if event.settings.display_net_prices else 'gross') # backwards-compat
if item.original_price else None
)
item.available_variations = [
v for v in item.available_variations if v._subevent_quotas and (
not voucher or not voucher.quota_id or v in restrict_vars
)
]
if event.settings.hide_sold_out:
item.available_variations = [v for v in item.available_variations
if v.cached_availability[0] >= Quota.AVAILABILITY_RESERVED]
if voucher and voucher.variation_id:
item.available_variations = [v for v in item.available_variations
if v.pk == voucher.variation_id]
if len(item.available_variations) > 0:
item.min_price = min([v.display_price.net if event.settings.display_net_prices else
v.display_price.gross for v in item.available_variations])
item.max_price = max([v.display_price.net if event.settings.display_net_prices else
v.display_price.gross for v in item.available_variations])
item._remove = not bool(item.available_variations)
if not external_quota_cache:
event.cache.set('item_quota_cache', quota_cache, 5)
items = [item for item in items
if (len(item.available_variations) > 0 or not item.has_variations) and not item._remove]
return items, display_add_to_cart
def with_user_orders(self, user):
qs = (Order.objects.filter(
user=user).with_total_value().with_payment().all())
return self.prefetch_related(
Prefetch('order_set', queryset=qs, to_attr='user_orders'))
class XFormViewSet(AnonymousUserPublicFormsMixin,
CacheControlMixin,
AuthenticateHeaderMixin,
ETagsMixin,
LabelsMixin,
BaseViewset,
ModelViewSet):
"""
Publish XLSForms, List, Retrieve Published Forms.
"""
renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [
renderers.XLSRenderer,
renderers.XLSXRenderer,
renderers.CSVRenderer,
renderers.CSVZIPRenderer,
renderers.SAVZIPRenderer,
renderers.SurveyRenderer,
renderers.OSMExportRenderer,
renderers.ZipRenderer,
renderers.GoogleSheetsRenderer
]
queryset = XForm.objects.select_related('user', 'created_by')\
.prefetch_related(
Prefetch(
'xformuserobjectpermission_set',
queryset=XFormUserObjectPermission.objects.select_related(
'user__profile__organizationprofile',
'permission'
)
),
Prefetch('metadata_set'),
Prefetch('tags'),
Prefetch('dataview_set')
).only(
'id', 'id_string', 'title', 'shared', 'shared_data',
'require_auth', 'created_by', 'num_of_submissions',
'downloadable', 'encrypted', 'sms_id_string',
'date_created', 'date_modified', 'last_submission_time',
'uuid', 'bamboo_dataset', 'instances_with_osm',
'instances_with_geopoints', 'version', 'has_hxl_support',
'project', 'last_updated_at', 'user', 'allows_sms', 'description',
'is_merged_dataset'
)
serializer_class = XFormSerializer
lookup_field = 'pk'
extra_lookup_fields = None
permission_classes = [XFormPermissions, ]
updatable_fields = set(('description', 'downloadable', 'require_auth',
'shared', 'shared_data', 'title'))
filter_backends = (filters.EnketoAnonDjangoObjectPermissionFilter,
filters.TagFilter,
filters.XFormOwnerFilter,
DjangoFilterBackend)
filter_fields = ('instances_with_osm',)
public_forms_endpoint = 'public'
def get_serializer_class(self):
if self.action == 'list':
return XFormBaseSerializer
return super(XFormViewSet, self).get_serializer_class()
def create(self, request, *args, **kwargs):
try:
owner = _get_owner(request)
except ValidationError as e:
return Response({'message': e.messages[0]},
status=status.HTTP_400_BAD_REQUEST)
survey = utils.publish_xlsform(request, owner)
if isinstance(survey, XForm):
# survey is a DataDictionary we need an XForm to return the correct
# role for the user after form publishing.
serializer = XFormCreateSerializer(
survey, context={'request': request})
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED,
headers=headers)
return Response(survey, status=status.HTTP_400_BAD_REQUEST)
@action(methods=['POST', 'GET'], detail=False)
def create_async(self, request, *args, **kwargs):
""" Temporary Endpoint for Async form creation """
resp = headers = {}
resp_code = status.HTTP_400_BAD_REQUEST
if request.method == 'GET':
self.etag_data = '{}'.format(timezone.now())
survey = tasks.get_async_status(
request.query_params.get('job_uuid'))
if 'pk' in survey:
xform = XForm.objects.get(pk=survey.get('pk'))
serializer = XFormSerializer(
xform, context={'request': request})
headers = self.get_success_headers(serializer.data)
resp = serializer.data
resp_code = status.HTTP_201_CREATED
else:
resp_code = status.HTTP_202_ACCEPTED
resp.update(survey)
else:
try:
owner = _get_owner(request)
except ValidationError as e:
return Response({'message': e.messages[0]},
status=status.HTTP_400_BAD_REQUEST)
fname = request.FILES.get('xls_file').name
if isinstance(request.FILES.get('xls_file'), InMemoryUploadedFile):
xls_file_path = default_storage.save(
f'tmp/async-upload-{owner.username}-{fname}',
ContentFile(request.FILES.get('xls_file').read()))
else:
xls_file_path = request.FILES.get(
'xls_file').temporary_file_path()
resp.update(
{u'job_uuid':
tasks.publish_xlsform_async.delay(
request.user.id,
request.POST,
owner.id,
{'name': fname, 'path': xls_file_path}).task_id})
resp_code = status.HTTP_202_ACCEPTED
return Response(data=resp, status=resp_code, headers=headers)
@action(methods=['GET', 'HEAD'], detail=True)
@never_cache
def form(self, request, format='json', **kwargs):
form = self.get_object()
if format not in ['json', 'xml', 'xls', 'csv']:
return HttpResponseBadRequest('400 BAD REQUEST',
content_type='application/json',
status=400)
self.etag_data = '{}'.format(form.date_modified)
filename = form.id_string + "." + format
response = response_for_format(form, format=format)
response['Content-Disposition'] = 'attachment; filename=' + filename
return response
@action(methods=['GET'], detail=False)
def login(self, request, **kwargs):
return_url = request.query_params.get('return')
if return_url:
redirect = parse_webform_return_url(return_url, request)
if redirect:
return redirect
login_vars = {"login_url": settings.ENKETO_CLIENT_LOGIN_URL,
"return_url": urlencode({'return_url': return_url})}
client_login = '******'.format(**login_vars)
return HttpResponseRedirect(client_login)
return HttpResponseForbidden(
"Authentication failure, cannot redirect")
@action(methods=['GET'], detail=True)
def enketo(self, request, **kwargs):
"""Expose enketo urls."""
survey_type = self.kwargs.get('survey_type') or \
request.GET.get('survey_type')
self.object = self.get_object()
form_url = get_form_url(
request, self.object.user.username,
protocol=settings.ENKETO_PROTOCOL,
xform_pk=self.object.pk, generate_consistent_urls=True)
data = {'message': _(u"Enketo not properly configured.")}
http_status = status.HTTP_400_BAD_REQUEST
try:
# pass default arguments to enketo_url to prepopulate form fields
request_vars = request.GET
defaults = generate_enketo_form_defaults(
self.object, **request_vars)
url = enketo_url(
form_url, self.object.id_string, **defaults)
preview_url = get_enketo_preview_url(request,
self.object.user.username,
self.object.id_string,
xform_pk=self.object.pk)
except EnketoError as e:
data = {'message': _(u"Enketo error: %s" % e)}
else:
if survey_type == 'single':
single_submit_url = get_enketo_single_submit_url(
request, self.object.user.username, self.object.id_string,
xform_pk=self.object.pk)
data = {"single_submit_url": single_submit_url}
elif url and preview_url:
http_status = status.HTTP_200_OK
data = {"enketo_url": url,
"enketo_preview_url": preview_url}
return Response(data, http_status)
@action(methods=['POST', 'GET'], detail=False)
def survey_preview(self, request, **kwargs):
username = request.user.username
if request.method.upper() == 'POST':
if not username:
raise ParseError("User has to be authenticated")
csv_data = request.data.get('body')
if csv_data:
rand_name = "survey_draft_%s.csv" % ''.join(
random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6))
csv_file = ContentFile(csv_data)
csv_name = default_storage.save(
upload_to_survey_draft(rand_name, username),
csv_file)
result = publish_form(lambda: get_survey_xml(csv_name))
if result_has_error(result):
raise ParseError(result.get('text'))
return Response(
{'unique_string': rand_name, 'username': username},
status=200)
else:
raise ParseError('Missing body')
if request.method.upper() == 'GET':
filename = request.query_params.get('filename')
username = request.query_params.get('username')
if not username:
raise ParseError('Username not provided')
if not filename:
raise ParseError("Filename MUST be provided")
csv_name = upload_to_survey_draft(filename, username)
result = publish_form(lambda: get_survey_xml(csv_name))
if result_has_error(result):
raise ParseError(result.get('text'))
self.etag_data = result
return Response(result, status=200)
def retrieve(self, request, *args, **kwargs):
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
if lookup == self.public_forms_endpoint:
self.object_list = self._get_public_forms_queryset()
page = self.paginate_queryset(self.object_list)
if page is not None:
serializer = self.get_pagination_serializer(page)
else:
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
xform = self.get_object()
export_type = kwargs.get('format') or \
request.query_params.get('format')
query = request.query_params.get("query")
token = request.GET.get('token')
meta = request.GET.get('meta')
if export_type is None or export_type in ['json', 'debug']:
# perform default viewset retrieve, no data export
return super(XFormViewSet, self).retrieve(request, *args, **kwargs)
return custom_response_handler(request,
xform,
query,
export_type,
token,
meta)
@action(methods=['POST'], detail=True)
def share(self, request, *args, **kwargs):
self.object = self.get_object()
usernames_str = request.data.get("usernames",
request.data.get("username"))
if not usernames_str:
return Response(status=status.HTTP_400_BAD_REQUEST)
role = request.data.get("role") # the serializer validates the role
xform_id = self.object.pk
data_list = [{"xform": xform_id,
"username": username,
"role": role}
for username in usernames_str.split(",")]
serializer = ShareXFormSerializer(data=data_list, many=True)
if serializer.is_valid():
serializer.save()
else:
return Response(data=serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_204_NO_CONTENT)
@action(methods=['POST'], detail=True)
def clone(self, request, *args, **kwargs):
self.object = self.get_object()
data = {'xform': self.object.pk,
'username': request.data.get('username')}
project = request.data.get('project_id')
if project:
data['project'] = project
serializer = CloneXFormSerializer(data=data)
if serializer.is_valid():
clone_to_user = User.objects.get(username=data['username'])
if not request.user.has_perm('can_add_xform',
clone_to_user.profile):
raise exceptions.PermissionDenied(
detail=_(u"User %(user)s has no permission to add "
"xforms to account %(account)s" %
{'user': request.user.username,
'account': data['username']}))
try:
xform = serializer.save()
except IntegrityError:
raise ParseError(
'A clone with the same id_string has already been created')
serializer = XFormSerializer(
xform.cloned_form, context={'request': request})
return Response(data=serializer.data,
status=status.HTTP_201_CREATED)
return Response(data=serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
@action(
methods=['POST', 'GET'], detail=True,
url_name='import', url_path='import')
def data_import(self, request, *args, **kwargs):
""" Endpoint for CSV and XLS data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling and
:py:func:`onadata.libs.utils.csv_import.submission_xls_to_csv`
for POST request passing the `request.FILES.get('xls_file')` upload for
import if xls_file is provided instead of csv_file
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
xls_file = request.FILES.get('xls_file', None)
if csv_file is None and xls_file is None:
resp.update({u'error': u'csv_file and xls_file field empty'})
elif xls_file and \
xls_file.name.split('.')[-1] not in XLS_EXTENSIONS:
resp.update({u'error': u'xls_file not an excel file'})
elif csv_file and csv_file.name.split('.')[-1] != CSV_EXTENSION:
resp.update({u'error': u'csv_file not a csv file'})
else:
if xls_file and xls_file.name.split('.')[-1] in XLS_EXTENSIONS:
csv_file = submission_xls_to_csv(xls_file)
overwrite = request.query_params.get('overwrite')
overwrite = True \
if overwrite and overwrite.lower() == 'true' else False
size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD
try:
csv_size = csv_file.size
except AttributeError:
csv_size = csv_file.__sizeof__()
if csv_size < size_threshold:
resp.update(submit_csv(request.user.username,
self.object, csv_file, overwrite))
else:
csv_file.seek(0)
upload_to = os.path.join(request.user.username,
'csv_imports', csv_file.name)
file_name = default_storage.save(upload_to, csv_file)
task = submit_csv_async.delay(request.user.username,
self.object.pk, file_name,
overwrite)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
@action(methods=['POST', 'GET'], detail=True)
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
elif csv_file.name.split('.')[-1] != CSV_EXTENSION:
resp.update({u'error': u'csv_file not a csv file'})
else:
overwrite = request.query_params.get('overwrite')
overwrite = True \
if overwrite and overwrite.lower() == 'true' else False
size_threshold = settings.CSV_FILESIZE_IMPORT_ASYNC_THRESHOLD
if csv_file.size < size_threshold:
resp.update(submit_csv(request.user.username,
self.object, csv_file, overwrite))
else:
csv_file.seek(0)
upload_to = os.path.join(request.user.username,
'csv_imports', csv_file.name)
file_name = default_storage.save(upload_to, csv_file)
task = submit_csv_async.delay(request.user.username,
self.object.pk, file_name,
overwrite)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
def partial_update(self, request, *args, **kwargs):
self.object = self.get_object()
owner = self.object.user
# updating the file
if request.FILES or set(['xls_url',
'dropbox_xls_url',
'text_xls_form']) & set(request.data):
return _try_update_xlsform(request, self.object, owner)
try:
return super(XFormViewSet, self).partial_update(request, *args,
**kwargs)
except XLSFormError as e:
raise ParseError(str(e))
@action(methods=['DELETE', 'GET'], detail=True)
def delete_async(self, request, *args, **kwargs):
if request.method == 'DELETE':
xform = self.get_object()
resp = {
u'job_uuid': tasks.delete_xform_async.delay(
xform.pk,
request.user.id).task_id,
u'time_async_triggered': datetime.now()}
# clear project from cache
safe_delete(f'{PROJ_OWNER_CACHE}{xform.project.pk}')
resp_code = status.HTTP_202_ACCEPTED
elif request.method == 'GET':
job_uuid = request.query_params.get('job_uuid')
resp = tasks.get_async_status(job_uuid)
resp_code = status.HTTP_202_ACCEPTED
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp, status=resp_code)
def destroy(self, request, *args, **kwargs):
xform = self.get_object()
user = request.user
xform.soft_delete(user=user)
return Response(status=status.HTTP_204_NO_CONTENT)
@action(methods=['GET'], detail=True)
def export_async(self, request, *args, **kwargs):
job_uuid = request.query_params.get('job_uuid')
export_type = request.query_params.get('format')
query = request.query_params.get("query")
xform = self.get_object()
token = request.query_params.get('token')
meta = request.query_params.get('meta')
data_id = request.query_params.get('data_id')
options = parse_request_export_options(request.query_params)
options.update({
'meta': meta,
'token': token,
'data_id': data_id,
})
if query:
options.update({'query': query})
if job_uuid:
try:
resp = get_async_response(job_uuid, request, xform)
except Export.DoesNotExist:
# if this does not exist retry it against the primary
try:
with use_master:
resp = get_async_response(job_uuid, request, xform)
except NameError:
resp = get_async_response(job_uuid, request, xform)
else:
resp = process_async_export(request, xform, export_type, options)
if isinstance(resp, HttpResponseRedirect):
payload = {
"details": _("Google authorization needed"),
"url": resp.url
}
return Response(data=payload,
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp,
status=status.HTTP_202_ACCEPTED,
content_type="application/json")
def _get_streaming_response(self):
"""
Get a StreamingHttpResponse response object
"""
# use queryset_iterator. Will need to change this to the Django
# native .iterator() method when we upgrade to django version 2
# because in Django 2 .iterator() has support for chunk size
queryset = queryset_iterator(self.object_list, chunksize=2000)
def get_json_string(item):
return json.dumps(XFormBaseSerializer(
instance=item,
context={'request': self.request}
).data)
response = StreamingHttpResponse(
json_stream(queryset, get_json_string),
content_type="application/json"
)
# calculate etag value and add it to response headers
if hasattr(self, 'etag_data'):
self.set_etag_header(None, self.etag_data)
self.set_cache_control(response)
# set headers on streaming response
for k, v in self.headers.items():
response[k] = v
return response
def list(self, request, *args, **kwargs):
STREAM_DATA = getattr(settings, 'STREAM_DATA', False)
try:
queryset = self.filter_queryset(self.get_queryset())
last_modified = queryset.values_list('date_modified', flat=True)\
.order_by('-date_modified')
if last_modified:
self.etag_data = last_modified[0].isoformat()
if STREAM_DATA:
self.object_list = queryset
resp = self._get_streaming_response()
else:
resp = super(XFormViewSet, self).list(request, *args, **kwargs)
except XLSFormError as e:
resp = HttpResponseBadRequest(e)
return resp
def get_teams(self, obj):
teams = Team.objects.filter(challenge=obj).prefetch_related('competitors') \
.prefetch_related(Prefetch('competitors__stats', queryset=Stats.objects.filter(challenge=obj)))
return TeamSerializer(teams, many=True).data
class XFormViewSet(AnonymousUserPublicFormsMixin,
CacheControlMixin,
AuthenticateHeaderMixin,
ETagsMixin,
LabelsMixin,
BaseViewset,
ModelViewSet):
"""
Publish XLSForms, List, Retrieve Published Forms.
"""
renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [
renderers.XLSRenderer,
renderers.XLSXRenderer,
renderers.CSVRenderer,
renderers.CSVZIPRenderer,
renderers.SAVZIPRenderer,
renderers.SurveyRenderer,
renderers.OSMExportRenderer,
renderers.ZipRenderer,
renderers.GoogleSheetsRenderer
]
queryset = XForm.objects.select_related('user', 'created_by')\
.prefetch_related(
Prefetch(
'xformuserobjectpermission_set',
queryset=XFormUserObjectPermission.objects.select_related(
'user__profile__organizationprofile',
'permission'
)
),
Prefetch('metadata_set'),
Prefetch('tags'),
Prefetch('dataview_set')
)
serializer_class = XFormSerializer
lookup_field = 'pk'
extra_lookup_fields = None
permission_classes = [XFormPermissions, ]
updatable_fields = set(('description', 'downloadable', 'require_auth',
'shared', 'shared_data', 'title'))
filter_backends = (filters.AnonDjangoObjectPermissionFilter,
filters.TagFilter,
filters.XFormOwnerFilter,
DjangoFilterBackend)
filter_fields = ('instances_with_osm',)
public_forms_endpoint = 'public'
def get_serializer_class(self):
if self.action == 'list':
return XFormBaseSerializer
return super(XFormViewSet, self).get_serializer_class()
def create(self, request, *args, **kwargs):
try:
owner = _get_owner(request)
except ValidationError as e:
return Response({'message': e.messages[0]},
status=status.HTTP_400_BAD_REQUEST)
survey = utils.publish_xlsform(request, owner)
if isinstance(survey, XForm):
serializer = XFormCreateSerializer(
survey, context={'request': request})
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED,
headers=headers)
return Response(survey, status=status.HTTP_400_BAD_REQUEST)
@list_route(methods=['POST', 'GET'])
def create_async(self, request, *args, **kwargs):
""" Temporary Endpoint for Async form creation """
resp = headers = {}
resp_code = status.HTTP_400_BAD_REQUEST
if request.method == 'GET':
self.etag_data = '{}'.format(timezone.now())
survey = tasks.get_async_status(
request.query_params.get('job_uuid'))
if 'pk' in survey:
xform = XForm.objects.get(pk=survey.get('pk'))
serializer = XFormSerializer(
xform, context={'request': request})
headers = self.get_success_headers(serializer.data)
resp = serializer.data
resp_code = status.HTTP_201_CREATED
else:
resp_code = status.HTTP_202_ACCEPTED
resp.update(survey)
else:
try:
owner = _get_owner(request)
except ValidationError as e:
return Response({'message': e.messages[0]},
status=status.HTTP_400_BAD_REQUEST)
fname = request.FILES.get('xls_file').name
resp.update(
{u'job_uuid':
tasks.publish_xlsform_async.delay(
request.user, request.POST, owner,
({'name': fname,
'data': request.FILES.get('xls_file').read()}
if isinstance(request.FILES.get('xls_file'),
InMemoryUploadedFile) else
{'name': fname,
'path': request.FILES.get(
'xls_file').temporary_file_path()})).task_id})
resp_code = status.HTTP_202_ACCEPTED
return Response(data=resp, status=resp_code, headers=headers)
@detail_route()
@never_cache
def form(self, request, format='json', **kwargs):
form = self.get_object()
if format not in ['json', 'xml', 'xls']:
return HttpResponseBadRequest('400 BAD REQUEST',
content_type='application/json',
status=400)
self.etag_data = '{}'.format(form.date_modified)
filename = form.id_string + "." + format
response = response_for_format(form, format=format)
response['Content-Disposition'] = 'attachment; filename=' + filename
return response
@list_route(methods=['GET'])
def login(self, request, **kwargs):
return_url = request.query_params.get('return')
if return_url:
redirect = parse_webform_return_url(return_url, request)
if redirect:
return redirect
login_vars = {"login_url": settings.ENKETO_CLIENT_LOGIN_URL,
"return_url": urlencode({'return_url': return_url})}
client_login = '******'.format(**login_vars)
return HttpResponseRedirect(client_login)
return HttpResponseForbidden(
"Authentication failure, cannot redirect")
@detail_route()
def enketo(self, request, **kwargs):
self.object = self.get_object()
form_url = get_form_url(
request, self.object.user.username, settings.ENKETO_PROTOCOL)
data = {'message': _(u"Enketo not properly configured.")}
http_status = status.HTTP_400_BAD_REQUEST
try:
# pass default arguments to enketo_url to prepopulate form fields
request_vars = request.GET
defaults = generate_enketo_form_defaults(
self.object, **request_vars)
url = enketo_url(form_url, self.object.id_string, **defaults)
preview_url = get_enketo_preview_url(request,
self.object.user.username,
self.object.id_string)
except EnketoError as e:
data = {'message': _(u"Enketo error: %s" % e)}
else:
if url and preview_url:
http_status = status.HTTP_200_OK
data = {"enketo_url": url, "enketo_preview_url": preview_url}
return Response(data, http_status)
@list_route(methods=['POST', 'GET'])
def survey_preview(self, request, **kwargs):
username = request.user.username
if request.method.upper() == 'POST':
if not username:
raise ParseError("User has to be authenticated")
csv_data = request.data.get('body')
if csv_data:
rand_name = "survey_draft_%s.csv" % ''.join(
random.sample("abcdefghijklmnopqrstuvwxyz0123456789", 6))
csv_file = ContentFile(csv_data)
csv_name = default_storage.save(
upload_to_survey_draft(rand_name, username),
csv_file)
result = publish_form(lambda: get_survey_xml(csv_name))
if result_has_error(result):
raise ParseError(result.get('text'))
return Response(
{'unique_string': rand_name, 'username': username},
status=200)
else:
raise ParseError('Missing body')
if request.method.upper() == 'GET':
filename = request.query_params.get('filename')
username = request.query_params.get('username')
if not username:
raise ParseError('Username not provided')
if not filename:
raise ParseError("Filename MUST be provided")
csv_name = upload_to_survey_draft(filename, username)
result = publish_form(lambda: get_survey_xml(csv_name))
if result_has_error(result):
raise ParseError(result.get('text'))
self.etag_data = result
return Response(result, status=200)
def retrieve(self, request, *args, **kwargs):
lookup_field = self.lookup_field
lookup = self.kwargs.get(lookup_field)
if lookup == self.public_forms_endpoint:
self.object_list = self._get_public_forms_queryset()
page = self.paginate_queryset(self.object_list)
if page is not None:
serializer = self.get_pagination_serializer(page)
else:
serializer = self.get_serializer(self.object_list, many=True)
return Response(serializer.data)
xform = self.get_object()
export_type = kwargs.get('format') or \
request.query_params.get('format')
query = request.query_params.get("query", {})
token = request.GET.get('token')
meta = request.GET.get('meta')
if export_type is None or export_type in ['json', 'debug']:
# perform default viewset retrieve, no data export
return super(XFormViewSet, self).retrieve(request, *args, **kwargs)
return custom_response_handler(request,
xform,
query,
export_type,
token,
meta)
@detail_route(methods=['POST'])
def share(self, request, *args, **kwargs):
self.object = self.get_object()
usernames_str = request.data.get("usernames",
request.data.get("username"))
if not usernames_str:
return Response(status=status.HTTP_400_BAD_REQUEST)
role = request.data.get("role") # the serializer validates the role
xform_id = self.object.pk
data_list = [{"xform": xform_id,
"username": username,
"role": role}
for username in usernames_str.split(",")]
serializer = ShareXFormSerializer(data=data_list, many=True)
if serializer.is_valid():
serializer.save()
else:
return Response(data=serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_204_NO_CONTENT)
@detail_route(methods=['POST'])
def clone(self, request, *args, **kwargs):
self.object = self.get_object()
data = {'xform': self.object.pk,
'username': request.data.get('username')}
project = request.data.get('project_id')
if project:
data['project'] = project
serializer = CloneXFormSerializer(data=data)
if serializer.is_valid():
clone_to_user = User.objects.get(username=data['username'])
if not request.user.has_perm('can_add_xform',
clone_to_user.profile):
raise exceptions.PermissionDenied(
detail=_(u"User %(user)s has no permission to add "
"xforms to account %(account)s" %
{'user': request.user.username,
'account': data['username']}))
try:
xform = serializer.save()
except IntegrityError:
raise ParseError(
'A clone with the same id_string has already been created')
serializer = XFormSerializer(
xform.cloned_form, context={'request': request})
return Response(data=serializer.data,
status=status.HTTP_201_CREATED)
return Response(data=serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
@detail_route(methods=['POST', 'GET'])
def csv_import(self, request, *args, **kwargs):
""" Endpoint for CSV data imports
Calls :py:func:`onadata.libs.utils.csv_import.submit_csv` for POST
requests passing the `request.FILES.get('csv_file')` upload
for import and
:py:func:onadata.libs.utils.csv_import.get_async_csv_submission_status
for GET requests passing `job_uuid` query param for job progress
polling
"""
self.object = self.get_object()
resp = {}
if request.method == 'GET':
try:
resp.update(get_async_csv_submission_status(
request.query_params.get('job_uuid')))
self.last_modified_date = timezone.now()
except ValueError:
raise ParseError(('The instance of the result is not a '
'basestring; the job_uuid variable might '
'be incorrect'))
else:
csv_file = request.FILES.get('csv_file', None)
if csv_file is None:
resp.update({u'error': u'csv_file field empty'})
else:
num_rows = sum(1 for row in csv_file) - 1
if num_rows < settings.CSV_ROW_IMPORT_ASYNC_THRESHOLD:
resp.update(submit_csv(request.user.username,
self.object, csv_file))
else:
tmp_file_path = utils.generate_tmp_path(csv_file)
task = submit_csv_async.delay(request.user.username,
self.object,
tmp_file_path)
if task is None:
raise ParseError('Task not found')
else:
resp.update({u'task_id': task.task_id})
return Response(
data=resp,
status=status.HTTP_200_OK if resp.get('error') is None else
status.HTTP_400_BAD_REQUEST)
def partial_update(self, request, *args, **kwargs):
self.object = self.get_object()
owner = self.object.user
# updating the file
if request.FILES or set(['xls_url',
'dropbox_xls_url',
'text_xls_form']) & set(request.data.keys()):
return _try_update_xlsform(request, self.object, owner)
return super(XFormViewSet, self).partial_update(request, *args,
**kwargs)
@detail_route(methods=['DELETE', 'GET'])
def delete_async(self, request, *args, **kwargs):
if request.method == 'DELETE':
xform = self.get_object()
resp = {
u'job_uuid': tasks.delete_xform_async.delay(xform).task_id,
u'time_async_triggered': datetime.now()}
resp_code = status.HTTP_202_ACCEPTED
elif request.method == 'GET':
job_uuid = request.query_params.get('job_uuid')
resp = tasks.get_async_status(job_uuid)
resp_code = status.HTTP_202_ACCEPTED
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp, status=resp_code)
def destroy(self, request, *args, **kwargs):
xform = self.get_object()
xform.soft_delete()
return Response(status=status.HTTP_204_NO_CONTENT)
@detail_route(methods=['GET'])
def export_async(self, request, *args, **kwargs):
job_uuid = request.query_params.get('job_uuid')
export_type = request.query_params.get('format')
query = request.query_params.get("query")
xform = self.get_object()
token = request.query_params.get('token')
meta = request.query_params.get('meta')
data_id = request.query_params.get('data_id')
options = parse_request_export_options(request.query_params)
options.update({
'meta': meta,
'token': token,
'data_id': data_id,
'query': query,
})
if job_uuid:
try:
resp = get_async_response(job_uuid, request, xform)
except Export.DoesNotExist:
# if this does not exist retry it against the primary
with use_master:
resp = get_async_response(job_uuid, request, xform)
else:
resp = process_async_export(request, xform, export_type, options)
if isinstance(resp, HttpResponseRedirect):
payload = {
"details": _("Google authorization needed"),
"url": resp.url
}
return Response(data=payload,
status=status.HTTP_403_FORBIDDEN,
content_type="application/json")
self.etag_data = '{}'.format(timezone.now())
return Response(data=resp,
status=status.HTTP_202_ACCEPTED,
content_type="application/json")
def list(self, request, *args, **kwargs):
try:
queryset = self.filter_queryset(self.get_queryset())
last_modified = queryset.values_list('date_modified', flat=True)\
.order_by('-date_modified')
if last_modified:
self.etag_data = last_modified[0]
resp = super(XFormViewSet, self).list(request, *args, **kwargs)
except XLSFormError, e:
resp = HttpResponseBadRequest(e.message)
return resp
def run_multiple_use_cases(id="", userid="", data_times=""):
'''运行多条用例'''
module_prefetch = Prefetch("moduleInfo",
queryset=ModuleInfo.objects.order_by(
"module_order")) # 这里是把我们需要过滤的东西在这里过滤
use_prefetch = Prefetch("usecase_set",
queryset=Usecase.objects.order_by("case_order"))
module_data = ProjectInfo.objects.filter(id=id).prefetch_related(
module_prefetch) # prefetch_related只用在一对多或多对一或多对多
use_data = ModuleInfo.objects.prefetch_related(use_prefetch)
for i in module_data:
print(i.project_name)
module_all = i.moduleInfo.all() # 这里可以使用all()，但不能使用filter()否则会产生多条查询语句
for sub_data in module_all:
for use_data_all in use_data:
if sub_data.module_name == use_data_all.module_name:
print(use_data_all.module_name)
usecase_data_all = use_data_all.usecase_set.all()
for usecase in usecase_data_all:
print(usecase.case_name)
method = usecase.req # 请求方式
print(method)
url1 = usecase.use_case1.host_url
url2 = usecase.case_url # 请求的后面的url
url = url1 + url2 # 用例总url
print(url)
headers = header(
key=eval(usecase.header_key),
value=eval(
reg(json_keyvalue=usecase.header_value)))
if usecase.value_type == "form-data":
datas = data(key=eval(usecase.data_key),
alltype=eval(usecase.data_type),
value=eval(usecase.data_value))
else:
datas = eval(
reg(json_keyvalue=usecase.json_keyvalue))
if method == "GET":
req = req_mode(method=method,
url=url,
params=datas,
headers=headers,
zt=usecase.value_type)
else:
req = req_mode(method=method,
url=url,
data=datas,
headers=headers,
zt=usecase.value_type)
dy_keylist = get_value(req=req,
key=eval(usecase.dy_key))
all_extract_stat = read_req(key=eval(
usecase.all_extract),
req_ode=req)
if all_extract_stat == []: # 如果空就pass跳过
pass
else:
for read in range(0,
len(eval(usecase.all_extract))):
cache.set(
eval(usecase.all_extract)[read],
all_extract_stat[read], 3600)
# result = cache.get(eval(i.all_extract)[0])#这个是取内存内容
asserts = assertion(key=dy_keylist,
alltype=eval(usecase.dy_type),
value=eval(usecase.dy_value))
print(data_times)
operation = operation_information.objects.create(
information=req,
result=asserts,
use_case_mode="many",
dy_value=dy_keylist,
function_time=data_times,
user_id=userid,
use_id=usecase.id)
# operation.save()
usecase_data_all.update(state=asserts)
else:
pass
def waitinglist_widgets(sender, subevent=None, lazy=False, **kwargs):
widgets = []
wles = WaitingListEntry.objects.filter(event=sender,
subevent=subevent,
voucher__isnull=True)
if wles.exists():
if not lazy:
quota_cache = {}
happy = 0
tuples = wles.values(
'item', 'variation').order_by().annotate(cnt=Count('id'))
items = {
i.pk: i
for i in sender.items.filter(
id__in=[t['item'] for t in tuples]).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=sender.quotas.using(
settings.DATABASE_REPLICA).filter(
subevent=subevent)), )
}
vars = {
i.pk: i
for i in ItemVariation.objects.filter(
item__event=sender,
id__in=[t['variation'] for t in tuples
if t['variation']]).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=sender.quotas.using(
settings.DATABASE_REPLICA).filter(
subevent=subevent)), )
}
for wlt in tuples:
item = items.get(wlt['item'])
variation = vars.get(wlt['variation'])
if not item:
continue
quotas = (variation._get_quotas(subevent=subevent) if variation
else item._get_quotas(subevent=subevent))
row = (variation.check_quotas(subevent=subevent,
count_waitinglist=False,
_cache=quota_cache) if variation
else item.check_quotas(subevent=subevent,
count_waitinglist=False,
_cache=quota_cache))
if row[1] is None:
happy += 1
elif row[1] > 0:
happy += 1
for q in quotas:
if q.size is not None:
quota_cache[q.pk] = (quota_cache[q.pk][0],
quota_cache[q.pk][1] - 1)
widgets.append({
'content':
None if lazy else NUM_WIDGET.format(
num=str(happy),
text=_('available to give to people on waiting list')),
'lazy':
'waitinglist-avail',
'priority':
50,
'url':
reverse('control:event.orders.waitinglist',
kwargs={
'event': sender.slug,
'organizer': sender.organizer.slug,
})
})
widgets.append({
'content':
None if lazy else NUM_WIDGET.format(
num=str(wles.count()), text=_('total waiting list length')),
'lazy':
'waitinglist-length',
'display_size':
'small',
'priority':
50,
'url':
reverse('control:event.orders.waitinglist',
kwargs={
'event': sender.slug,
'organizer': sender.organizer.slug,
})
})
return widgets
def prefetch_tags_with_posts_count(self):
return self.prefetch_related(
Prefetch('tags', queryset=Tag.objects.annotate(Count('posts'))))
def get(self, request, slug):
comment_query = Comment.objects.annotate(count_like=Count("users_like")).select_related("author")
comments = Prefetch("comments", comment_query)
book = Book.objects.prefetch_related("authors", comments).get(slug=slug)
return render(request, "book_detail.html", {"book": book, "rate": 2})
def generateFullReport(self, pk, base_url):
self.base_url = base_url
# Our container for 'Flowable' objects
elements = []
toc = TableOfContents()
toc.levelStyles = [
PS(fontName='arialuni',
fontSize=12,
name='TOCHeading1',
leftIndent=20,
firstLineIndent=-20,
spaceBefore=5,
leading=10),
PS(fontName='arialuni',
fontSize=10,
name='TOCHeading2',
leftIndent=40,
firstLineIndent=-20,
spaceBefore=3,
leading=10),
PS(fontName='arialuni',
fontSize=9,
name='TOCHeading3',
leftIndent=40,
firstLineIndent=-20,
spaceBefore=3,
leading=10),
]
elements.append(Paragraph('Responses Report for Site', self.centered))
elements.append(PageBreak())
elements.append(Paragraph('Table of contents', self.centered))
elements.append(toc)
elements.append(PageBreak())
# A large collection of style sheets pre-made for us
styles = getSampleStyleSheet()
styles.add(ParagraphStyle(name='centered', alignment=TA_CENTER))
site = Site.objects.select_related('project').get(pk=pk)
self.project_name = site.project.name
self.project_logo = site.project.logo.url
elements.append(Paragraph(site.name, self.h1))
elements.append(Paragraph(site.identifier, styles['Normal']))
if site.address:
elements.append(Paragraph(site.address, styles['Normal']))
if site.phone:
elements.append(Paragraph(site.phone, styles['Normal']))
if site.region:
elements.append(Paragraph(site.region.name, styles['Normal']))
elements.append(PageBreak())
elements.append(Paragraph('Responses', self.h2))
forms = FieldSightXF.objects.select_related('xf').filter(
is_survey=False, is_deleted=False).filter(
Q(site_id=site.id, from_project=False)
| Q(project_id=site.project_id)).prefetch_related(
Prefetch(
'site_form_instances',
queryset=FInstance.objects.select_related('instance')),
Prefetch(
'project_form_instances',
queryset=FInstance.objects.select_related(
'instance').filter(site_id=site.id))).order_by(
'-is_staged', 'is_scheduled')
if not forms:
elements.append(
Paragraph("No Any Responses Yet.", styles['Heading5']))
#a=FieldSightXF.objects.select_related('xf').filter(site_id=291).prefetch_related(Prefetch('site_form_instances', queryset=FInstance.objects.select_related('instance')))
styNormal = styleSheet['Normal']
styBackground = ParagraphStyle('background',
parent=styNormal,
backColor=colors.white)
for form in forms:
elements.append(Spacer(0, 10))
elements.append(Paragraph(form.xf.title, self.h3))
elements.append(
Paragraph(form.form_type() + " Form", styles['Heading4']))
if form.stage:
if form.stage.stage:
elements.append(
Paragraph("Stage Id: " + str(form.stage.stage.order),
self.paragraphstyle))
elements.append(
Paragraph("Sub Stage Id: " + str(form.stage.order),
self.paragraphstyle))
else:
elements.append(
Paragraph("Stage Id: " + str(form.stage.order),
self.paragraphstyle))
json_question = form.xf.json
form_user_name = form.xf.user.username
self.media_folder = form_user_name
#cursor = get_instaces_for_site_individual_form(form.id)
sub_count = 0
if not form.from_project and form.site_form_instances.all():
for instance in form.site_form_instances.all():
self.instance_id = instance.instance_id
self.append_answers(json_question, instance, sub_count)
elif form.project_form_instances.all():
for instance in form.project_form_instances.all():
self.instance_id = instance.instance_id
self.append_answers(json_question, instance, sub_count)
else:
elements.append(
Paragraph("No Submisions Yet. ", styles['Heading5']))
elements.append(Spacer(0, 10))
self.doc.multiBuild(elements, onLaterPages=self._header_footer)
def __init__(self, *args, **kwargs):
"""
Takes additional keyword arguments:
:param category: The category to choose from
:param event: The event this belongs to
:param subevent: The event the parent cart position belongs to
:param initial: The current set of add-ons
:param quota_cache: A shared dictionary for quota caching
:param item_cache: A shared dictionary for item/category caching
"""
category = kwargs.pop('category')
event = kwargs.pop('event')
subevent = kwargs.pop('subevent')
current_addons = kwargs.pop('initial')
quota_cache = kwargs.pop('quota_cache')
item_cache = kwargs.pop('item_cache')
self.price_included = kwargs.pop('price_included')
super().__init__(*args, **kwargs)
if subevent:
item_price_override = subevent.item_price_overrides
var_price_override = subevent.var_price_overrides
else:
item_price_override = {}
var_price_override = {}
ckey = '{}-{}'.format(subevent.pk if subevent else 0, category.pk)
if ckey not in item_cache:
# Get all items to possibly show
items = category.items.filter(
Q(active=True)
& Q(
Q(available_from__isnull=True)
| Q(available_from__lte=now()))
& Q(
Q(available_until__isnull=True)
| Q(available_until__gte=now()))
& Q(hide_without_voucher=False)
).select_related('tax_rule').prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(subevent=subevent)),
Prefetch('variations',
to_attr='available_variations',
queryset=ItemVariation.objects.filter(
active=True,
quotas__isnull=False).prefetch_related(
Prefetch('quotas',
to_attr='_subevent_quotas',
queryset=event.quotas.filter(
subevent=subevent))).distinct()),
).annotate(quotac=Count('quotas'),
has_variations=Count('variations')).filter(
quotac__gt=0).order_by('category__position',
'category_id', 'position',
'name')
item_cache[ckey] = items
else:
items = item_cache[ckey]
for i in items:
if i.has_variations:
choices = [('', _('no selection'), '')]
for v in i.available_variations:
cached_availability = v.check_quotas(subevent=subevent,
_cache=quota_cache)
if v._subevent_quotas:
choices.append(
(v.pk,
self._label(event,
v,
cached_availability,
override_price=var_price_override.get(
v.pk)), v.description))
field = AddOnVariationField(
choices=choices,
label=i.name,
required=False,
widget=AddOnRadioSelect,
help_text=rich_text(str(i.description)),
initial=current_addons.get(i.pk),
)
if len(choices) > 1:
self.fields['item_%s' % i.pk] = field
else:
if not i._subevent_quotas:
continue
cached_availability = i.check_quotas(subevent=subevent,
_cache=quota_cache)
field = forms.BooleanField(
label=self._label(event,
i,
cached_availability,
override_price=item_price_override.get(
i.pk)),
required=False,
initial=i.pk in current_addons,
help_text=rich_text(str(i.description)),
)
self.fields['item_%s' % i.pk] = field
def _needed_remote_artifacts(self, batch):
"""
Build a list of only :class:`~pulpcore.plugin.models.RemoteArtifact` that need
to be created for the batch.
Args:
batch (list): List of :class:`~pulpcore.plugin.stages.DeclarativeContent`.
Returns:
List: Of :class:`~pulpcore.plugin.models.RemoteArtifact`.
"""
remotes_present = set()
for d_content in batch:
# If the attribute is set in a previous batch on the very first item in this batch, the
# rest of the items in this batch will not get the attribute set during prefetch.
# https://code.djangoproject.com/ticket/32089
if hasattr(d_content.content, "_remote_artifact_saver_cas"):
delattr(d_content.content, "_remote_artifact_saver_cas")
for d_artifact in d_content.d_artifacts:
if d_artifact.remote:
remotes_present.add(d_artifact.remote)
prefetch_related_objects(
[d_c.content for d_c in batch],
Prefetch(
"contentartifact_set",
queryset=ContentArtifact.objects.prefetch_related(
Prefetch(
"remoteartifact_set",
queryset=RemoteArtifact.objects.filter(remote__in=remotes_present),
to_attr="_remote_artifact_saver_ras",
)
),
to_attr="_remote_artifact_saver_cas",
),
)
# Now return the list of RemoteArtifacts that need to be saved.
#
# We can end up with duplicates (diff pks, same sha256) in the sequence below,
# so we store by-sha256 and then return the final values
needed_ras = {} # { str(<sha256>): RemoteArtifact, ... }
for d_content in batch:
for d_artifact in d_content.d_artifacts:
if not d_artifact.remote:
continue
for content_artifact in d_content.content._remote_artifact_saver_cas:
if d_artifact.relative_path == content_artifact.relative_path:
break
else:
msg = _('No declared artifact with relative path "{rp}" for content "{c}"')
raise ValueError(
msg.format(rp=content_artifact.relative_path, c=d_content.content)
)
for remote_artifact in content_artifact._remote_artifact_saver_ras:
if remote_artifact.remote_id == d_artifact.remote.pk:
break
else:
remote_artifact = self._create_remote_artifact(d_artifact, content_artifact)
needed_ras[remote_artifact.sha256] = remote_artifact
return list(needed_ras.values())
def handle_upload_content(slug, code, part, f, user):
"""
Update translations in the database from uploaded file.
:arg str slug: Project slug.
:arg str code: Locale code.
:arg str part: Resource path or Subpage name.
:arg UploadedFile f: UploadedFile instance.
:arg User user: User uploading the file.
"""
# Avoid circular import; someday we should refactor to avoid.
from pontoon.sync import formats
from pontoon.sync.changeset import ChangeSet
from pontoon.sync.vcs.models import VCSProject
from pontoon.base.models import (
ChangedEntityLocale,
Entity,
Locale,
Project,
Resource,
TranslatedResource,
Translation,
)
relative_path = _get_relative_path_from_part(slug, part)
project = get_object_or_404(Project, slug=slug)
locale = get_object_or_404(Locale, code=code)
resource = get_object_or_404(Resource,
project__slug=slug,
path=relative_path)
# Store uploaded file to a temporary file and parse it
extension = os.path.splitext(f.name)[1]
with tempfile.NamedTemporaryFile(
prefix="strings" if extension == ".xml" else "",
suffix=extension,
) as temp:
for chunk in f.chunks():
temp.write(chunk)
temp.flush()
resource_file = formats.parse(temp.name)
# Update database objects from file
changeset = ChangeSet(project, VCSProject(project, locales=[locale]),
timezone.now())
entities_qs = (Entity.objects.filter(
resource__project=project,
resource__path=relative_path,
obsolete=False).prefetch_related(
Prefetch(
"translation_set",
queryset=Translation.objects.filter(locale=locale),
to_attr="db_translations",
)).prefetch_related(
Prefetch(
"translation_set",
queryset=Translation.objects.filter(
locale=locale, approved_date__lte=timezone.now()),
to_attr="db_translations_approved_before_sync",
)))
entities_dict = {entity.key: entity for entity in entities_qs}
for vcs_translation in resource_file.translations:
key = vcs_translation.key
if key in entities_dict:
entity = entities_dict[key]
changeset.update_entity_translations_from_vcs(
entity,
locale.code,
vcs_translation,
user,
entity.db_translations,
entity.db_translations_approved_before_sync,
)
changeset.bulk_create_translations()
changeset.bulk_update_translations()
changeset.bulk_log_actions()
if changeset.changed_translations:
# Update 'active' status of all changed translations and their siblings,
# i.e. translations of the same entity to the same locale.
changed_pks = {t.pk for t in changeset.changed_translations}
(Entity.objects.filter(
translation__pk__in=changed_pks).reset_active_translations(
locale=locale))
# Run checks and create TM entries for translations that pass them
valid_translations = changeset.bulk_check_translations()
changeset.bulk_create_translation_memory_entries(valid_translations)
TranslatedResource.objects.get(resource=resource,
locale=locale).calculate_stats()
# Mark translations as changed
changed_entities = {}
existing = ChangedEntityLocale.objects.values_list("entity",
"locale").distinct()
for t in changeset.changed_translations:
key = (t.entity.pk, t.locale.pk)
# Remove duplicate changes to prevent unique constraint violation
if key not in existing:
changed_entities[key] = ChangedEntityLocale(entity=t.entity,
locale=t.locale)
ChangedEntityLocale.objects.bulk_create(changed_entities.values())
# Update latest translation
if changeset.translations_to_create:
changeset.translations_to_create[-1].update_latest_translation()
def operator_vehicles(request, slug):
operators = Operator.objects.select_related('region')
try:
operator = get_object_or_404(operators, slug=slug)
except Http404:
operator = get_object_or_404(operators, operatorcode__code=slug, operatorcode__source__name='slug')
vehicles = operator.vehicle_set
latest_journeys = Subquery(VehicleJourney.objects.filter(
vehicle=OuterRef('pk')
).order_by('-datetime').values('pk')[:1])
latest_journeys = vehicles.filter(latest_location=None).annotate(latest_journey=latest_journeys)
latest_journeys = VehicleJourney.objects.filter(id__in=latest_journeys.values('latest_journey'))
prefetch = Prefetch('vehiclejourney_set',
queryset=latest_journeys.select_related('service'), to_attr='latest_journeys')
vehicles = vehicles.prefetch_related(prefetch)
vehicles = vehicles.order_by('fleet_number', 'reg', 'code')
vehicles = vehicles.select_related('vehicle_type', 'livery', 'latest_location__journey__service')
edit = request.path.endswith('/edit')
submitted = False
if edit:
form = EditVehiclesForm(request.POST, operator=operator, initial={
'operator': operator
})
if request.POST and form.is_valid():
ticked_vehicles = (vehicle for vehicle in vehicles if str(vehicle.id) in request.POST.getlist('vehicle'))
data = {key: form.cleaned_data[key] for key in form.changed_data}
submitted = len(VehicleEdit.objects.bulk_create(
get_vehicle_edit(vehicle, data) for vehicle in ticked_vehicles
))
if form.cleaned_data.get('operator') and form.cleaned_data['operator'] != operator:
Vehicle.objects.filter(id__in=request.POST.getlist('vehicle')).update(operator=operator)
else:
form = None
pending_edits = VehicleEdit.objects.filter(approved=False, vehicle=OuterRef('id'))
vehicles = vehicles.annotate(pending_edits=Exists(pending_edits))
if not vehicles:
raise Http404()
if operator.name == 'National Express':
for v in vehicles:
parts = v.notes.split()
if parts and parts[-1][-1].isdigit():
v.fleet_number = parts[-1]
if v.fleet_number.isdigit():
v.fleet_number = int(v.fleet_number)
v.notes = ' '.join(parts[:-1])
vehicles = sorted(vehicles, key=lambda v: v.fleet_number if type(v.fleet_number) is int else 0)
vehicles = sorted(vehicles, key=lambda v: v.notes or 'z')
return render(request, 'operator_vehicles.html', {
'breadcrumb': [operator.region, operator],
'object': operator,
'today': timezone.localtime().date(),
'vehicles': vehicles,
'code_column': any(v.fleet_number_mismatch() for v in vehicles),
'branding_column': any(vehicle.branding for vehicle in vehicles),
'name_column': any(vehicle.name for vehicle in vehicles),
'notes_column': any(vehicle.notes for vehicle in vehicles),
'edit_url': reverse('admin:vehicles_vehicle_changelist'),
'edit': edit,
'submitted': submitted,
'form': form,
})
def index_enterprise_catalog_courses_in_algolia_task(
content_keys,
algolia_fields,
uuid_batch_size=ALGOLIA_UUID_BATCH_SIZE,
):
"""
Index course data in Algolia with enterprise-related fields.
Arguments:
content_keys (list): A list of content_keys. It's important that this is the first positional argument,
so that the passing of return values to the signature of the next chained celery task
works as expected.
algolia_fields (list): A list of course fields we want to index in Algolia
uuid_batch_size (int): The threshold of distinct catalog/customer UUIDs associated with a piece of content,
at which duplicate course records are created in the index,
batching the uuids (flattened records) to reduce the payload size of the Algolia objects.
Defaults to ``ALGOLIA_UUID_BATCH_SIZE``.
"""
algolia_client = get_initialized_algolia_client()
if not algolia_fields or not content_keys:
logger.error(
'Must provide algolia_fields and content_keys as arguments.')
return
# Update the index in batches
for content_keys_batch in batch(content_keys, batch_size=TASK_BATCH_SIZE):
courses = []
catalog_uuids_by_course_key = defaultdict(set)
customer_uuids_by_course_key = defaultdict(set)
# retrieve ContentMetadata records that match the specified content_keys in the
# content_key or parent_content_key. returns both courses and course runs.
query = Q(content_key__in=content_keys_batch) | Q(
parent_content_key__in=content_keys_batch)
catalog_queries = CatalogQuery.objects.prefetch_related(
'enterprise_catalogs', )
content_metadata = ContentMetadata.objects.filter(
query).prefetch_related(
Prefetch('catalog_queries', queryset=catalog_queries), )
# iterate through ContentMetadata records, retrieving the enterprise_catalog_uuids
# and enterprise_customer_uuids associated with each ContentMetadata record (either
# a course or a course run), storing them in a dictionary with the related course's
# content_key as a key for later retrieval. the course's content_key is determined by
# the content_key field if the metadata is a `COURSE` or by the parent_content_key
# field if the metadata is a `COURSE_RUN`.
for metadata in content_metadata:
is_course_content_type = metadata.content_type == COURSE
course_content_key = metadata.content_key if is_course_content_type else metadata.parent_content_key
associated_queries = metadata.catalog_queries.all()
enterprise_catalog_uuids = set()
enterprise_customer_uuids = set()
for query in associated_queries:
associated_catalogs = query.enterprise_catalogs.all()
for catalog in associated_catalogs:
enterprise_catalog_uuids.add(str(catalog.uuid))
enterprise_customer_uuids.add(str(catalog.enterprise_uuid))
# add to any existing enterprise catalog uuids or enterprise customer uuids
catalog_uuids_by_course_key[course_content_key].update(
enterprise_catalog_uuids)
customer_uuids_by_course_key[course_content_key].update(
enterprise_customer_uuids)
# iterate through only the courses, retrieving the enterprise-related uuids from the
# dictionary created above. there is at least 2 duplicate course records per course,
# each including the catalog uuids and customer uuids respectively.
#
# if the number of uuids for both catalogs/customers exceeds uuid_batch_size, then
# create duplicate course records, batching the uuids (flattened records) to reduce
# the payload size of the Algolia objects.
course_content_metadata = content_metadata.filter(content_type=COURSE)
for metadata in course_content_metadata:
content_key = metadata.content_key
# add enterprise-related uuids to json_metadata
json_metadata = copy.deepcopy(metadata.json_metadata)
json_metadata.update({
'objectID':
get_algolia_object_id(json_metadata.get('uuid')),
})
# enterprise catalog uuids
catalog_uuids = sorted(
list(catalog_uuids_by_course_key[content_key]))
batched_metadata = _batched_metadata(
json_metadata,
catalog_uuids,
'enterprise_catalog_uuids',
'{}-catalog-uuids-{}',
uuid_batch_size,
)
courses.extend(batched_metadata)
# enterprise customer uuids
customer_uuids = sorted(
list(customer_uuids_by_course_key[content_key]))
batched_metadata = _batched_metadata(
json_metadata,
customer_uuids,
'enterprise_customer_uuids',
'{}-customer-uuids-{}',
uuid_batch_size,
)
courses.extend(batched_metadata)
# extract out only the fields we care about and send to Algolia index
algolia_objects = create_algolia_objects_from_courses(
courses, algolia_fields)
algolia_client.partially_update_index(algolia_objects)
def get_queryset(self):
return Recipe.objects.prefetch_related(
Prefetch('ingredients',
queryset=Ingredient.objects.select_related(
'food').order_by('rank')))
def async_dupe_delete(*args, **kwargs):
try:
system_settings = System_Settings.objects.get()
enabled = system_settings.delete_dupulicates
dupe_max = system_settings.max_dupes
total_duplicate_delete_count_max_per_run = settings.DUPE_DELETE_MAX_PER_RUN
except System_Settings.DoesNotExist:
enabled = False
if enabled and dupe_max is None:
logger.info(
'skipping deletion of excess duplicates: max_dupes not configured')
return
if enabled:
logger.info(
"delete excess duplicates (max_dupes per finding: %s, max deletes per run: %s)",
dupe_max, total_duplicate_delete_count_max_per_run)
deduplicationLogger.info(
"delete excess duplicates (max_dupes per finding: %s, max deletes per run: %s)",
dupe_max, total_duplicate_delete_count_max_per_run)
# limit to 100 to prevent overlapping jobs
results = Finding.objects \
.filter(duplicate=True) \
.order_by() \
.values('duplicate_finding') \
.annotate(num_dupes=Count('id')) \
.filter(num_dupes__gt=dupe_max)[:total_duplicate_delete_count_max_per_run]
originals_with_too_many_duplicates_ids = [
result['duplicate_finding'] for result in results
]
originals_with_too_many_duplicates = Finding.objects.filter(
id__in=originals_with_too_many_duplicates_ids).order_by('id')
# prefetch to make it faster
originals_with_too_many_duplicates = originals_with_too_many_duplicates.prefetch_related(
(Prefetch("original_finding",
queryset=Finding.objects.filter(
duplicate=True).order_by('date'))))
total_deleted_count = 0
for original in originals_with_too_many_duplicates:
duplicate_list = original.original_finding.all()
dupe_count = len(duplicate_list) - dupe_max
for finding in duplicate_list:
deduplicationLogger.debug(
'deleting finding {}:{} ({}))'.format(
finding.id, finding.title, finding.hash_code))
finding.delete()
total_deleted_count += 1
dupe_count -= 1
if dupe_count <= 0:
break
if total_deleted_count >= total_duplicate_delete_count_max_per_run:
break
if total_deleted_count >= total_duplicate_delete_count_max_per_run:
break
logger.info('total number of excess duplicates deleted: %s',
total_deleted_count)
