def _validate_data_value(self, attrs):
if self.instance and self.instance.pk:
# ToDo , Should we allow PATCHing the object with different file?
attrs['data_file'] = attrs.get('data_file',
self.instance.data_file)
attrs['data_type'] = attrs.get('data_type',
self.instance.data_type)
try:
attrs['data_value']
except KeyError:
attrs['data_value'] = self.instance.data_value
return attrs
data_value = attrs.get('data_value')
if attrs.get(
'data_type') == 'media' and attrs.get('data_file') is None:
message = {'data_value': t('Invalid url {}').format(data_value)}
URLValidator(message=message)(data_value)
if not data_value:
raise serializers.ValidationError(
{'data_value': t('This field is required.')})
return attrs
def delete_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(t('Not shared.'))
export_id = request.POST.get('export_id')
# find the export entry in the db
export = get_object_or_404(Export, id=export_id)
export.delete()
audit = {"xform": xform.id_string, "export_type": export.export_type}
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
t("Deleted %(export_type)s export '%(filename)s'"
" on '%(id_string)s'.") % {
'export_type': export.export_type.upper(),
'filename': export.filename,
'id_string': xform.id_string,
}, audit, request)
return HttpResponseRedirect(
reverse(export_list,
kwargs={
"username": username,
"id_string": id_string,
"export_type": export_type
}))
def publish_xlsform(request, user, existing_xform=None):
"""
If `existing_xform` is specified, that form will be overwritten with the
new XLSForm
"""
if not request.user.has_perm(
'can_add_xform',
UserProfile.objects.get_or_create(user=user)[0]
):
raise exceptions.PermissionDenied(
detail=t("User %(user)s has no permission to add xforms to "
"account %(account)s" % {'user': request.user.username,
'account': user.username}))
if existing_xform and not request.user.has_perm(
'change_xform', existing_xform):
raise exceptions.PermissionDenied(
detail=t("User %(user)s has no permission to change this "
"form." % {'user': request.user.username, })
)
def set_form():
form = QuickConverterForm(request.POST, request.FILES)
if existing_xform:
return form.publish(user, existing_xform.id_string)
else:
return form.publish(user)
return publish_form(set_form)
def retry(self, request, uid=None, *args, **kwargs):
"""
Retries to send data to external service.
:param request: rest_framework.request.Request
:param uid: str
:return: Response
"""
response = {
"detail": "",
"status_code": KOBO_INTERNAL_ERROR_STATUS_CODE
}
status_code = status.HTTP_200_OK
hook_log = self.get_object()
if hook_log.can_retry():
hook_log.change_status()
success = hook_log.retry()
if success:
# Return status_code of remote server too.
# `response["status_code"]` is not the same as `status_code`
response["detail"] = hook_log.message
response["status_code"] = hook_log.status_code
else:
response["detail"] = t(
"An error has occurred when sending the data. Please try again later."
)
status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
else:
response["detail"] = t(
"Data is being or has already been processed")
status_code = status.HTTP_400_BAD_REQUEST
return Response(response, status=status_code)
def _validate_metadata(self,
metadata: dict,
validate_redirect_url: bool = False) -> dict:
if metadata is None:
raise serializers.ValidationError(
{'metadata': t('This field is required')})
if validate_redirect_url:
try:
metadata['redirect_url']
except KeyError:
raise serializers.ValidationError(
{'metadata': t('`redirect_url` is required')})
else:
parsed_url = urlparse(metadata['redirect_url'])
metadata['filename'] = os.path.basename(parsed_url.path)
try:
metadata['filename']
except KeyError:
raise serializers.ValidationError(
{'metadata': t('`filename` is required')})
return metadata
def validate_source(self, source):
asset = self.context['asset']
if self.instance and self.instance.source_uid != source.uid:
raise serializers.ValidationError(
t('Source cannot be changed')
)
# Source data sharing must be enabled before going further
if not source.data_sharing.get('enabled'):
raise serializers.ValidationError(t(
'Data sharing for `{source_uid}` is not enabled'
).format(source_uid=source.uid))
# Validate whether owner of the asset is allowed to link their form
# with the source. Validation is made with owner of the asset instead of
# `request.user`
required_perms = [
PERM_PARTIAL_SUBMISSIONS,
PERM_VIEW_SUBMISSIONS,
]
if not source.has_perms(asset.owner, required_perms, all_=False):
raise serializers.ValidationError(t(
'Pairing data with `{source_uid}` is not allowed'
).format(source_uid=source.uid))
if not self.instance and source.uid in asset.paired_data:
raise serializers.ValidationError(t(
'Source `{source}` is already paired'
).format(source=source.name))
return source
def export_list(request, username, id_string, export_type):
try:
Export.EXPORT_TYPE_DICT[export_type]
except KeyError:
return HttpResponseBadRequest(t('Invalid export type'))
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(t('Not shared.'))
data = {
'username':
owner.username,
'xform':
xform,
'export_type':
export_type,
'export_type_name':
Export.EXPORT_TYPE_DICT[export_type],
'exports':
Export.objects.filter(xform=xform,
export_type=export_type).order_by('-created_on')
}
return render(request, 'export_list.html', data)
def export_download(request, username, id_string, export_type, filename):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(t('Not shared.'))
# find the export entry in the db
export = get_object_or_404(Export, xform=xform, filename=filename)
ext, mime_type = export_def_from_filename(export.filename)
audit = {"xform": xform.id_string, "export_type": export.export_type}
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
t("Downloaded %(export_type)s export '%(filename)s' "
"on '%(id_string)s'.") % {
'export_type': export.export_type.upper(),
'filename': export.filename,
'id_string': xform.id_string,
}, audit, request)
if request.GET.get('raw'):
id_string = None
default_storage = get_storage_class()()
if not isinstance(default_storage, FileSystemStorage):
return HttpResponseRedirect(default_storage.url(export.filepath))
basename = os.path.splitext(export.filename)[0]
response = response_with_mimetype_and_name(mime_type,
name=basename,
extension=ext,
file_path=export.filepath,
show_date=False)
return response
def download_metadata(request, username, id_string, data_id):
xform = get_object_or_404(XForm,
user__username__iexact=username,
id_string__exact=id_string)
owner = xform.user
# FIXME: couldn't non-owner users be allowed to access these files even
# without the form being entirely public (shared=True)?
if username == request.user.username or xform.shared:
data = get_object_or_404(MetaData, pk=data_id)
file_path = data.data_file.name
filename, extension = os.path.splitext(file_path.split('/')[-1])
extension = extension.strip('.')
dfs = get_storage_class()()
if dfs.exists(file_path):
audit = {'xform': xform.id_string}
audit_log(
Actions.FORM_UPDATED, request.user, owner,
t("Document '%(filename)s' for '%(id_string)s' downloaded.") %
{
'id_string': xform.id_string,
'filename': "%s.%s" % (filename, extension)
}, audit, request)
response = response_with_mimetype_and_name(data.data_file_type,
filename,
extension=extension,
show_date=False,
file_path=file_path)
return response
else:
return HttpResponseNotFound()
return HttpResponseForbidden(t('Permission denied.'))
def one_time_login(request):
"""
If the request provides a key that matches a OneTimeAuthenticationKey
object, log in the User specified in that object and redirect to the
location specified in the 'next' parameter
"""
try:
key = request.POST['key']
except KeyError:
return HttpResponseBadRequest(t('No key provided'))
try:
next_ = request.GET['next']
except KeyError:
next_ = None
if not next_ or not is_safe_url(url=next_, host=request.get_host()):
next_ = resolve_url(settings.LOGIN_REDIRECT_URL)
# Clean out all expired keys, just to keep the database tidier
OneTimeAuthenticationKey.objects.filter(
expiry__lt=datetime.datetime.now()).delete()
with transaction.atomic():
try:
otak = OneTimeAuthenticationKey.objects.get(
key=key,
expiry__gte=datetime.datetime.now()
)
except OneTimeAuthenticationKey.DoesNotExist:
return HttpResponseBadRequest(t('Invalid or expired key'))
# Nevermore
otak.delete()
# The request included a valid one-time key. Log in the associated user
user = otak.user
user.backend = settings.AUTHENTICATION_BACKENDS[0]
login(request, user)
return HttpResponseRedirect(next_)
def bulk_validation_status(self, request, *args, **kwargs):
xform = self.get_object()
try:
new_validation_status_uid = request.data['validation_status.uid']
except KeyError:
raise ValidationError({
'payload': t('No `validation_status.uid` provided')
})
# Create new validation_status object
new_validation_status = get_validation_status(
new_validation_status_uid, xform, request.user.username)
postgres_query, mongo_query = self.__build_db_queries(xform,
request.data)
# Update Postgres & Mongo
updated_records_count = Instance.objects.filter(
**postgres_query
).update(validation_status=new_validation_status)
ParsedInstance.bulk_update_validation_statuses(mongo_query,
new_validation_status)
return Response({
'detail': t('{} submissions have been updated').format(
updated_records_count)
}, status.HTTP_200_OK)
class ShowBasketSerializer(DefaultModelSerializer):
ShopSerializer = ModelPresenter(Shop, (
'id',
'url',
'name',
))
ProductInfoSerializer = ModelPresenter(
ProductInfo, ('id', 'url', 'product', 'shop', 'price', 'price_rrc'), {
'product': serializers.StringRelatedField(),
'shop': ShopSerializer()
})
OrderedItemsSerializer = ModelPresenter(
OrderItem, ('id', 'product_info', 'quantity'),
{'product_info': ProductInfoSerializer()})
ordered_items = OrderedItemsSerializer(read_only=True,
many=True,
label=t('Заказанные товары'),
help_text=t('Заказанные товары'))
total_sum = serializers.DecimalField(read_only=True,
max_digits=20,
decimal_places=2,
min_value=0,
label=t('Total'),
help_text=t('Общая сумма'))
class Meta:
model = Order
fields = (
'ordered_items',
'total_sum',
'Errors',
'Status',
)
def kml_export(request, username, id_string):
# read the locations from the database
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__exact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(t('Not shared.'))
data = {'data': kml_export_data(id_string, user=owner)}
response = \
render(request, "survey.kml", data,
content_type="application/vnd.google-earth.kml+xml")
response['Content-Disposition'] = \
disposition_ext_and_date(id_string, 'kml')
audit = {"xform": xform.id_string, "export_type": Export.KML_EXPORT}
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
t("Created KML export on '%(id_string)s'.") % {
'id_string': xform.id_string,
}, audit, request)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
t("Downloaded KML export on '%(id_string)s'.") % {
'id_string': xform.id_string,
}, audit, request)
return response
def _get_response(
self,
request,
submission_id_or_uuid: Union[str, int],
attachment_id: Optional[int] = None,
xpath: Optional[str] = None,
) -> Response:
try:
attachment = self.asset.deployment.get_attachment(
submission_id_or_uuid, request.user, attachment_id, xpath)
except (SubmissionNotFoundException, AttachmentNotFoundException):
raise Http404
except InvalidXPathException:
raise serializers.ValidationError(
{'detail': t('Invalid XPath syntax')}, 'invalid_xpath')
except XPathNotFoundException:
raise serializers.ValidationError(
{'detail': t('The path could not be found in the submission')},
'xpath_not_found')
try:
protected_path = attachment.protected_path(
request.accepted_renderer.format)
except FFMpegException:
raise serializers.ValidationError(
{'detail': t('The error occurred during conversion')},
'ffmpeg_error')
except NotSupportedFormatException:
raise serializers.ValidationError(
{
'detail':
t('Conversion is not supported for {}').format(
attachment.mimetype)
}, 'not_supported_format')
# If unit tests are running, pytest webserver does not support
# `X-Accel-Redirect` header (or ignores it?). We need to pass
# the content to the Response object
if settings.TESTING:
# setting the content type to `None` here allows the renderer to
# specify the content type for the response
content_type = (attachment.mimetype
if request.accepted_renderer.format !=
MP3ConversionRenderer.format else None)
return Response(
attachment.content,
content_type=content_type,
)
# Otherwise, let NGINX determine the correct content type and serve
# the file
headers = {
'Content-Disposition':
f'inline; filename={attachment.media_file_basename}',
'X-Accel-Redirect': protected_path
}
response = Response(content_type='', headers=headers)
return response
class Meta:
ordering = ('-completed', '-created')
get_latest_by = ('created', )
permissions = (
('rerun', t('Can rerun failed tasks.')),
('cancel', t('Can cancel failed tasks.')),
)
default_manager_name = 'objects'
class Meta:
ordering = ("-completed", "-created")
get_latest_by = ("created", )
permissions = (
("rerun", t("Can rerun failed tasks.")),
("cancel", t("Can cancel failed tasks.")),
)
default_manager_name = "objects"
def _validate_filename(self, attrs: dict):
if self.instance and 'filename' not in attrs:
return
asset = self.context['asset']
source = attrs['source']
filename, extension = os.path.splitext(attrs['filename'])
if not re.match(r'^[\w\d-]+$', filename):
raise serializers.ValidationError(
{
'filename': t('Only letters, numbers and `-` are allowed')
}
)
if extension.lower() != '.xml' and extension != '':
raise serializers.ValidationError(
{
'filename': t('Extension must be `xml`')
}
)
# force XML extension
basename = filename
filename = f'{filename}.xml'
# Validate uniqueness of `filename`
# It cannot be used by any other asset files
media_filenames = (
AssetFile.objects.values_list('metadata__filename', flat=True)
.filter(asset_id=asset.pk)
.exclude(file_type=AssetFile.PAIRED_DATA)
)
paired_data_filenames = {}
for p_uid, values in asset.paired_data.items():
paired_data_filenames[p_uid] = values['filename']
pd_filename = paired_data_filenames.get(source.uid)
is_new = pd_filename is None
if (
filename in media_filenames
or (
filename in paired_data_filenames.values()
and (is_new or (not is_new and pd_filename != filename))
)
):
raise serializers.ValidationError(
{
'filename': t(
'`{basename}` is already used'
).format(basename=basename)
}
)
attrs['filename'] = filename
class PartnerUpdateSchema(ResponsesSchema):
"""
Класс документирует partner-update
(добавляет статус коды в openapi документацию)
"""
status_descriptions = {
'201': t('Created'),
'404': t('URL не найден'),
}
class ConfirmUserSerializer(DefaultSerializer):
email = serializers.EmailField(required=True,
allow_blank=False,
label=t('Email'),
help_text=t('Iput email'))
token = serializers.CharField(required=True,
allow_blank=False,
label=t('Auth token'),
help_text=t('Auth token'))
def bulksubmission(request, username):
# puts it in a temp directory.
# runs "import_tools(temp_directory)"
# deletes
posting_user = get_object_or_404(User, username__iexact=username)
# request.FILES is a django.utils.datastructures.MultiValueDict
# for each key we have a list of values
try:
temp_postfile = request.FILES.pop("zip_submission_file", [])
except IOError:
return HttpResponseBadRequest(
t("There was a problem receiving your "
"ODK submission. [Error: IO Error "
"reading data]"))
if len(temp_postfile) != 1:
return HttpResponseBadRequest(
t("There was a problem receiving your"
" ODK submission. [Error: multiple "
"submission files (?)]"))
postfile = temp_postfile[0]
tempdir = tempfile.gettempdir()
our_tfpath = os.path.join(tempdir, postfile.name)
with open(our_tfpath, 'wb') as f:
f.write(postfile.read())
with open(our_tfpath, 'rb') as f:
total_count, success_count, errors = import_instances_from_zip(
f, posting_user)
# chose the try approach as suggested by the link below
# http://stackoverflow.com/questions/82831
try:
os.remove(our_tfpath)
except IOError:
# TODO: log this Exception somewhere
pass
json_msg = {
'message': t("Submission complete. Out of %(total)d "
"survey instances, %(success)d were imported, "
"(%(rejected)d were rejected as duplicates, "
"missing forms, etc.)") % {
'total': total_count,
'success': success_count,
'rejected': total_count - success_count
},
'errors': "%d %s" % (len(errors), errors)
}
audit = {"bulk_submission_log": json_msg}
audit_log(Actions.USER_BULK_SUBMISSION, request.user, posting_user,
t("Made bulk submissions."), audit, request)
response = HttpResponse(json.dumps(json_msg))
response.status_code = 200
response['Location'] = request.build_absolute_uri(request.path)
return response
def check_for_migration_from_shared_database(self):
def db_contains_app_migrations(db_connection, app):
# Does the migrations table exist?
with db_connection.cursor() as cursor:
cursor.execute('''SELECT (1) AS "exists" FROM "pg_tables" '''
'''WHERE "tablename" = 'django_migrations' '''
'''LIMIT 1;''')
if not cursor.fetchone():
return False
# Does the migrations table contain any KPI migration?
with db_connection.cursor() as cursor:
cursor.execute(
'''SELECT (1) AS "exists" FROM "django_migrations" '''
'''WHERE "app" = %s '''
'''LIMIT 1;''', [app])
if not cursor.fetchone():
return False
return True
kpi_connection = connections['default']
kc_connection = connections['kobocat']
kpi_app = 'kpi'
kc_app = 'logger'
if db_contains_app_migrations(kc_connection, kpi_app):
# This was formerly a single-database setup, since the KC database
# contains KPI migrations
if not db_contains_app_migrations(kpi_connection, kc_app):
# When migrating from the single, shared database setup:
# 1. A new database for KPI should have been created;
# 2. Certain tables, including `django_migrations`, should
# have been copied from the original, shared database to
# the new one;
# 3. KoBoCAT should have been configured to continue using
# the original database;
# 4. KPI should have been configured to use the newly-created
# database;
# 5. Because `django_migrations` should have been copied from
# the original database to the new one, we should see
# KoBoCAT migrations in the KPI database.
# Since we _do not_ see any KoBoCAT migrations, stop now and
# get the human to fix their installation.
self.errors.append(
Error(
t('Incomplete migration from shared-database installation'
),
hint=t(
'The KoBoCAT database was originally shared by '
'KPI, but the KPI tables were not copied from that '
'shared database to the new, KPI-only database.') +
self.guidance_message,
obj=self,
id='KPI.E023',
))
return False
return True
class DefaultSerializer(serializers.Serializer):
"""
Базовый класс сериалайзера с полями для документирования ошибок и статуса
"""
Errors = serializers.CharField(read_only=True,
help_text=t('Error message(s)'),
label=t('Error'))
Status = serializers.BooleanField(read_only=True,
help_text=t('Http Status code'),
label=t('Status code'))
def validate_endpoint(self, value):
"""
Check if endpoint is valid
"""
if not value.startswith('http'):
raise serializers.ValidationError(t('Invalid scheme'))
elif not constance.config.ALLOW_UNSECURED_HOOK_ENDPOINTS and \
value.startswith('http:'):
raise serializers.ValidationError(
t('Unsecured endpoint is not allowed'))
return value
def report_error(user):
send_multi_alternative.delay(
# title:
t('Не удалось обновить прайс'),
# message:
t('Произошла ошибка при обновлении прайса'),
# from:
settings.EMAIL_HOST_USER,
# to:
[user.email]
)
def report_success(user):
send_multi_alternative.delay(
# title:
t('Обновление прайс листов завершено'),
# message:
t('Все возможные прайс листы обновлены'),
# from:
settings.EMAIL_HOST_USER,
# to:
[user.email]
)
def validate_xform(self, xform):
request = self.context.get('request')
if not request.user.has_perm(CAN_VIEW_XFORM, xform):
raise serializers.ValidationError(t('Project not found'))
if not request.user.has_perm(CAN_CHANGE_XFORM, xform):
raise serializers.ValidationError(
t('You do not have sufficient permissions to perform this action'
))
return xform
def validate_fields(self, data: dict) -> list:
fields = data[EXPORT_SETTING_FIELDS]
if not isinstance(fields, list):
raise serializers.ValidationError(
{EXPORT_SETTING_FIELDS: t('Must be an array')})
if not all((isinstance(field, str) for field in fields)):
raise serializers.ValidationError({
EXPORT_SETTING_FIELDS:
t('All values in the array must be strings')
})
return fields
def clean(self, value):
try:
instance = json.loads(value)
jsonschema.validate(instance, self.schema)
except json.JSONDecodeError as e:
# Message written to match `IntegerField`, which uses the
# imperative: "Enter a whole number."
raise ValidationError(t('Enter valid JSON.') + ' ' + str(e))
except jsonschema.exceptions.ValidationError as e:
# `str(e)` is too verbose (it includes the entire schema)
raise ValidationError(t('Enter valid JSON.') + ' ' + e.message)
return value
def validate_submission_ids(self, data: dict) -> list:
submission_ids = data[EXPORT_SETTING_SUBMISSION_IDS]
if not isinstance(submission_ids, list):
raise serializers.ValidationError(
{EXPORT_SETTING_SUBMISSION_IDS: t('Must be an array')})
if (submission_ids
and not all(isinstance(_id, int) for _id in submission_ids)):
raise serializers.ValidationError({
EXPORT_SETTING_SUBMISSION_IDS:
t('All values in the array must be integers')
})
return submission_ids
def download_xlsform(request, username, id_string):
xform = get_object_or_404(XForm,
user__username__iexact=username,
id_string__exact=id_string)
owner = User.objects.get(username__iexact=username)
helper_auth_helper(request)
if not has_permission(xform, owner, request, xform.shared):
return HttpResponseForbidden('Not shared.')
file_path = xform.xls.name
default_storage = get_storage_class()()
if file_path != '' and default_storage.exists(file_path):
audit = {"xform": xform.id_string}
audit_log(
Actions.FORM_XLS_DOWNLOADED, request.user, xform.user,
t("Downloaded XLS file for form '%(id_string)s'.") %
{"id_string": xform.id_string}, audit, request)
if file_path.endswith('.csv'):
with default_storage.open(file_path) as ff:
xls_io = convert_csv_to_xls(ff.read())
response = StreamingHttpResponse(
xls_io,
content_type='application/vnd.ms-excel; charset=utf-8')
response[
'Content-Disposition'] = 'attachment; filename=%s.xls' % xform.id_string
return response
split_path = file_path.split(os.extsep)
extension = 'xls'
if len(split_path) > 1:
extension = split_path[len(split_path) - 1]
response = response_with_mimetype_and_name('vnd.ms-excel',
id_string,
show_date=False,
extension=extension,
file_path=file_path)
return response
else:
messages.add_message(
request, messages.WARNING,
t('No XLS file for your form '
'<strong>%(id)s</strong>') % {'id': id_string})
return HttpResponseRedirect("/%s" % username)
def send_email_for_message(message):
send_mail(t('You are Hero - new Message - %s') % message.title,
message.text,
'*****@*****.**',
[message.recipient.email],
fail_silently=True
)
