def create_external_export(username, id_string, export_id, **options):
"""
XLSReport export task.
"""
export = get_object_or_404(Export, id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_external_export(
Export.EXTERNAL_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError, ConnectionError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"External Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def get_async_response(job_uuid, request, xform, count=0):
"""
Returns the status of an async task for the given job_uuid.
"""
def _get_response():
export = get_object_or_404(Export, task_id=job_uuid)
return export_async_export_response(request, export)
try:
job = AsyncResult(job_uuid)
if job.state == 'SUCCESS':
resp = _get_response()
else:
resp = async_status(celery_state_to_status(job.state))
# append task result to the response
if job.result:
result = job.result
if isinstance(result, dict):
resp.update(result)
else:
resp.update({'progress': str(result)})
except (OperationalError, ConnectionError) as e:
report_exception("Connection Error", e, sys.exc_info())
if count > 0:
raise ServiceUnavailable
return get_async_response(job_uuid, request, xform, count + 1)
except BacklogLimitExceeded:
# most likely still processing
resp = async_status(celery_state_to_status('PENDING'))
return resp
def create_osm_export(username, id_string, export_id, **options):
"""
OSM export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_osm_export(
Export.OSM_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"OSM Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_xls_export(username, id_string, export_id, **options):
"""
XLS export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
force_xlsx = options.get("force_xlsx", True)
options["extension"] = 'xlsx' if force_xlsx else 'xls'
try:
export = _get_export_object(export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.XLS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def publish_form(callback):
"""
Calls the callback function to publish a XLSForm and returns appropriate
message depending on exception throw during publishing of a XLSForm.
"""
try:
return callback()
except (PyXFormError, XLSFormError) as e:
return {'type': 'alert-error', 'text': text(e)}
except IntegrityError:
return {
'type': 'alert-error',
'text': _(u'Form with this id or SMS-keyword already exists.'),
}
except ProcessTimedOut:
# catch timeout errors
return {
'type': 'alert-error',
'text': _(u'Form validation timeout, please try again.'),
}
except (MemoryError, OSError, BadStatusLine):
return {
'type':
'alert-error',
'text':
_((u'An error occurred while publishing the form. '
'Please try again.')),
}
except (AttributeError, Exception, ValidationError) as e:
report_exception("Form publishing exception: {}".format(e), text(e),
sys.exc_info())
return {'type': 'alert-error', 'text': text(e)}
def create_external_export(username, id_string, export_id, **options):
"""
XLSReport export task.
"""
try:
export = _get_export_object(export_id)
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_external_export(
Export.EXTERNAL_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError, ConnectionError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"External Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def publish_form(callback):
"""
Calls the callback function to publish a XLSForm and returns appropriate
message depending on exception throw during publishing of a XLSForm.
"""
try:
return callback()
except (PyXFormError, XLSFormError) as e:
return {'type': 'alert-error', 'text': text(e)}
except IntegrityError:
return {
'type': 'alert-error',
'text': _(u'Form with this id or SMS-keyword already exists.'),
}
except ProcessTimedOut:
# catch timeout errors
return {
'type': 'alert-error',
'text': _(u'Form validation timeout, please try again.'),
}
except (MemoryError, OSError):
return {
'type': 'alert-error',
'text': _((u'An error occurred while publishing the form. '
'Please try again.')),
}
except (AttributeError, Exception, ValidationError) as e:
report_exception("Form publishing exception: {}".format(e), text(e),
sys.exc_info())
return {'type': 'alert-error', 'text': text(e)}
def create_xls_export(username, id_string, export_id, **options):
"""
XLS export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
force_xlsx = options.get("force_xlsx", True)
options["extension"] = 'xlsx' if force_xlsx else 'xls'
try:
export = _get_export_object(export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.XLS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def create_zip_export(username, id_string, export_id, **options):
"""
Attachments zip export task.
"""
export = _get_export_object(export_id)
try:
gen_export = generate_attachments_zip_export(
Export.ZIP_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"Zip Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e)
raise
else:
if not settings.TESTING_MODE:
delete_export.apply_async(
(), {'export_id': gen_export.id},
countdown=settings.ZIP_EXPORT_COUNTDOWN)
return gen_export.id
def create_osm_export(username, id_string, export_id, **options):
"""
OSM export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_osm_export(
Export.OSM_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"OSM Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_zip_export(username, id_string, export_id, **options):
"""
Attachments zip export task.
"""
export = _get_export_object(export_id)
try:
gen_export = generate_attachments_zip_export(
Export.ZIP_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"Zip Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e)
raise
else:
if not settings.TESTING_MODE:
delete_export.apply_async(
(), {'export_id': gen_export.id},
countdown=settings.ZIP_EXPORT_COUNTDOWN)
return gen_export.id
def create_csv_export(username, id_string, export_id, **options):
"""
CSV export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.CSV_EXPORT, export.xform,
export_id, options)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_csv_export(username, id_string, export_id, **options):
"""
CSV export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.CSV_EXPORT, export.xform,
export_id, options)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_attachments_zipfile(attachments):
"""Return a zip file with submission attachments."""
# create zip_file
tmp = NamedTemporaryFile()
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z:
for attachment in attachments:
default_storage = get_storage_class()()
filename = attachment.media_file.name
if default_storage.exists(filename):
try:
with default_storage.open(filename) as f:
if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT:
report_exception(
"Create attachment zip exception",
"File is greater than {} bytes".format(
settings.ZIP_REPORT_ATTACHMENT_LIMIT)
)
break
else:
z.writestr(attachment.media_file.name, f.read())
except IOError as e:
report_exception("Create attachment zip exception", e)
break
return tmp
def create_attachments_zipfile(attachments):
"""Return a zip file with submission attachments."""
# create zip_file
tmp = NamedTemporaryFile()
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z:
for attachment in attachments:
default_storage = get_storage_class()()
filename = attachment.media_file.name
if default_storage.exists(filename):
try:
with default_storage.open(filename) as f:
if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT:
report_exception(
"Create attachment zip exception",
"File is greater than {} bytes".format(
settings.ZIP_REPORT_ATTACHMENT_LIMIT))
break
else:
z.writestr(attachment.media_file.name, f.read())
except IOError as e:
report_exception("Create attachment zip exception", e)
break
return tmp
def enketo_url(form_url,
id_string,
instance_xml=None,
instance_id=None,
return_url=None,
**kwargs):
"""
Returns Enketo webform URL.
"""
if not hasattr(settings, 'ENKETO_URL')\
and not hasattr(settings, 'ENKETO_API_SURVEY_PATH')\
and (not hasattr(settings, 'ENKETO_API_TOKEN') or
settings.ENKETO_API_TOKEN == ''):
return False
url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_SURVEY_PATH)
values = {'form_id': id_string, 'server_url': form_url}
if instance_id is not None and instance_xml is not None:
url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_INSTANCE_PATH)
values.update({
'instance': instance_xml,
'instance_id': instance_id,
'return_url': return_url
})
if kwargs:
# Kwargs need to take note of xform variable paths i.e.
# kwargs = {'defaults[/widgets/text_widgets/my_string]': "Hey Mark"}
values.update(kwargs)
response = requests.post(
url,
data=values,
auth=(settings.ENKETO_API_TOKEN, ''),
verify=getattr(settings, 'VERIFY_SSL', True))
if response.status_code in [200, 201]:
try:
data = response.json()
except ValueError:
pass
else:
url = (data.get('edit_url') or data.get('offline_url')
or data.get('url'))
if url:
return url
else:
try:
data = response.json()
except ValueError:
report_exception("HTTP Error {}".format(response.status_code),
response.text, sys.exc_info())
raise EnketoError()
else:
if 'message' in data:
raise EnketoError(data['message'])
raise EnketoError(response.text)
raise EnketoError()
def test_report_exception_with_exc_info(self):
e = Exception("A test exception")
try:
raise e
except Exception as e:
exc_info = sys.exc_info()
try:
report_exception(subject="Test report exception",
info=e,
exc_info=exc_info)
except Exception as e:
raise AssertionError("%s" % e)
def failed_import(rollback_uuids, xform, exception, status_message):
""" Report a failed import.
:param rollback_uuids: The rollback UUIDs
:param xform: The XForm that failed to import to
:param exception: The exception object
:return: The async_status result
"""
Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete()
report_exception(
'CSV Import Failed : %d - %s - %s' %
(xform.pk, xform.id_string, xform.title), exception, sys.exc_info())
return async_status(FAILED, status_message)
def _set_project_perms():
try:
xform = XForm.objects.get(id=xform_id)
project = Project.objects.get(id=project_id)
except (Project.DoesNotExist, XForm.DoesNotExist) as e:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
# make a report only on the 3rd try.
if self.request.retries > 2:
report_exception(msg, e, sys.exc_info())
self.retry(countdown=60 * self.request.retries, exc=e)
else:
set_project_perms_to_xform(xform, project)
def _set_project_perms():
try:
xform = XForm.objects.get(id=xform_id)
project = Project.objects.get(id=project_id)
except (Project.DoesNotExist, XForm.DoesNotExist) as e:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
# make a report only on the 3rd try.
if self.request.retries > 2:
report_exception(msg, e, sys.exc_info())
self.retry(countdown=60 * self.request.retries, exc=e)
else:
set_project_perms_to_xform(xform, project)
def create_attachments_zipfile(attachments):
# create zip_file
tmp = NamedTemporaryFile()
with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z:
for attachment in attachments:
default_storage = get_storage_class()()
if default_storage.exists(attachment.media_file.name):
try:
with default_storage.open(attachment.media_file.name) as f:
z.writestr(attachment.media_file.name, f.read())
except Exception as e:
report_exception("Create attachment zip exception", e)
return tmp
def track(user, event_name, properties=None, context=None, request=None):
"""Record actions with the track() API call to the analytics agents."""
if _segment or appoptics_api:
context = context or {}
context['source'] = settings.HOSTNAME
properties = properties or {}
user_id = get_user_id(user)
if 'value' not in properties:
properties['value'] = 1
if 'submitted_by' in properties:
submitted_by_user = properties.pop('submitted_by')
submitted_by = get_user_id(submitted_by_user)
properties['event_by'] = submitted_by
if submitted_by_user:
context['event_by'] = submitted_by_user.username
else:
context['event_by'] = submitted_by
if 'organization' in properties:
context['organization'] = properties.get('organization')
if 'from' in properties:
context['action_from'] = properties.get('from')
if 'xform_id' in properties:
context['xform_id'] = properties['xform_id']
if request:
context['path'] = request.path
context['url'] = request.build_absolute_uri()
context['ip'] = request.META.get('REMOTE_ADDR', '')
context['userId'] = user.id
if _segment:
segment_analytics.track(user_id, event_name, properties, context)
if appoptics_api:
tags = sanitize_metric_values(context)
try:
appoptics_api.submit_measurement(event_name,
properties['value'],
tags=tags)
except exceptions.BadRequest as e:
report_exception("Bad AppOptics Request", e, sys.exc_info())
def handle_enketo_error(response):
"""Handle enketo error response."""
try:
data = json.loads(response.content)
except (ValueError, JSONDecodeError):
report_exception("HTTP Error {}".format(response.status_code),
response.text, sys.exc_info())
if response.status_code == 502:
raise EnketoError(
u"Sorry, we cannot load your form right now. Please try "
"again later.")
raise EnketoError()
else:
if 'message' in data:
raise EnketoError(data['message'])
raise EnketoError(response.text)
def handle_enketo_error(response):
"""Handle enketo error response."""
try:
data = json.loads(response.content)
except (ValueError, JSONDecodeError):
report_exception("HTTP Error {}".format(response.status_code),
response.text, sys.exc_info())
if response.status_code == 502:
raise EnketoError(
u"Sorry, we cannot load your form right now. Please try "
"again later.")
raise EnketoError()
else:
if 'message' in data:
raise EnketoError(data['message'])
raise EnketoError(response.text)
def set_project_perms_to_xform_async(xform_id, project_id):
try:
xform = XForm.objects.get(id=xform_id)
project = Project.objects.get(id=project_id)
except (Project.DoesNotExist, XForm.DoesNotExist):
pass
else:
try:
if len(getattr(settings, 'SLAVE_DATABASES', [])):
from multidb.pinning import use_master
with use_master:
set_project_perms_to_xform(xform, project)
else:
set_project_perms_to_xform(xform, project)
except Exception as e:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
report_exception(msg, e, sys.exc_info())
def create_sav_zip_export(username, id_string, export_id, **options):
export = _get_export_object(id=export_id)
options["extension"] = Export.ZIP_EXPORT
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, TypeError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def set_project_perms_to_xform_async(self, xform_id, project_id):
"""
Apply project permissions for ``project_id`` to a form ``xform_id`` task.
"""
def _set_project_perms():
try:
xform = XForm.objects.get(id=xform_id)
project = Project.objects.get(id=project_id)
except (Project.DoesNotExist, XForm.DoesNotExist) as e:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
# make a report only on the 3rd try.
if self.request.retries > 2:
report_exception(msg, e, sys.exc_info())
self.retry(countdown=60 * self.request.retries, exc=e)
else:
set_project_perms_to_xform(xform, project)
try:
if getattr(settings, 'SLAVE_DATABASES', []):
from multidb.pinning import use_master
with use_master:
_set_project_perms()
else:
_set_project_perms()
except (Project.DoesNotExist, XForm.DoesNotExist) as e:
# make a report only on the 3rd try.
if self.request.retries > 2:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
report_exception(msg, e, sys.exc_info())
# let's retry if the record may still not be available in read replica.
self.retry(countdown=60 * self.request.retries)
except IntegrityError:
# Nothing to do, fail silently, permissions seems to have been applied
# already.
pass
except Exception as e: # pylint: disable=broad-except
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
report_exception(msg, e, sys.exc_info())
def set_project_perms_to_xform_async(self, xform_id, project_id):
"""
Apply project permissions for ``project_id`` to a form ``xform_id`` task.
"""
def _set_project_perms():
try:
xform = XForm.objects.get(id=xform_id)
project = Project.objects.get(id=project_id)
except (Project.DoesNotExist, XForm.DoesNotExist) as e:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
# make a report only on the 3rd try.
if self.request.retries > 2:
report_exception(msg, e, sys.exc_info())
self.retry(countdown=60 * self.request.retries, exc=e)
else:
set_project_perms_to_xform(xform, project)
try:
if getattr(settings, 'SLAVE_DATABASES', []):
from multidb.pinning import use_master
with use_master:
_set_project_perms()
else:
_set_project_perms()
except (Project.DoesNotExist, XForm.DoesNotExist) as e:
# make a report only on the 3rd try.
if self.request.retries > 2:
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
report_exception(msg, e, sys.exc_info())
# let's retry if the record may still not be available in read replica.
self.retry(countdown=60 * self.request.retries)
except IntegrityError:
# Nothing to do, fail silently, permissions seems to have been applied
# already.
pass
except Exception as e: # pylint: disable=broad-except
msg = '%s: Setting project %d permissions to form %d failed.' % (
type(e), project_id, xform_id)
report_exception(msg, e, sys.exc_info())
def create_google_sheet_export(username, id_string, export_id, **options):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
try:
export = _get_export_object(id=export_id)
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, ConnectionError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"Google Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_sav_zip_export(username, id_string, export_id, **options):
"""
SPSS sav export task.
"""
export = _get_export_object(export_id)
options["extension"] = Export.ZIP_EXPORT
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, TypeError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_async_export(xform, export_type, query, force_xlsx, options=None):
"""
Starts asynchronous export tasks and returns an export object.
Throws Export.ExportTypeError if export_type is not in EXPORT_TYPES.
Throws Export.ExportConnectionError if rabbitmq broker is down.
"""
username = xform.user.username
id_string = xform.id_string
def _create_export(xform, export_type, options):
export_options = {
key: get_boolean_value(value, default=True)
for (key, value) in iteritems(options)
if key in Export.EXPORT_OPTION_FIELDS
}
if query and 'query' not in export_options:
export_options['query'] = query
return Export.objects.create(
xform=xform, export_type=export_type, options=export_options)
export = _create_export(xform, export_type, options)
result = None
export_id = export.id
options.update({
'username': username,
'id_string': id_string,
'export_id': export_id,
'query': query,
'force_xlsx': force_xlsx
})
export_types = {
Export.XLS_EXPORT: create_xls_export,
Export.GOOGLE_SHEETS_EXPORT: create_google_sheet_export,
Export.CSV_EXPORT: create_csv_export,
Export.CSV_ZIP_EXPORT: create_csv_zip_export,
Export.SAV_ZIP_EXPORT: create_sav_zip_export,
Export.ZIP_EXPORT: create_zip_export,
Export.KML_EXPORT: create_kml_export,
Export.OSM_EXPORT: create_osm_export,
Export.EXTERNAL_EXPORT: create_external_export
}
# start async export
if export_type in export_types:
try:
result = export_types[export_type].apply_async((), kwargs=options)
except OperationalError as e:
export.internal_status = Export.FAILED
export.error_message = "Error connecting to broker."
export.save()
report_exception(export.error_message, e, sys.exc_info())
raise Export.ExportConnectionError
else:
raise ExportTypeError
if result:
# when celery is running eager, the export has been generated by the
# time we get here so lets retrieve the export object a fresh before we
# save
if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False):
export = get_object_or_404(Export, id=export.id)
export.task_id = result.task_id
export.save()
return export, result
return None
def submit_csv(username, xform, csv_file):
""" Imports CSV data to an existing form
Takes a csv formatted file or string containing rows of submission/instance
and converts those to xml submissions and finally submits them by calling
:py:func:`onadata.libs.utils.logger_tools.safe_create_instance`
:param str username: the subission user
:param onadata.apps.logger.models.XForm xfrom: The submission's XForm.
:param (str or file): A CSV formatted file with submission rows.
:return: If sucessful, a dict with import summary else dict with error str.
:rtype: Dict
"""
if isinstance(csv_file, unicode):
csv_file = cStringIO.StringIO(csv_file)
elif csv_file is None or not hasattr(csv_file, 'read'):
return async_status(
FAILED,
(u'Invalid param type for `csv_file`. '
'Expected utf-8 encoded file or unicode'
' string got {} instead.'.format(type(csv_file).__name__)))
num_rows = sum(1 for row in csv_file) - 1
csv_file.seek(0)
csv_reader = ucsv.DictReader(csv_file, encoding='utf-8-sig')
csv_header = csv_reader.fieldnames
# check for spaces in headers
if any(' ' in header for header in csv_header):
return async_status(FAILED,
u'CSV file fieldnames should not contain spaces')
# Get the data dictionary
xform_header = xform.get_headers()
missing_col = set(xform_header).difference(csv_header)
addition_col = set(csv_header).difference(xform_header)
# change to list
missing_col = list(missing_col)
addition_col = list(addition_col)
# remove all metadata columns
missing = [col for col in missing_col if not col.startswith("_")]
# remove all meta/instanceid columns
while 'meta/instanceID' in missing:
missing.remove('meta/instanceID')
# remove all metadata inside groups
missing = [col for col in missing if not ("/_" in col)]
# ignore if is multiple select question
for col in csv_header:
# this col is a multiple select question
survey_element = xform.get_survey_element(col)
if survey_element and \
survey_element.get('type') == MULTIPLE_SELECT_TYPE:
# remove from the missing and additional list
missing = [x for x in missing if not x.startswith(col)]
addition_col.remove(col)
# remove headers for repeats that might be missing from csv
missing = [m for m in missing if m.find('[') == -1]
# Include additional repeats
addition_col = [a for a in addition_col if a.find('[') == -1]
if missing:
return async_status(
FAILED, u"Sorry uploaded file does not match the form. "
u"The file is missing the column(s): "
u"{0}.".format(', '.join(missing)))
rollback_uuids = []
submission_time = datetime.utcnow().isoformat()
ona_uuid = {'formhub': {'uuid': xform.uuid}}
error = None
additions = inserts = 0
try:
for row in csv_reader:
# remove the additional columns
for index in addition_col:
del row[index]
# fetch submission uuid before purging row metadata
row_uuid = row.get('_uuid')
submitted_by = row.get('_submitted_by')
submission_date = row.get('_submission_time', submission_time)
location_data = {}
for key in row.keys(): # seems faster than a comprehension
# remove metadata (keys starting with '_')
if key.startswith('_'):
del row[key]
# Collect row location data into separate location_data dict
if key.endswith(
('.latitude', '.longitude', '.altitude', '.precision')):
location_key, location_prop = key.rsplit(u'.', 1)
location_data.setdefault(location_key, {}).update(
{location_prop: row.get(key, '0')})
# remove 'n/a' values
if not key.startswith('_') and row[key] == 'n/a':
del row[key]
# collect all location K-V pairs into single geopoint field(s)
# in location_data dict
for location_key in location_data.keys():
location_data.update({
location_key: (u'%(latitude)s %(longitude)s '
'%(altitude)s %(precision)s') %
defaultdict(lambda: '', location_data.get(location_key))
})
row = csv_dict_to_nested_dict(row)
location_data = csv_dict_to_nested_dict(location_data)
row = dict_merge(row, location_data)
# inject our form's uuid into the submission
row.update(ona_uuid)
old_meta = row.get('meta', {})
new_meta, update = get_submission_meta_dict(xform, row_uuid)
inserts += update
old_meta.update(new_meta)
row.update({'meta': old_meta})
row_uuid = row.get('meta').get('instanceID')
rollback_uuids.append(row_uuid.replace('uuid:', ''))
xml_file = cStringIO.StringIO(
dict2xmlsubmission(row, xform, row_uuid, submission_date))
try:
error, instance = safe_create_instance(username, xml_file, [],
xform.uuid, None)
except ValueError as e:
error = e
if error:
Instance.objects.filter(uuid__in=rollback_uuids,
xform=xform).delete()
return async_status(FAILED, str(error))
else:
additions += 1
if additions % PROGRESS_BATCH_UPDATE == 0:
try:
current_task.update_state(state='PROGRESS',
meta={
'progress': additions,
'total': num_rows,
'info': addition_col
})
except Exception:
pass
finally:
xform.submission_count(True)
users = User.objects.filter(
username=submitted_by) if submitted_by else []
if users:
instance.user = users[0]
instance.save()
except UnicodeDecodeError as e:
Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete()
report_exception(
'CSV Import Failed : %d - %s - %s' %
(xform.pk, xform.id_string, xform.title), e, sys.exc_info())
return async_status(FAILED, u'CSV file must be utf-8 encoded')
except Exception as e:
Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete()
report_exception(
'CSV Import Failed : %d - %s - %s' %
(xform.pk, xform.id_string, xform.title), e, sys.exc_info())
return async_status(FAILED, str(e))
finally:
xform.submission_count(True)
return {
u"additions":
additions - inserts,
u"updates":
inserts,
u"info":
u"Additional column(s) excluded from the upload: '{0}'.".format(
', '.join(list(addition_col)))
}
def generate_export(export_type,
xform,
export_id=None,
options=None,
retries=0):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(user__username__iexact=username,
id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview,
all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform,
query=filter_query,
start=start,
end=end,
count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get("group_delimiter",
DEFAULT_GROUP_DELIMITER)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
export_builder.INCLUDE_LABELS_ONLY = options.get('include_labels_only',
False)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get("repeat_index_tags",
DEFAULT_INDEX_TAGS)
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.set_survey(xform.survey, xform)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(temp_file.name,
records,
username,
id_string,
filter_query,
start=start,
end=end,
dataview=dataview,
xform=xform,
options=options,
columns_with_hxl=columns_with_hxl,
total_records=total_records)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
def create_async_export(xform, export_type, query, force_xlsx, options=None):
"""
Starts asynchronous export tasks and returns an export object.
Throws Export.ExportTypeError if export_type is not in EXPORT_TYPES.
Throws Export.ExportConnectionError if rabbitmq broker is down.
"""
username = xform.user.username
id_string = xform.id_string
def _create_export(xform, export_type, options):
export_options = {
key: get_boolean_value(value, default=True)
for (key, value) in iteritems(options)
if key in Export.EXPORT_OPTION_FIELDS
}
if query and 'query' not in export_options:
export_options['query'] = query
return Export.objects.create(
xform=xform, export_type=export_type, options=export_options)
export = _create_export(xform, export_type, options)
result = None
export_id = export.id
options.update({
'username': username,
'id_string': id_string,
'export_id': export_id,
'query': query,
'force_xlsx': force_xlsx
})
export_types = {
Export.XLS_EXPORT: create_xls_export,
Export.GOOGLE_SHEETS_EXPORT: create_google_sheet_export,
Export.CSV_EXPORT: create_csv_export,
Export.CSV_ZIP_EXPORT: create_csv_zip_export,
Export.SAV_ZIP_EXPORT: create_sav_zip_export,
Export.ZIP_EXPORT: create_zip_export,
Export.KML_EXPORT: create_kml_export,
Export.OSM_EXPORT: create_osm_export,
Export.EXTERNAL_EXPORT: create_external_export
}
# start async export
if export_type in export_types:
try:
result = export_types[export_type].apply_async((), kwargs=options)
except OperationalError as e:
export.internal_status = Export.FAILED
export.error_message = "Error connecting to broker."
export.save()
report_exception(export.error_message, e, sys.exc_info())
raise Export.ExportConnectionError
else:
raise ExportTypeError
if result:
# when celery is running eager, the export has been generated by the
# time we get here so lets retrieve the export object a fresh before we
# save
if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False):
export = get_object_or_404(Export, id=export.id)
export.task_id = result.task_id
export.save()
return export, result
return None
def generate_export(export_type, xform, export_id=None, options=None):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
show_choice_labels: boolean flag
language: language labels as in the XLSForm/XForm
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview, all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform, query=filter_query, start=start,
end=end, count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get(
"group_delimiter", DEFAULT_GROUP_DELIMITER
)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True
)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False
)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
include_reviews = options.get('include_reviews', False)
export_builder.INCLUDE_LABELS_ONLY = options.get(
'include_labels_only', False
)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get(
"repeat_index_tags", DEFAULT_INDEX_TAGS
)
export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels',
False)
export_builder.language = options.get('language')
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.INCLUDE_REVIEWS = include_reviews
export_builder.set_survey(xform.survey, xform,
include_reviews=include_reviews)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(
temp_file.name, records, username, id_string, filter_query,
start=start, end=end, dataview=dataview, xform=xform,
options=options, columns_with_hxl=columns_with_hxl,
total_records=total_records
)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
