def create_external_export(username, id_string, export_id, **options): """ XLSReport export task. """ export = get_object_or_404(Export, id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_external_export( Export.EXTERNAL_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "External Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def get_async_response(job_uuid, request, xform, count=0): """ Returns the status of an async task for the given job_uuid. """ def _get_response(): export = get_object_or_404(Export, task_id=job_uuid) return export_async_export_response(request, export) try: job = AsyncResult(job_uuid) if job.state == 'SUCCESS': resp = _get_response() else: resp = async_status(celery_state_to_status(job.state)) # append task result to the response if job.result: result = job.result if isinstance(result, dict): resp.update(result) else: resp.update({'progress': str(result)}) except (OperationalError, ConnectionError) as e: report_exception("Connection Error", e, sys.exc_info()) if count > 0: raise ServiceUnavailable return get_async_response(job_uuid, request, xform, count + 1) except BacklogLimitExceeded: # most likely still processing resp = async_status(celery_state_to_status('PENDING')) return resp
def create_osm_export(username, id_string, export_id, **options): """ OSM export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_osm_export( Export.OSM_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "OSM Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_xls_export(username, id_string, export_id, **options): """ XLS export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) options["extension"] = 'xlsx' if force_xlsx else 'xls' try: export = _get_export_object(export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.XLS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "XLS Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def publish_form(callback): """ Calls the callback function to publish a XLSForm and returns appropriate message depending on exception throw during publishing of a XLSForm. """ try: return callback() except (PyXFormError, XLSFormError) as e: return {'type': 'alert-error', 'text': text(e)} except IntegrityError: return { 'type': 'alert-error', 'text': _(u'Form with this id or SMS-keyword already exists.'), } except ProcessTimedOut: # catch timeout errors return { 'type': 'alert-error', 'text': _(u'Form validation timeout, please try again.'), } except (MemoryError, OSError, BadStatusLine): return { 'type': 'alert-error', 'text': _((u'An error occurred while publishing the form. ' 'Please try again.')), } except (AttributeError, Exception, ValidationError) as e: report_exception("Form publishing exception: {}".format(e), text(e), sys.exc_info()) return {'type': 'alert-error', 'text': text(e)}
def create_external_export(username, id_string, export_id, **options): """ XLSReport export task. """ try: export = _get_export_object(export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_external_export( Export.EXTERNAL_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "External Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def publish_form(callback): """ Calls the callback function to publish a XLSForm and returns appropriate message depending on exception throw during publishing of a XLSForm. """ try: return callback() except (PyXFormError, XLSFormError) as e: return {'type': 'alert-error', 'text': text(e)} except IntegrityError: return { 'type': 'alert-error', 'text': _(u'Form with this id or SMS-keyword already exists.'), } except ProcessTimedOut: # catch timeout errors return { 'type': 'alert-error', 'text': _(u'Form validation timeout, please try again.'), } except (MemoryError, OSError): return { 'type': 'alert-error', 'text': _((u'An error occurred while publishing the form. ' 'Please try again.')), } except (AttributeError, Exception, ValidationError) as e: report_exception("Form publishing exception: {}".format(e), text(e), sys.exc_info()) return {'type': 'alert-error', 'text': text(e)}
def create_xls_export(username, id_string, export_id, **options): """ XLS export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) options["extension"] = 'xlsx' if force_xlsx else 'xls' try: export = _get_export_object(export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.XLS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "XLS Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def create_zip_export(username, id_string, export_id, **options): """ Attachments zip export task. """ export = _get_export_object(export_id) try: gen_export = generate_attachments_zip_export( Export.ZIP_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e) raise else: if not settings.TESTING_MODE: delete_export.apply_async( (), {'export_id': gen_export.id}, countdown=settings.ZIP_EXPORT_COUNTDOWN) return gen_export.id
def create_osm_export(username, id_string, export_id, **options): """ OSM export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_osm_export( Export.OSM_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "OSM Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_zip_export(username, id_string, export_id, **options): """ Attachments zip export task. """ export = _get_export_object(export_id) try: gen_export = generate_attachments_zip_export( Export.ZIP_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e) raise else: if not settings.TESTING_MODE: delete_export.apply_async( (), {'export_id': gen_export.id}, countdown=settings.ZIP_EXPORT_COUNTDOWN) return gen_export.id
def create_csv_export(username, id_string, export_id, **options): """ CSV export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.CSV_EXPORT, export.xform, export_id, options) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records export.internal_status = Export.FAILED export.save() except Exception as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "CSV Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_csv_export(username, id_string, export_id, **options): """ CSV export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.CSV_EXPORT, export.xform, export_id, options) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records export.internal_status = Export.FAILED export.save() except Exception as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "CSV Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_attachments_zipfile(attachments): """Return a zip file with submission attachments.""" # create zip_file tmp = NamedTemporaryFile() with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() filename = attachment.media_file.name if default_storage.exists(filename): try: with default_storage.open(filename) as f: if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: report_exception( "Create attachment zip exception", "File is greater than {} bytes".format( settings.ZIP_REPORT_ATTACHMENT_LIMIT) ) break else: z.writestr(attachment.media_file.name, f.read()) except IOError as e: report_exception("Create attachment zip exception", e) break return tmp
def create_attachments_zipfile(attachments): """Return a zip file with submission attachments.""" # create zip_file tmp = NamedTemporaryFile() with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() filename = attachment.media_file.name if default_storage.exists(filename): try: with default_storage.open(filename) as f: if f.size > settings.ZIP_REPORT_ATTACHMENT_LIMIT: report_exception( "Create attachment zip exception", "File is greater than {} bytes".format( settings.ZIP_REPORT_ATTACHMENT_LIMIT)) break else: z.writestr(attachment.media_file.name, f.read()) except IOError as e: report_exception("Create attachment zip exception", e) break return tmp
def enketo_url(form_url, id_string, instance_xml=None, instance_id=None, return_url=None, **kwargs): """ Returns Enketo webform URL. """ if not hasattr(settings, 'ENKETO_URL')\ and not hasattr(settings, 'ENKETO_API_SURVEY_PATH')\ and (not hasattr(settings, 'ENKETO_API_TOKEN') or settings.ENKETO_API_TOKEN == ''): return False url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_SURVEY_PATH) values = {'form_id': id_string, 'server_url': form_url} if instance_id is not None and instance_xml is not None: url = urljoin(settings.ENKETO_URL, settings.ENKETO_API_INSTANCE_PATH) values.update({ 'instance': instance_xml, 'instance_id': instance_id, 'return_url': return_url }) if kwargs: # Kwargs need to take note of xform variable paths i.e. # kwargs = {'defaults[/widgets/text_widgets/my_string]': "Hey Mark"} values.update(kwargs) response = requests.post( url, data=values, auth=(settings.ENKETO_API_TOKEN, ''), verify=getattr(settings, 'VERIFY_SSL', True)) if response.status_code in [200, 201]: try: data = response.json() except ValueError: pass else: url = (data.get('edit_url') or data.get('offline_url') or data.get('url')) if url: return url else: try: data = response.json() except ValueError: report_exception("HTTP Error {}".format(response.status_code), response.text, sys.exc_info()) raise EnketoError() else: if 'message' in data: raise EnketoError(data['message']) raise EnketoError(response.text) raise EnketoError()
def test_report_exception_with_exc_info(self): e = Exception("A test exception") try: raise e except Exception as e: exc_info = sys.exc_info() try: report_exception(subject="Test report exception", info=e, exc_info=exc_info) except Exception as e: raise AssertionError("%s" % e)
def failed_import(rollback_uuids, xform, exception, status_message): """ Report a failed import. :param rollback_uuids: The rollback UUIDs :param xform: The XForm that failed to import to :param exception: The exception object :return: The async_status result """ Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete() report_exception( 'CSV Import Failed : %d - %s - %s' % (xform.pk, xform.id_string, xform.title), exception, sys.exc_info()) return async_status(FAILED, status_message)
def _set_project_perms(): try: xform = XForm.objects.get(id=xform_id) project = Project.objects.get(id=project_id) except (Project.DoesNotExist, XForm.DoesNotExist) as e: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) # make a report only on the 3rd try. if self.request.retries > 2: report_exception(msg, e, sys.exc_info()) self.retry(countdown=60 * self.request.retries, exc=e) else: set_project_perms_to_xform(xform, project)
def _set_project_perms(): try: xform = XForm.objects.get(id=xform_id) project = Project.objects.get(id=project_id) except (Project.DoesNotExist, XForm.DoesNotExist) as e: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) # make a report only on the 3rd try. if self.request.retries > 2: report_exception(msg, e, sys.exc_info()) self.retry(countdown=60 * self.request.retries, exc=e) else: set_project_perms_to_xform(xform, project)
def create_attachments_zipfile(attachments): # create zip_file tmp = NamedTemporaryFile() with zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: for attachment in attachments: default_storage = get_storage_class()() if default_storage.exists(attachment.media_file.name): try: with default_storage.open(attachment.media_file.name) as f: z.writestr(attachment.media_file.name, f.read()) except Exception as e: report_exception("Create attachment zip exception", e) return tmp
def track(user, event_name, properties=None, context=None, request=None): """Record actions with the track() API call to the analytics agents.""" if _segment or appoptics_api: context = context or {} context['source'] = settings.HOSTNAME properties = properties or {} user_id = get_user_id(user) if 'value' not in properties: properties['value'] = 1 if 'submitted_by' in properties: submitted_by_user = properties.pop('submitted_by') submitted_by = get_user_id(submitted_by_user) properties['event_by'] = submitted_by if submitted_by_user: context['event_by'] = submitted_by_user.username else: context['event_by'] = submitted_by if 'organization' in properties: context['organization'] = properties.get('organization') if 'from' in properties: context['action_from'] = properties.get('from') if 'xform_id' in properties: context['xform_id'] = properties['xform_id'] if request: context['path'] = request.path context['url'] = request.build_absolute_uri() context['ip'] = request.META.get('REMOTE_ADDR', '') context['userId'] = user.id if _segment: segment_analytics.track(user_id, event_name, properties, context) if appoptics_api: tags = sanitize_metric_values(context) try: appoptics_api.submit_measurement(event_name, properties['value'], tags=tags) except exceptions.BadRequest as e: report_exception("Bad AppOptics Request", e, sys.exc_info())
def handle_enketo_error(response): """Handle enketo error response.""" try: data = json.loads(response.content) except (ValueError, JSONDecodeError): report_exception("HTTP Error {}".format(response.status_code), response.text, sys.exc_info()) if response.status_code == 502: raise EnketoError( u"Sorry, we cannot load your form right now. Please try " "again later.") raise EnketoError() else: if 'message' in data: raise EnketoError(data['message']) raise EnketoError(response.text)
def handle_enketo_error(response): """Handle enketo error response.""" try: data = json.loads(response.content) except (ValueError, JSONDecodeError): report_exception("HTTP Error {}".format(response.status_code), response.text, sys.exc_info()) if response.status_code == 502: raise EnketoError( u"Sorry, we cannot load your form right now. Please try " "again later.") raise EnketoError() else: if 'message' in data: raise EnketoError(data['message']) raise EnketoError(response.text)
def set_project_perms_to_xform_async(xform_id, project_id): try: xform = XForm.objects.get(id=xform_id) project = Project.objects.get(id=project_id) except (Project.DoesNotExist, XForm.DoesNotExist): pass else: try: if len(getattr(settings, 'SLAVE_DATABASES', [])): from multidb.pinning import use_master with use_master: set_project_perms_to_xform(xform, project) else: set_project_perms_to_xform(xform, project) except Exception as e: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) report_exception(msg, e, sys.exc_info())
def create_sav_zip_export(username, id_string, export_id, **options): export = _get_export_object(id=export_id) options["extension"] = Export.ZIP_EXPORT try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "SAV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def set_project_perms_to_xform_async(self, xform_id, project_id): """ Apply project permissions for ``project_id`` to a form ``xform_id`` task. """ def _set_project_perms(): try: xform = XForm.objects.get(id=xform_id) project = Project.objects.get(id=project_id) except (Project.DoesNotExist, XForm.DoesNotExist) as e: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) # make a report only on the 3rd try. if self.request.retries > 2: report_exception(msg, e, sys.exc_info()) self.retry(countdown=60 * self.request.retries, exc=e) else: set_project_perms_to_xform(xform, project) try: if getattr(settings, 'SLAVE_DATABASES', []): from multidb.pinning import use_master with use_master: _set_project_perms() else: _set_project_perms() except (Project.DoesNotExist, XForm.DoesNotExist) as e: # make a report only on the 3rd try. if self.request.retries > 2: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) report_exception(msg, e, sys.exc_info()) # let's retry if the record may still not be available in read replica. self.retry(countdown=60 * self.request.retries) except IntegrityError: # Nothing to do, fail silently, permissions seems to have been applied # already. pass except Exception as e: # pylint: disable=broad-except msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) report_exception(msg, e, sys.exc_info())
def set_project_perms_to_xform_async(self, xform_id, project_id): """ Apply project permissions for ``project_id`` to a form ``xform_id`` task. """ def _set_project_perms(): try: xform = XForm.objects.get(id=xform_id) project = Project.objects.get(id=project_id) except (Project.DoesNotExist, XForm.DoesNotExist) as e: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) # make a report only on the 3rd try. if self.request.retries > 2: report_exception(msg, e, sys.exc_info()) self.retry(countdown=60 * self.request.retries, exc=e) else: set_project_perms_to_xform(xform, project) try: if getattr(settings, 'SLAVE_DATABASES', []): from multidb.pinning import use_master with use_master: _set_project_perms() else: _set_project_perms() except (Project.DoesNotExist, XForm.DoesNotExist) as e: # make a report only on the 3rd try. if self.request.retries > 2: msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) report_exception(msg, e, sys.exc_info()) # let's retry if the record may still not be available in read replica. self.retry(countdown=60 * self.request.retries) except IntegrityError: # Nothing to do, fail silently, permissions seems to have been applied # already. pass except Exception as e: # pylint: disable=broad-except msg = '%s: Setting project %d permissions to form %d failed.' % ( type(e), project_id, xform_id) report_exception(msg, e, sys.exc_info())
def create_google_sheet_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state try: export = _get_export_object(id=export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "Google Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_sav_zip_export(username, id_string, export_id, **options): """ SPSS sav export task. """ export = _get_export_object(export_id) options["extension"] = Export.ZIP_EXPORT try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "SAV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_async_export(xform, export_type, query, force_xlsx, options=None): """ Starts asynchronous export tasks and returns an export object. Throws Export.ExportTypeError if export_type is not in EXPORT_TYPES. Throws Export.ExportConnectionError if rabbitmq broker is down. """ username = xform.user.username id_string = xform.id_string def _create_export(xform, export_type, options): export_options = { key: get_boolean_value(value, default=True) for (key, value) in iteritems(options) if key in Export.EXPORT_OPTION_FIELDS } if query and 'query' not in export_options: export_options['query'] = query return Export.objects.create( xform=xform, export_type=export_type, options=export_options) export = _create_export(xform, export_type, options) result = None export_id = export.id options.update({ 'username': username, 'id_string': id_string, 'export_id': export_id, 'query': query, 'force_xlsx': force_xlsx }) export_types = { Export.XLS_EXPORT: create_xls_export, Export.GOOGLE_SHEETS_EXPORT: create_google_sheet_export, Export.CSV_EXPORT: create_csv_export, Export.CSV_ZIP_EXPORT: create_csv_zip_export, Export.SAV_ZIP_EXPORT: create_sav_zip_export, Export.ZIP_EXPORT: create_zip_export, Export.KML_EXPORT: create_kml_export, Export.OSM_EXPORT: create_osm_export, Export.EXTERNAL_EXPORT: create_external_export } # start async export if export_type in export_types: try: result = export_types[export_type].apply_async((), kwargs=options) except OperationalError as e: export.internal_status = Export.FAILED export.error_message = "Error connecting to broker." export.save() report_exception(export.error_message, e, sys.exc_info()) raise Export.ExportConnectionError else: raise ExportTypeError if result: # when celery is running eager, the export has been generated by the # time we get here so lets retrieve the export object a fresh before we # save if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False): export = get_object_or_404(Export, id=export.id) export.task_id = result.task_id export.save() return export, result return None
def submit_csv(username, xform, csv_file): """ Imports CSV data to an existing form Takes a csv formatted file or string containing rows of submission/instance and converts those to xml submissions and finally submits them by calling :py:func:`onadata.libs.utils.logger_tools.safe_create_instance` :param str username: the subission user :param onadata.apps.logger.models.XForm xfrom: The submission's XForm. :param (str or file): A CSV formatted file with submission rows. :return: If sucessful, a dict with import summary else dict with error str. :rtype: Dict """ if isinstance(csv_file, unicode): csv_file = cStringIO.StringIO(csv_file) elif csv_file is None or not hasattr(csv_file, 'read'): return async_status( FAILED, (u'Invalid param type for `csv_file`. ' 'Expected utf-8 encoded file or unicode' ' string got {} instead.'.format(type(csv_file).__name__))) num_rows = sum(1 for row in csv_file) - 1 csv_file.seek(0) csv_reader = ucsv.DictReader(csv_file, encoding='utf-8-sig') csv_header = csv_reader.fieldnames # check for spaces in headers if any(' ' in header for header in csv_header): return async_status(FAILED, u'CSV file fieldnames should not contain spaces') # Get the data dictionary xform_header = xform.get_headers() missing_col = set(xform_header).difference(csv_header) addition_col = set(csv_header).difference(xform_header) # change to list missing_col = list(missing_col) addition_col = list(addition_col) # remove all metadata columns missing = [col for col in missing_col if not col.startswith("_")] # remove all meta/instanceid columns while 'meta/instanceID' in missing: missing.remove('meta/instanceID') # remove all metadata inside groups missing = [col for col in missing if not ("/_" in col)] # ignore if is multiple select question for col in csv_header: # this col is a multiple select question survey_element = xform.get_survey_element(col) if survey_element and \ survey_element.get('type') == MULTIPLE_SELECT_TYPE: # remove from the missing and additional list missing = [x for x in missing if not x.startswith(col)] addition_col.remove(col) # remove headers for repeats that might be missing from csv missing = [m for m in missing if m.find('[') == -1] # Include additional repeats addition_col = [a for a in addition_col if a.find('[') == -1] if missing: return async_status( FAILED, u"Sorry uploaded file does not match the form. " u"The file is missing the column(s): " u"{0}.".format(', '.join(missing))) rollback_uuids = [] submission_time = datetime.utcnow().isoformat() ona_uuid = {'formhub': {'uuid': xform.uuid}} error = None additions = inserts = 0 try: for row in csv_reader: # remove the additional columns for index in addition_col: del row[index] # fetch submission uuid before purging row metadata row_uuid = row.get('_uuid') submitted_by = row.get('_submitted_by') submission_date = row.get('_submission_time', submission_time) location_data = {} for key in row.keys(): # seems faster than a comprehension # remove metadata (keys starting with '_') if key.startswith('_'): del row[key] # Collect row location data into separate location_data dict if key.endswith( ('.latitude', '.longitude', '.altitude', '.precision')): location_key, location_prop = key.rsplit(u'.', 1) location_data.setdefault(location_key, {}).update( {location_prop: row.get(key, '0')}) # remove 'n/a' values if not key.startswith('_') and row[key] == 'n/a': del row[key] # collect all location K-V pairs into single geopoint field(s) # in location_data dict for location_key in location_data.keys(): location_data.update({ location_key: (u'%(latitude)s %(longitude)s ' '%(altitude)s %(precision)s') % defaultdict(lambda: '', location_data.get(location_key)) }) row = csv_dict_to_nested_dict(row) location_data = csv_dict_to_nested_dict(location_data) row = dict_merge(row, location_data) # inject our form's uuid into the submission row.update(ona_uuid) old_meta = row.get('meta', {}) new_meta, update = get_submission_meta_dict(xform, row_uuid) inserts += update old_meta.update(new_meta) row.update({'meta': old_meta}) row_uuid = row.get('meta').get('instanceID') rollback_uuids.append(row_uuid.replace('uuid:', '')) xml_file = cStringIO.StringIO( dict2xmlsubmission(row, xform, row_uuid, submission_date)) try: error, instance = safe_create_instance(username, xml_file, [], xform.uuid, None) except ValueError as e: error = e if error: Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete() return async_status(FAILED, str(error)) else: additions += 1 if additions % PROGRESS_BATCH_UPDATE == 0: try: current_task.update_state(state='PROGRESS', meta={ 'progress': additions, 'total': num_rows, 'info': addition_col }) except Exception: pass finally: xform.submission_count(True) users = User.objects.filter( username=submitted_by) if submitted_by else [] if users: instance.user = users[0] instance.save() except UnicodeDecodeError as e: Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete() report_exception( 'CSV Import Failed : %d - %s - %s' % (xform.pk, xform.id_string, xform.title), e, sys.exc_info()) return async_status(FAILED, u'CSV file must be utf-8 encoded') except Exception as e: Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete() report_exception( 'CSV Import Failed : %d - %s - %s' % (xform.pk, xform.id_string, xform.title), e, sys.exc_info()) return async_status(FAILED, str(e)) finally: xform.submission_count(True) return { u"additions": additions - inserts, u"updates": inserts, u"info": u"Additional column(s) excluded from the upload: '{0}'.".format( ', '.join(list(addition_col))) }
def generate_export(export_type, xform, export_id=None, options=None, retries=0): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get("group_delimiter", DEFAULT_GROUP_DELIMITER) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False) export_builder.INCLUDE_LABELS = options.get('include_labels', False) export_builder.INCLUDE_LABELS_ONLY = options.get('include_labels_only', False) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get("repeat_index_tags", DEFAULT_INDEX_TAGS) # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.set_survey(xform.survey, xform) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__(temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export
def create_async_export(xform, export_type, query, force_xlsx, options=None): """ Starts asynchronous export tasks and returns an export object. Throws Export.ExportTypeError if export_type is not in EXPORT_TYPES. Throws Export.ExportConnectionError if rabbitmq broker is down. """ username = xform.user.username id_string = xform.id_string def _create_export(xform, export_type, options): export_options = { key: get_boolean_value(value, default=True) for (key, value) in iteritems(options) if key in Export.EXPORT_OPTION_FIELDS } if query and 'query' not in export_options: export_options['query'] = query return Export.objects.create( xform=xform, export_type=export_type, options=export_options) export = _create_export(xform, export_type, options) result = None export_id = export.id options.update({ 'username': username, 'id_string': id_string, 'export_id': export_id, 'query': query, 'force_xlsx': force_xlsx }) export_types = { Export.XLS_EXPORT: create_xls_export, Export.GOOGLE_SHEETS_EXPORT: create_google_sheet_export, Export.CSV_EXPORT: create_csv_export, Export.CSV_ZIP_EXPORT: create_csv_zip_export, Export.SAV_ZIP_EXPORT: create_sav_zip_export, Export.ZIP_EXPORT: create_zip_export, Export.KML_EXPORT: create_kml_export, Export.OSM_EXPORT: create_osm_export, Export.EXTERNAL_EXPORT: create_external_export } # start async export if export_type in export_types: try: result = export_types[export_type].apply_async((), kwargs=options) except OperationalError as e: export.internal_status = Export.FAILED export.error_message = "Error connecting to broker." export.save() report_exception(export.error_message, e, sys.exc_info()) raise Export.ExportConnectionError else: raise ExportTypeError if result: # when celery is running eager, the export has been generated by the # time we get here so lets retrieve the export object a fresh before we # save if getattr(settings, 'CELERY_TASK_ALWAYS_EAGER', False): export = get_object_or_404(Export, id=export.id) export.task_id = result.task_id export.save() return export, result return None
def generate_export(export_type, xform, export_id=None, options=None): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') show_choice_labels: boolean flag language: language labels as in the XLSForm/XForm """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True ) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) export_builder.INCLUDE_LABELS = options.get('include_labels', False) include_reviews = options.get('include_reviews', False) export_builder.INCLUDE_LABELS_ONLY = options.get( 'include_labels_only', False ) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels', False) export_builder.language = options.get('language') # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.INCLUDE_REVIEWS = include_reviews export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records ) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export