def create_osm_export(username, id_string, export_id, **options):
"""
OSM export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_osm_export(
Export.OSM_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"OSM Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_osm_export(username, id_string, export_id, **options):
"""
OSM export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_osm_export(
Export.OSM_EXPORT,
username,
id_string,
export_id,
options,
xform=export.xform)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"OSM Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def test_generate_osm_export(self):
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(os.path.dirname(api_tests.__file__), 'fixtures',
'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {"extension": Attachment.OSM}
export = generate_osm_export(Attachment.OSM, self.user.username,
self.xform.id_string, None, options)
self.assertTrue(export.is_successful)
with open(combined_osm_path) as f:
osm = f.read()
with default_storage.open(export.filepath) as f2:
content = f2.read()
self.assertMultiLineEqual(content.strip(), osm.strip())
def test_generate_osm_export(self):
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(os.path.join(
os.path.dirname(api_tests.__file__), 'fixtures', 'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename)
for filename in filenames]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
export = generate_osm_export(Attachment.OSM, Attachment.OSM,
self.user.username, self.xform.id_string)
self.assertTrue(export.is_successful)
with open(combined_osm_path) as f:
osm = f.read()
with default_storage.open(export.filepath) as f2:
content = f2.read()
self.assertMultiLineEqual(content, osm)
def create_osm_export(username, id_string, export_id, query=None):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_osm_export(
Export.OSM_EXPORT, 'osm', username, id_string, export_id, query)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("OSM Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
raise
else:
return gen_export.id
def test_generate_osm_export(self):
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(
os.path.dirname(api_tests.__file__), 'fixtures', 'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {"extension": Attachment.OSM}
export = generate_osm_export(Attachment.OSM, self.user.username,
self.xform.id_string, None, options)
self.assertTrue(export.is_successful)
with open(combined_osm_path, encoding='utf-8') as f:
osm = f.read()
with default_storage.open(export.filepath) as f2:
content = f2.read().decode('utf-8')
self.assertMultiLineEqual(content.strip(), osm.strip())
# delete submission and check that content is no longer in export
submission = self.xform.instances.filter().first()
submission.deleted_at = timezone.now()
submission.save()
export = generate_osm_export(Attachment.OSM, self.user.username,
self.xform.id_string, None, options)
self.assertTrue(export.is_successful)
with default_storage.open(export.filepath) as f2:
content = f2.read()
self.assertEqual(content, b'')
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(export_type, xform.user.username,
xform.id_string, None,
request.GET.get('token'), query,
request.GET.get('meta'),
request.GET.get('data_id'))
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(export_type,
extension,
xform.user.username,
xform.id_string,
export_id=None,
filter_query=None)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(export_type,
extension,
xform.user.username,
xform.id_string,
export_id=None,
filter_query=None)
else:
remove_group_name = False
if "remove_group_name" in request.QUERY_PARAMS:
remove_group_name = \
str_to_bool(request.QUERY_PARAMS["remove_group_name"])
export = generate_export(export_type,
extension,
xform.user.username,
xform.id_string,
None,
query,
remove_group_name=remove_group_name)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(
export_type, xform.user.username,
xform.id_string, None, request.GET.get('token'), query,
request.GET.get('meta'), request.GET.get('data_id')
)
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(
export_type, extension, xform.user.username,
xform.id_string, export_id=None, filter_query=None)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(
export_type, extension, xform.user.username,
xform.id_string, export_id=None, filter_query=None)
else:
remove_group_name = False
if "remove_group_name" in request.QUERY_PARAMS:
remove_group_name = \
str_to_bool(request.QUERY_PARAMS["remove_group_name"])
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query,
remove_group_name=remove_group_name
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def _generate_new_export(request,
xform,
query,
export_type,
dataview_pk=False):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
options = {
"extension": extension,
"username": xform.user.username,
"id_string": xform.id_string,
}
if query:
options['query'] = query
options["dataview_pk"] = dataview_pk
if export_type == Export.GOOGLE_SHEETS_EXPORT:
options['google_credentials'] = \
_get_google_credential(request).to_json()
try:
if export_type == Export.EXTERNAL_EXPORT:
options['token'] = request.GET.get('token')
options['data_id'] = request.GET.get('data_id')
options['meta'] = request.GET.get('meta')
export = generate_external_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.ZIP_EXPORT:
export = generate_attachments_zip_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(export_type,
xform.user.username,
xform.id_string,
None,
options,
xform=xform)
else:
options.update(parse_request_export_options(request.query_params))
export = generate_export(export_type, xform, None, options)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return async_status(FAILED, str(e))
except SPSSIOError as e:
raise exceptions.ParseError(str(e))
else:
return export
