def test_delete_oldest_export_on_limit(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create first export
first_export = generate_export(
Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string)
self.assertIsNotNone(first_export.pk)
# create exports that exceed set limit
for i in range(Export.MAX_EXPORTS):
generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
# first export should be deleted
exports = Export.objects.filter(id=first_export.id)
self.assertEqual(len(exports), 0)
def test_dotted_fields_csv_export(self):
"""
Test CSV export with dotted field names
"""
path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone',
'userone_with_dot_name_fields.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone',
'userone_with_dot_name_fields.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
# test csv
self.options['id_string'] = 'userone'
export = generate_export(
Export.CSV_EXPORT,
self.xform,
None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
test_file_path = os.path.join(
os.path.dirname(__file__), 'fixtures', 'userone',
'userone_with_dot_name_fields.csv')
with storage.open(export.filepath, 'r') as csv_file:
self._test_csv_files(csv_file, test_file_path)
def create_csv_export(username, id_string, export_id, **options):
"""
CSV export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.CSV_EXPORT, export.xform,
export_id, options)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.error_message = str(e)
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def test_csv_nested_repeat_output(self):
path = os.path.join(self.fixture_dir, 'double_repeat.xls')
self._publish_xls_file(path)
path = os.path.join(self.fixture_dir, 'instance.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
self.maxDiff = None
dd = DataDictionary.objects.all()[0]
xpaths = [
'/double_repeat/bed_net[1]/member[1]/name',
'/double_repeat/bed_net[1]/member[2]/name',
'/double_repeat/bed_net[2]/member[1]/name',
'/double_repeat/bed_net[2]/member[2]/name',
'/double_repeat/meta/instanceID'
]
self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'double_repeat')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(self.fixture_dir, 'export.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(
export_type, xform.user.username,
xform.id_string, None, request.GET.get('token'), query,
request.GET.get('meta')
)
else:
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def create_sav_zip_export(username,
id_string,
export_id,
query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.SAV_ZIP_EXPORT, 'zip', username,
id_string, export_id, query,
group_delimiter, split_select_multiples,
binary_select_multiples)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception(
"SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_csv_export(username, id_string, export_id, query=None,
group_delimiter='/', split_select_multiples=True,
binary_select_multiples=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(
Export.CSV_EXPORT, 'csv', username, id_string, export_id, query,
group_delimiter, split_select_multiples, binary_select_multiples)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
raise
else:
return gen_export.id
def test_duplicate_export_filename_is_renamed(self):
self._publish_transportation_form()
self._submit_transport_instance()
# TODO: mock the time
# only works if the time we time we generate the basename
# is exact to the second with the time the 2nd export is created
# create an export object in the db
basename = "%s_%s" % (
self.xform.id_string,
datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + ".csv"
export = Export.objects.create(
xform=self.xform, export_type=Export.CSV_EXPORT, filename=filename)
# 2nd export
export_2 = generate_export(
Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string)
if export.created_on.timetuple() == export_2.created_on.timetuple():
new_filename = increment_index_in_filename(filename)
self.assertEqual(new_filename, export_2.filename)
else:
self.skipTest("duplicate export filename test skipped "
"because export times differ.")
def create_xls_export(username, id_string, export_id, query=None,
force_xlsx=True, group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
ext = 'xls' if not force_xlsx else 'xlsx'
export = Export.objects.get(id=export_id)
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(
Export.XLS_EXPORT, ext, username, id_string, export_id, query,
group_delimiter, split_select_multiples, binary_select_multiples)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
#raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def test_dotted_fields_csv_export(self):
"""
Test CSV export with dotted field names
"""
path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone',
'userone_with_dot_name_fields.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone',
'userone_with_dot_name_fields.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
# test csv
self.options['id_string'] = 'userone'
export = generate_export(
Export.CSV_EXPORT,
self.xform,
None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
test_file_path = os.path.join(
os.path.dirname(__file__), 'fixtures', 'userone',
'userone_with_dot_name_fields.csv')
with storage.open(export.filepath, 'r') as csv_file:
self._test_csv_files(csv_file, test_file_path)
def test_csv_repeat_with_note(self):
"""
Test that note field in repeat is not in csv export
"""
path = os.path.join(self.fixture_dir, 'repeat_w_note.xlsx')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'repeat_w_note.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
export = generate_export(
Export.CSV_EXPORT,
self.xform,
None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with storage.open(export.filepath, 'r') as csv_file:
reader = csv.reader(csv_file)
rows = [row for row in reader]
actual_headers = [h for h in rows[0]]
expected_headers = [
'chnum', 'chrepeat[1]/chname', 'chrepeat[2]/chname',
'meta/instanceID', '_id', '_uuid', '_submission_time',
'_tags', '_notes', '_version', '_duration', '_submitted_by',
'_total_media', '_media_count', '_media_all_received']
self.assertEqual(sorted(expected_headers), sorted(actual_headers))
def create_xls_export(username, id_string, export_id, **options):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
force_xlsx = options.get("force_xlsx", True)
options["extension"] = 'xlsx' if force_xlsx else 'xls'
try:
export = _get_export_object(id=export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.XLS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(
export_type, xform.user.username,
xform.id_string, None, request.GET.get('token'), query,
request.GET.get('meta')
)
else:
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def create_csv_export(username, id_string, export_id, query=None,
group_delimiter='/', split_select_multiples=True,
binary_select_multiples=False, remove_group_name=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(
Export.CSV_EXPORT, 'csv', username, id_string, export_id, query,
group_delimiter, split_select_multiples, binary_select_multiples,
remove_group_name=remove_group_name
)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
raise
else:
return gen_export.id
def test_csv_repeat_with_note(self):
"""
Test that note field in repeat is not in csv export
"""
path = os.path.join(self.fixture_dir, 'repeat_w_note.xlsx')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'repeat_w_note.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
export = generate_export(
Export.CSV_EXPORT,
self.xform,
None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with storage.open(export.filepath, 'r') as csv_file:
reader = csv.reader(csv_file)
rows = [row for row in reader]
actual_headers = [h for h in rows[0]]
expected_headers = [
'chnum', 'chrepeat[1]/chname', 'chrepeat[2]/chname',
'meta/instanceID', '_id', '_uuid', '_submission_time',
'_tags', '_notes', '_version', '_duration', '_submitted_by',
'_total_media', '_media_count', '_media_all_received']
self.assertEqual(sorted(expected_headers), sorted(actual_headers))
def create_xls_export(username, id_string, export_id, **options):
"""
XLS export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
force_xlsx = options.get("force_xlsx", True)
options["extension"] = 'xlsx' if force_xlsx else 'xls'
try:
export = _get_export_object(export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.XLS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def test_csv_nested_repeat_output(self):
"""
Test CSV export with nested repeats
"""
path = os.path.join(self.fixture_dir, 'double_repeat.xls')
self._publish_xls_file(path)
self.xform = XForm.objects.get(id_string='double_repeat')
path = os.path.join(self.fixture_dir, 'instance.xml')
self._make_submission(path,
forced_submission_time=self._submission_time)
self.maxDiff = None
data_dictionary = DataDictionary.objects.all()[0]
xpaths = [
u'/double_repeat/bed_net[1]/member[1]/name',
u'/double_repeat/bed_net[1]/member[2]/name',
u'/double_repeat/bed_net[2]/member[1]/name',
u'/double_repeat/bed_net[2]/member[2]/name',
u'/double_repeat/meta/instanceID'
]
self.assertEquals(data_dictionary.xpaths(repeat_iterations=2), xpaths)
# test csv
export = generate_export(Export.CSV_EXPORT, self.xform, None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
test_file_path = os.path.join(self.fixture_dir, 'export.csv')
with storage.open(export.filepath) as csv_file:
self._test_csv_files(csv_file, test_file_path)
def create_sav_zip_export(username, id_string, export_id, query=None,
group_delimiter='/', split_select_multiples=True,
binary_select_multiples=False):
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(
Export.SAV_ZIP_EXPORT, 'zip', username, id_string, export_id,
query, group_delimiter, split_select_multiples,
binary_select_multiples
)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
raise
else:
return gen_export.id
def create_csv_export(username, id_string, export_id, **options):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = _get_export_object(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.CSV_EXPORT, export.xform,
export_id, options)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def test_csv_nested_repeat_output(self):
path = os.path.join(self.fixture_dir, 'double_repeat.xls')
self._publish_xls_file(path)
path = os.path.join(self.fixture_dir, 'instance.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
self.maxDiff = None
dd = DataDictionary.objects.all()[0]
xpaths = [
u'/double_repeat/bed_net[1]/member[1]/name',
u'/double_repeat/bed_net[1]/member[2]/name',
u'/double_repeat/bed_net[2]/member[1]/name',
u'/double_repeat/bed_net[2]/member[2]/name',
u'/double_repeat/meta/instanceID'
]
self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'double_repeat')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(self.fixture_dir, 'export.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def test_csv_nested_repeat_output(self):
"""
Test CSV export with nested repeats
"""
path = os.path.join(self.fixture_dir, 'double_repeat.xls')
self._publish_xls_file(path)
self.xform = XForm.objects.get(id_string='double_repeat')
path = os.path.join(self.fixture_dir, 'instance.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
self.maxDiff = None
data_dictionary = DataDictionary.objects.all()[0]
xpaths = [
u'/double_repeat/bed_net[1]/member[1]/name',
u'/double_repeat/bed_net[1]/member[2]/name',
u'/double_repeat/bed_net[2]/member[1]/name',
u'/double_repeat/bed_net[2]/member[2]/name',
u'/double_repeat/meta/instanceID'
]
self.assertEquals(data_dictionary.xpaths(repeat_iterations=2), xpaths)
# test csv
export = generate_export(
Export.CSV_EXPORT,
self.xform,
None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
test_file_path = os.path.join(self.fixture_dir, 'export.csv')
with storage.open(export.filepath, 'r') as csv_file:
self._test_csv_files(csv_file, test_file_path)
def test_last_submission_time_on_export(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
generate_export(
Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string)
num_exports = Export.objects.filter(
xform=self.xform, export_type=Export.XLS_EXPORT).count()
# check that our function knows there are no more submissions
self.assertFalse(
Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
sleep(1)
# force new last submission date on xform
last_submission = self.xform.instances.order_by('-date_created')[0]
last_submission.date_created += datetime.timedelta(hours=1)
last_submission.save()
# check that our function knows data has changed
self.assertTrue(
Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
# check that requesting list url will generate a new export
export_list_url = reverse(export_list, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': Export.XLS_EXPORT
})
self.client.get(export_list_url)
self.assertEqual(
Export.objects.filter(xform=self.xform,
export_type=Export.XLS_EXPORT).count(),
num_exports + 1)
# make sure another export type causes auto-generation
num_exports = Export.objects.filter(
xform=self.xform, export_type=Export.CSV_EXPORT).count()
export_list_url = reverse(export_list, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': Export.CSV_EXPORT
})
self.client.get(export_list_url)
self.assertEqual(
Export.objects.filter(xform=self.xform,
export_type=Export.CSV_EXPORT).count(),
num_exports + 1)
def test_delete_file_on_export_delete(self):
self._publish_transportation_form()
self._submit_transport_instance()
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
# delete export object
export.delete()
self.assertFalse(storage.exists(export.filepath))
def test_export_progress_output(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create exports
for i in range(2):
generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
self.assertEqual(Export.objects.count(), 2)
# progress for multiple exports
progress_url = reverse(export_progress, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': 'xls'
})
get_data = {'export_ids': [e.id for e in Export.objects.all()]}
response = self.client.get(progress_url, get_data)
content = json.loads(response.content)
self.assertEqual(len(content), 2)
self.assertEqual(sorted(['url', 'export_id', 'complete', 'filename']),
sorted(content[0].keys()))
def test_last_submission_time_empty(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
# set time of last submission to None
export.time_of_last_submission = None
export.save()
self.assertTrue(Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
def test_generate_csv_zip_export(self):
# publish xls form
self._publish_transportation_form_and_submit_instance()
# create export db object
export = generate_export(
Export.CSV_ZIP_EXPORT, "zip", self.user.username,
self.xform.id_string, group_delimiter='/',
split_select_multiples=True)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.zip')
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(export_type, xform.user.username,
xform.id_string, None,
request.GET.get('token'), query,
request.GET.get('meta'),
request.GET.get('data_id'))
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(export_type,
extension,
xform.user.username,
xform.id_string,
export_id=None,
filter_query=None)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(export_type,
extension,
xform.user.username,
xform.id_string,
export_id=None,
filter_query=None)
else:
remove_group_name = False
if "remove_group_name" in request.QUERY_PARAMS:
remove_group_name = \
str_to_bool(request.QUERY_PARAMS["remove_group_name"])
export = generate_export(export_type,
extension,
xform.user.username,
xform.id_string,
None,
query,
remove_group_name=remove_group_name)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def test_export_public_project(self):
"""
Test export of a public form for anonymous users.
"""
self._create_user_and_login()
self._publish_transportation_form()
self.xform.shared_data = True
self.xform.save()
export = generate_export(Export.CSV_EXPORT, self.xform, None,
{"extension": "csv"})
request = self.factory.get('/export')
response = self.view(request, pk=export.pk)
self.assertEqual(status.HTTP_200_OK, response.status_code)
def test_export_non_public_export(self):
"""
Test export of a private form for anonymous users results in
HTTP_404_NOT_FOUND response.
"""
self._create_user_and_login()
self._publish_transportation_form()
self.xform.shared_data = False
self.xform.save()
export = generate_export(Export.CSV_EXPORT, self.xform, None,
{"extension": "csv"})
request = self.factory.get('/export')
response = self.view(request, pk=export.pk)
self.assertEqual(status.HTTP_404_NOT_FOUND, response.status_code)
def test_create_export(self):
self._publish_transportation_form_and_submit_instance()
storage = get_storage_class()()
# test xls
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.xls')
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
# test xls with existing export_id
existing_export = Export.objects.create(xform=self.xform,
export_type=Export.XLS_EXPORT)
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string, existing_export.id)
self.assertEqual(existing_export.id, export.id)
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(
export_type, xform.user.username,
xform.id_string, None, request.GET.get('token'), query,
request.GET.get('meta'), request.GET.get('data_id')
)
elif export_type == Export.OSM_EXPORT:
export = generate_osm_export(
export_type, extension, xform.user.username,
xform.id_string, export_id=None, filter_query=None)
elif export_type == Export.KML_EXPORT:
export = generate_kml_export(
export_type, extension, xform.user.username,
xform.id_string, export_id=None, filter_query=None)
else:
remove_group_name = False
if "remove_group_name" in request.QUERY_PARAMS:
remove_group_name = \
str_to_bool(request.QUERY_PARAMS["remove_group_name"])
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query,
remove_group_name=remove_group_name
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def test_export_download_url(self):
self._publish_transportation_form()
self._submit_transport_instance()
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
csv_export_url = reverse(export_download, kwargs={
"username": self.user.username,
"id_string": self.xform.id_string,
"export_type": Export.CSV_EXPORT,
"filename": export.filename
})
response = self.client.get(csv_export_url)
self.assertEqual(response.status_code, 200)
# test xls
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
xls_export_url = reverse(export_download, kwargs={
"username": self.user.username,
"id_string": self.xform.id_string,
"export_type": Export.XLS_EXPORT,
"filename": export.filename
})
response = self.client.get(xls_export_url)
self.assertEqual(response.status_code, 200)
def create_db_export(username,
id_string,
export_id,
query=None,
force_xlsx=False,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
show_label=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
ext = 'txt' if not force_xlsx else 'txt'
try:
logger = create_db_export.get_logger()
logger.info("Enter to get export_id")
# print 'Enter to get export_id'
export = Export.objects.get(id=export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.DB_EXPORT, ext, username,
id_string, export_id, query,
group_delimiter, split_select_multiples,
binary_select_multiples, show_label)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception(
"DB Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def test_allow_export_download_for_basic_auth(self):
extra = {
'HTTP_AUTHORIZATION': http_auth_string(self.login_username,
self.login_password)
}
# create export
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
self.assertTrue(isinstance(export, Export))
url = reverse(export_download, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': export.export_type,
'filename': export.filename
})
response = self.anon.get(url, **extra)
self.assertEqual(response.status_code, 200)
def test_allow_export_download_for_basic_auth(self):
extra = {
'HTTP_AUTHORIZATION':
http_auth_string(self.login_username, self.login_password)
}
# create export
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
self.assertTrue(isinstance(export, Export))
url = reverse(export_download,
kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': export.export_type,
'filename': export.filename
})
response = self.anon.get(url, **extra)
self.assertEqual(response.status_code, 200)
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
export = generate_export(export_type, extension, xform.user.username,
xform.id_string, None, query)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
t("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(t("No records found to export"))
else:
return export
def test_delete_export_url(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
exports = Export.objects.filter(id=export.id)
self.assertEqual(len(exports), 1)
delete_url = reverse(delete_export, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': 'xls'
})
post_data = {'export_id': export.id}
response = self.client.post(delete_url, post_data)
self.assertEqual(response.status_code, 302)
exports = Export.objects.filter(id=export.id)
self.assertEqual(len(exports), 0)
def create_analyser_export(username, id_string, export_id, query=None):
# Mostly a serving of copy pasta based on the above `create_xls_export()`. Enjoy.
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
ext = 'xlsx'
try:
export = Export.objects.get(id=export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.ANALYSER_EXPORT,
ext,
username,
id_string,
export_id,
query,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception(
"Analyser Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def create_sav_zip_export(username, id_string, export_id, **options):
export = _get_export_object(id=export_id)
options["extension"] = Export.ZIP_EXPORT
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, TypeError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception("SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" %
details, e, sys.exc_info())
raise
else:
return gen_export.id
def test_csv_export_output(self):
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'tutorial_w_repeats')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(
self.fixture_dir, 'tutorial_w_repeats.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def test_csv_export_output(self):
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'tutorial_w_repeats')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(
self.fixture_dir, 'tutorial_w_repeats.csv'), 'rb') as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEqual(actual_content, expected_content)
def test_404_on_export_io_error(self):
"""
Test that we return a 404 when the response_with_mimetype_and_name
encounters an IOError
"""
self._publish_transportation_form()
self._submit_transport_instance()
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
export_url = reverse(export_download, kwargs={
"username": self.user.username,
"id_string": self.xform.id_string,
"export_type": Export.CSV_EXPORT,
"filename": export.filename
})
# delete the export
export.delete()
# access the export
response = self.client.get(export_url)
self.assertEqual(response.status_code, 404)
def create_google_sheet_export(username, id_string, export_id, **options):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
try:
export = _get_export_object(id=export_id)
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, ConnectionError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception("Google Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" %
details, e, sys.exc_info())
raise
else:
return gen_export.id
def test_csv_export_truncated_titles(self):
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
self._make_submission(path,
forced_submission_time=self._submission_time)
# test csv
self.options['remove_group_name'] = True
export = generate_export(Export.CSV_EXPORT, self.xform, None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(
self.fixture_dir, 'tutorial_w_repeats_truncate_titles.csv')) \
as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def test_csv_export_output(self):
"""
Test CSV export output
"""
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
self._make_submission(path,
forced_submission_time=self._submission_time)
# test csv
export = generate_export(Export.CSV_EXPORT, self.xform, None,
self.options)
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
test_file_path = os.path.join(self.fixture_dir,
'tutorial_w_repeats.csv')
with storage.open(export.filepath, 'r') as csv_file:
self._test_csv_files(csv_file, test_file_path)
def test_export_not_found(self):
export_type = "csv"
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._publish_transportation_form_and_submit_instance()
self._create_old_export(self.xform, export_type, options)
export = Export(
xform=self.xform, export_type=export_type, options=options)
export.save()
export_id = export.pk
export.delete()
export = generate_export(export_type, self.xform, export_id, options)
self.assertIsNotNone(export)
self.assertTrue(export.is_successful)
self.assertNotEqual(export_id, export.pk)
def test_export_not_found(self):
export_type = "csv"
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._publish_transportation_form_and_submit_instance()
self._create_old_export(self.xform, export_type, options)
export = Export(
xform=self.xform, export_type=export_type, options=options)
export.save()
export_id = export.pk
export.delete()
export = generate_export(export_type, self.xform, export_id, options)
self.assertIsNotNone(export)
self.assertTrue(export.is_successful)
self.assertNotEqual(export_id, export.pk)
def test_graceful_exit_on_export_delete_if_file_doesnt_exist(self):
self._publish_transportation_form()
self._submit_transport_instance()
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
storage = get_storage_class()()
# delete file
storage.delete(export.filepath)
self.assertFalse(storage.exists(export.filepath))
# clear filename, like it would be in an incomplete export
export.filename = None
export.filedir = None
export.save()
# delete export record, which should try to delete file as well
delete_url = reverse(delete_export, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': 'xls'
})
post_data = {'export_id': export.id}
response = self.client.post(delete_url, post_data)
self.assertEqual(response.status_code, 302)
def create_sav_zip_export(username, id_string, export_id, **options):
"""
SPSS sav export task.
"""
export = _get_export_object(export_id)
options["extension"] = Export.ZIP_EXPORT
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, TypeError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_google_sheet_export(username, id_string, export_id, **options):
"""
Google Sheets export task.
"""
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
try:
export = _get_export_object(export_id)
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform,
export_id, options)
except (Exception, NoRecordsFoundError, ConnectionError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = _get_export_details(username, id_string, export_id)
report_exception(
"Google Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_analyser_export(username, id_string, export_id, query=None):
# Mostly a serving of copy pasta based on the above `create_xls_export()`. Enjoy.
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
ext = 'xlsx'
try:
export = Export.objects.get(id=export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.ANALYSER_EXPORT, ext, username, id_string, export_id,
query, group_delimiter='/', split_select_multiples=True,
binary_select_multiples=False)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("Analyser Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
else:
return export
