def test_csv_nested_repeat_output(self):
path = os.path.join(self.fixture_dir, 'double_repeat.xls')
self._publish_xls_file(path)
path = os.path.join(self.fixture_dir, 'instance.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
self.maxDiff = None
dd = DataDictionary.objects.all()[0]
xpaths = [
u'/double_repeat/bed_net[1]/member[1]/name',
u'/double_repeat/bed_net[1]/member[2]/name',
u'/double_repeat/bed_net[2]/member[1]/name',
u'/double_repeat/bed_net[2]/member[2]/name',
u'/double_repeat/meta/instanceID'
]
self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'double_repeat')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(self.fixture_dir, 'export.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(
export_type, xform.user.username,
xform.id_string, None, request.GET.get('token'), query,
request.GET.get('meta')
)
else:
export = generate_export(
export_type, extension, xform.user.username,
xform.id_string, None, query
)
audit = {
"xform": xform.id_string,
"export_type": export_type
}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def create_sav_zip_export(username, id_string, export_id, query=None,
group_delimiter='/', split_select_multiples=True,
binary_select_multiples=False):
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(
Export.SAV_ZIP_EXPORT, 'zip', username, id_string, export_id,
query, group_delimiter, split_select_multiples,
binary_select_multiples
)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
raise
else:
return gen_export.id
def _generate_new_export(request, xform, query, export_type):
query = _set_start_end_params(request, query)
extension = _get_extension_from_export_type(export_type)
try:
if export_type == Export.EXTERNAL_EXPORT:
export = generate_external_export(export_type, xform.user.username,
xform.id_string, None,
request.GET.get('token'), query,
request.GET.get('meta'))
else:
export = generate_export(export_type, extension,
xform.user.username, xform.id_string,
None, query)
audit = {"xform": xform.id_string, "export_type": export_type}
log.audit_log(
log.Actions.EXPORT_CREATED, request.user, xform.user,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
raise Http404(_("No records found to export"))
except J2XException as e:
# j2x exception
return {'error': str(e)}
else:
return export
def test_csv_nested_repeat_output(self):
path = os.path.join(self.fixture_dir, 'double_repeat.xls')
self._publish_xls_file(path)
path = os.path.join(self.fixture_dir, 'instance.xml')
self._make_submission(path,
forced_submission_time=self._submission_time)
self.maxDiff = None
dd = DataDictionary.objects.all()[0]
xpaths = [
u'/double_repeat/bed_net[1]/member[1]/name',
u'/double_repeat/bed_net[1]/member[2]/name',
u'/double_repeat/bed_net[2]/member[1]/name',
u'/double_repeat/bed_net[2]/member[2]/name',
u'/double_repeat/meta/instanceID'
]
self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'double_repeat')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(self.fixture_dir, 'export.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def create_csv_export(username, id_string, export_id, query=None,
group_delimiter='/', split_select_multiples=True,
binary_select_multiples=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(
Export.CSV_EXPORT, 'csv', username, id_string, export_id, query,
group_delimiter, split_select_multiples, binary_select_multiples)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
raise
else:
return gen_export.id
def create_sav_zip_export(username,
id_string,
export_id,
query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.SAV_ZIP_EXPORT, 'zip', username,
id_string, export_id, query,
group_delimiter, split_select_multiples,
binary_select_multiples)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception(
"SAV ZIP Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def test_allow_export_download_for_basic_auth(self):
extra = {
'HTTP_AUTHORIZATION': http_auth_string(self.login_username,
self.login_password)
}
# create export
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
self.assertTrue(isinstance(export, Export))
url = reverse(export_download, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': export.export_type,
'filename': export.filename
})
response = self.anon.get(url, **extra)
self.assertEqual(response.status_code, 200)
def test_allow_export_download_for_basic_auth(self):
extra = {
'HTTP_AUTHORIZATION':
http_auth_string(self.login_username, self.login_password)
}
# create export
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
self.xform.id_string)
self.assertTrue(isinstance(export, Export))
url = reverse(export_download,
kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': export.export_type,
'filename': export.filename
})
response = self.anon.get(url, **extra)
self.assertEqual(response.status_code, 200)
def test_csv_export_output(self):
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
self._make_submission(
path, forced_submission_time=self._submission_time)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'tutorial_w_repeats')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(
self.fixture_dir, 'tutorial_w_repeats.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def test_csv_export_output(self):
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
self._publish_xls_file_and_set_xform(path)
path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
self._make_submission(path,
forced_submission_time=self._submission_time)
# test csv
export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
'tutorial_w_repeats')
storage = get_storage_class()()
self.assertTrue(storage.exists(export.filepath))
path, ext = os.path.splitext(export.filename)
self.assertEqual(ext, '.csv')
with open(os.path.join(self.fixture_dir,
'tutorial_w_repeats.csv')) as f1:
with storage.open(export.filepath) as f2:
expected_content = f1.read()
actual_content = f2.read()
self.assertEquals(actual_content, expected_content)
def create_xls_export(username,
id_string,
export_id,
query=None,
force_xlsx=True,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
ext = 'xls' if not force_xlsx else 'xlsx'
try:
export = Export.objects.get(id=export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(Export.XLS_EXPORT, ext, username,
id_string, export_id, query,
group_delimiter, split_select_multiples,
binary_select_multiples)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception(
"XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def create_csv_export(username,
id_string,
export_id,
query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
export = Export.objects.get(id=export_id)
try:
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
gen_export = generate_export(Export.CSV_EXPORT, 'csv', username,
id_string, export_id, query,
group_delimiter, split_select_multiples,
binary_select_multiples)
except NoRecordsFoundError:
# not much we can do but we don't want to report this as the user
# should not even be on this page if the survey has no records
export.internal_status = Export.FAILED
export.save()
except Exception as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception(
"CSV Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s" % details, e,
sys.exc_info())
raise
else:
return gen_export.id
def create_xls_export(username, id_string, export_id, query=None,
force_xlsx=True, group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
# we re-query the db instead of passing model objects according to
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
ext = 'xls' if not force_xlsx else 'xlsx'
try:
export = Export.objects.get(id=export_id)
except Export.DoesNotExist:
# no export for this ID return None.
return None
# though export is not available when for has 0 submissions, we
# catch this since it potentially stops celery
try:
gen_export = generate_export(
Export.XLS_EXPORT, ext, username, id_string, export_id, query,
group_delimiter, split_select_multiples, binary_select_multiples)
except (Exception, NoRecordsFoundError) as e:
export.internal_status = Export.FAILED
export.save()
# mail admins
details = {
'export_id': export_id,
'username': username,
'id_string': id_string
}
report_exception("XLS Export Exception: Export ID - "
"%(export_id)s, /%(username)s/%(id_string)s"
% details, e, sys.exc_info())
# Raise for now to let celery know we failed
# - doesnt seem to break celery`
raise
else:
return gen_export.id
def data_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__iexact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {"xform": xform.id_string, "export_type": export_type}
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type) or query or\
'start' in request.GET or 'end' in request.GET:
# check for start and end params
if 'start' in request.GET or 'end' in request.GET:
if not query:
query = '{}'
query = json.dumps(
_set_submission_time_to_query(json.loads(query), request))
try:
export = generate_export(export_type, extension, username,
id_string, None, query)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
else:
export = newset_export_for(xform, export_type)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") % {
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
path, ext = os.path.splitext(export.filename)
ext = ext[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext],
id_string,
extension=ext,
file_path=export.filepath)
return response
def data_export(request, username, id_string, export_type):
owner = get_object_or_404(User, username__iexact=username)
xform = get_object_or_404(XForm, id_string__iexact=id_string, user=owner)
helper_auth_helper(request)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u'Not shared.'))
query = request.GET.get("query")
extension = export_type
# check if we should force xlsx
force_xlsx = request.GET.get('xls') != 'true'
if export_type == Export.XLS_EXPORT and force_xlsx:
extension = 'xlsx'
elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
extension = 'zip'
audit = {
"xform": xform.id_string,
"export_type": export_type
}
# check if we need to re-generate,
# we always re-generate if a filter is specified
if should_create_new_export(xform, export_type) or query or\
'start' in request.GET or 'end' in request.GET:
# check for start and end params
if 'start' in request.GET or 'end' in request.GET:
if not query:
query = '{}'
query = json.dumps(
_set_submission_time_to_query(json.loads(query), request))
try:
export = generate_export(
export_type, extension, username, id_string, None, query)
audit_log(
Actions.EXPORT_CREATED, request.user, owner,
_("Created %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
except NoRecordsFoundError:
return HttpResponseNotFound(_("No records found to export"))
else:
export = newset_export_for(xform, export_type)
# log download as well
audit_log(
Actions.EXPORT_DOWNLOADED, request.user, owner,
_("Downloaded %(export_type)s export on '%(id_string)s'.") %
{
'id_string': xform.id_string,
'export_type': export_type.upper()
}, audit, request)
if not export.filename:
# tends to happen when using newset_export_for.
return HttpResponseNotFound("File does not exist!")
# get extension from file_path, exporter could modify to
# xlsx if it exceeds limits
path, ext = os.path.splitext(export.filename)
ext = ext[1:]
if request.GET.get('raw'):
id_string = None
response = response_with_mimetype_and_name(
Export.EXPORT_MIMES[ext], id_string, extension=ext,
file_path=export.filepath)
return response