def test_edit_updated_geopoint_cache(self):
query_args = {
'xform': self.xform,
'query': '{}',
'fields': '[]',
'count': True
}
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml")
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
# query mongo for the _geopoint field
query_args['count'] = False
records = query_data(**query_args)
self.assertEqual(len(records), 1)
# submit the edited instance
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited.xml")
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
records = query_data(**query_args)
self.assertEqual(len(records), 1)
cached_geopoint = records[0][GEOLOCATION]
# the cached geopoint should equal the gps field
gps = records[0]['gps'].split(" ")
self.assertEqual(float(gps[0]), float(cached_geopoint[0]))
self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_query_filter_by_integer(self):
self._publish_transportation_form()
self._make_submissions()
oldest = Instance.objects.filter(xform=self.xform).first().pk
data = [
i.get('_id')
for i in query_data(self.xform, query='[{"_id": %s}]' % (oldest))
]
self.assertEqual(len(data), 1)
self.assertEqual(data, [oldest])
# with fields
data = [
i.get('_id') for i in query_data(
self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]')
]
self.assertEqual(len(data), 1)
self.assertEqual(data, [oldest])
# mongo $gt
data = [
i.get('_id')
for i in query_data(self.xform,
query='{"_id": {"$gt": %s}}' % (oldest),
fields='["_id"]')
]
self.assertEqual(self.xform.instances.count(), 4)
self.assertEqual(len(data), 3)
def set_object_list_and_total_count(self, query, fields, sort, start,
limit, is_public_request):
try:
if not is_public_request:
xform = self.get_object()
where, where_params = get_where_clause(query)
if where:
self.object_list = self.object_list.extra(where=where,
params=where_params)
if (start and limit or limit) and (not sort and not fields):
start = start if start is not None else 0
limit = limit if start is None or start == 0 else start + limit
self.object_list = filter_queryset_xform_meta_perms(
self.get_object(), self.request.user, self.object_list)
self.object_list = \
self.object_list.order_by('pk')[start: limit]
self.total_count = self.object_list.count()
elif (sort or limit or start or fields) and not is_public_request:
try:
query = \
filter_queryset_xform_meta_perms_sql(self.get_object(),
self.request.user,
query)
self.object_list = query_data(xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields)
self.total_count = query_data(xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields,
count=True)[0].get('count')
except NoRecordsPermission:
self.object_list = []
self.total_count = 0
else:
self.total_count = self.object_list.count()
if self.total_count and isinstance(self.object_list, QuerySet):
self.etag_hash = get_etag_hash_from_query(self.object_list)
elif self.total_count:
sql, params, records = get_sql_with_params(xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields)
self.etag_hash = get_etag_hash_from_query(records, sql, params)
except ValueError as e:
raise ParseError(unicode(e))
except DataError as e:
raise ParseError(unicode(e))
def test_edit_updated_geopoint_cache(self):
query_args = {
'xform': self.xform,
'query': '{}',
'fields': '[]',
'count': True
}
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid.xml"
)
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
# query mongo for the _geopoint field
query_args['count'] = False
records = query_data(**query_args)
self.assertEqual(len(records), 1)
# submit the edited instance
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"
)
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
records = query_data(**query_args)
self.assertEqual(len(records), 1)
cached_geopoint = records[0][GEOLOCATION]
# the cached geopoint should equal the gps field
gps = records[0]['gps'].split(" ")
self.assertEqual(float(gps[0]), float(cached_geopoint[0]))
self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_query_filter_by_datetime_field(self, mock_time):
self._publish_transportation_form()
now = datetime(2014, 1, 1, tzinfo=utc)
times = [
now, now + timedelta(seconds=1), now + timedelta(seconds=2),
now + timedelta(seconds=3)
]
mock_time.side_effect = times
self._make_submissions()
atime = None
for i in self.xform.instances.all().order_by('-pk'):
i.date_created = times.pop()
i.save()
if atime is None:
atime = i.date_created.strftime(MONGO_STRFTIME)
# mongo $gt
data = [
i.get('_submission_time')
for i in query_data(self.xform,
query='{"_submission_time": {"$lt": "%s"}}' %
(atime),
fields='["_submission_time"]')
]
self.assertEqual(self.xform.instances.count(), 4)
self.assertEqual(len(data), 3)
self.assertNotIn(atime, data)
def set_object_list(self, query, fields, sort, start, limit,
is_public_request):
try:
enable_etag = True
if not is_public_request:
xform = self.get_object()
self.data_count = xform.num_of_submissions
enable_etag = self.data_count <\
SUBMISSION_RETRIEVAL_THRESHOLD
where, where_params = get_where_clause(query)
if where:
self.object_list = self.object_list.extra(where=where,
params=where_params)
if (start and limit or limit) and (not sort and not fields):
start = start if start is not None else 0
limit = limit if start is None or start == 0 else start + limit
self.object_list = filter_queryset_xform_meta_perms(
self.get_object(), self.request.user, self.object_list)
self.object_list = self.object_list[start:limit]
elif (sort or limit or start or fields) and not is_public_request:
try:
query = \
filter_queryset_xform_meta_perms_sql(self.get_object(),
self.request.user,
query)
self.object_list = query_data(
xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields,
json_only=not self.kwargs.get('format') == 'xml')
except NoRecordsPermission:
self.object_list = []
# ETags are Disabled for XForms with Submissions that surpass
# the configured SUBMISSION_RETRIEVAL_THRESHOLD setting
if enable_etag:
if isinstance(self.object_list, QuerySet):
self.etag_hash = get_etag_hash_from_query(self.object_list)
else:
sql, params, records = get_sql_with_params(
xform,
query=query,
sort=sort,
start_index=start,
limit=limit,
fields=fields)
self.etag_hash = get_etag_hash_from_query(
records, sql, params)
except ValueError as e:
raise ParseError(text(e))
except DataError as e:
raise ParseError(text(e))
def _query_data(self,
query='{}',
start=0,
limit=ParsedInstance.DEFAULT_LIMIT,
fields='[]',
count=False):
# query_data takes params as json strings
# so we dumps the fields dictionary
count_args = {
'xform': self.xform,
'query': query,
'start': self.start,
'end': self.end,
'fields': '[]',
'sort': '{}',
'count': True
}
count_object = list(query_data(**count_args))
record_count = count_object[0]["count"]
if record_count < 1:
raise NoRecordsFoundError("No records found for your query")
# if count was requested, return the count
if count:
return record_count
else:
query_args = {
'xform': self.xform,
'query': query,
'fields': fields,
'start': self.start,
'end': self.end,
# TODO: we might want to add this in for the user
# to sepcify a sort order
'sort': 'id',
'start_index': start,
'limit': limit,
'count': False
}
cursor = query_data(**query_args)
return cursor
def test_query_data_sort(self):
self._publish_transportation_form()
self._make_submissions()
latest = Instance.objects.filter(xform=self.xform).latest('pk').pk
oldest = Instance.objects.filter(xform=self.xform).first().pk
data = [i.get('_id') for i in query_data(
self.xform, sort='-_id')]
self.assertEqual(data[0], latest)
self.assertEqual(data[len(data) - 1], oldest)
# sort with a json field
data = [i.get('_id') for i in query_data(
self.xform, sort='{"_id": "-1"}')]
self.assertEqual(data[0], latest)
self.assertEqual(data[len(data) - 1], oldest)
# sort with a json field
data = [i.get('_id') for i in query_data(
self.xform, sort='{"_id": -1}')]
self.assertEqual(data[0], latest)
self.assertEqual(data[len(data) - 1], oldest)
def test_query_data_sort(self):
self._publish_transportation_form()
self._make_submissions()
latest = Instance.objects.filter(xform=self.xform).latest('pk').pk
oldest = Instance.objects.filter(xform=self.xform).first().pk
data = [i.get('_id') for i in query_data(
self.xform, sort='-_id')]
self.assertEqual(data[0], latest)
self.assertEqual(data[len(data) - 1], oldest)
# sort with a json field
data = [i.get('_id') for i in query_data(
self.xform, sort='{"_id": "-1"}')]
self.assertEqual(data[0], latest)
self.assertEqual(data[len(data) - 1], oldest)
# sort with a json field
data = [i.get('_id') for i in query_data(
self.xform, sort='{"_id": -1}')]
self.assertEqual(data[0], latest)
self.assertEqual(data[len(data) - 1], oldest)
def test_query_filter_by_integer(self):
self._publish_transportation_form()
self._make_submissions()
oldest = Instance.objects.filter(xform=self.xform).first().pk
data = [i.get('_id') for i in query_data(
self.xform, query='[{"_id": %s}]' % (oldest))]
self.assertEqual(len(data), 1)
self.assertEqual(data, [oldest])
# with fields
data = [i.get('_id') for i in query_data(
self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]')]
self.assertEqual(len(data), 1)
self.assertEqual(data, [oldest])
# mongo $gt
data = [i.get('_id') for i in query_data(
self.xform, query='{"_id": {"$gt": %s}}' % (oldest),
fields='["_id"]')]
self.assertEqual(self.xform.instances.count(), 4)
self.assertEqual(len(data), 3)
def _query_data(self, query='{}', start=0,
limit=ParsedInstance.DEFAULT_LIMIT,
fields='[]', count=False):
# query_data takes params as json strings
# so we dumps the fields dictionary
count_args = {
'xform': self.xform,
'query': query,
'start': self.start,
'end': self.end,
'fields': '[]',
'sort': '{}',
'count': True
}
count_object = list(query_data(**count_args))
record_count = count_object[0]["count"]
if record_count < 1:
raise NoRecordsFoundError("No records found for your query")
# if count was requested, return the count
if count:
return record_count
else:
query_args = {
'xform': self.xform,
'query': query,
'fields': fields,
'start': self.start,
'end': self.end,
# TODO: we might want to add this in for the user
# to sepcify a sort order
'sort': 'id',
'start_index': start,
'limit': limit,
'count': False
}
cursor = query_data(**query_args)
return cursor
def test_owner_can_delete(self):
# Test if Form owner can delete
# check record exist before delete and after delete
count = Instance.objects.filter(deleted_at=None).count()
instance = Instance.objects.filter(
xform=self.xform).latest('date_created')
self.assertEqual(instance.deleted_at, None)
# delete
params = {'id': instance.id}
response = self.client.post(self.delete_url, params)
self.assertEqual(response.status_code, 200)
self.assertEqual(
Instance.objects.filter(deleted_at=None).count(), count - 1)
instance = Instance.objects.get(id=instance.id)
self.assertTrue(isinstance(instance.deleted_at, datetime))
self.assertNotEqual(instance.deleted_at, None)
query = '{"_id": %s}' % instance.id
self.data_args.update({"query": query})
after = [r for r in query_data(**self.data_args)]
self.assertEqual(len(after), count - 1)
def set_object_list(
self, query, fields, sort, start, limit, is_public_request):
try:
if not is_public_request:
xform = self.get_object()
where, where_params = get_where_clause(query)
if where:
self.object_list = self.object_list.extra(where=where,
params=where_params)
if (start and limit or limit) and (not sort and not fields):
start = start if start is not None else 0
limit = limit if start is None or start == 0 else start + limit
self.object_list = filter_queryset_xform_meta_perms(
self.get_object(), self.request.user, self.object_list)
self.object_list = self.object_list[start:limit]
elif (sort or limit or start or fields) and not is_public_request:
try:
query = \
filter_queryset_xform_meta_perms_sql(self.get_object(),
self.request.user,
query)
self.object_list = query_data(xform, query=query,
sort=sort, start_index=start,
limit=limit, fields=fields)
except NoRecordsPermission:
self.object_list = []
if isinstance(self.object_list, QuerySet):
self.etag_hash = get_etag_hash_from_query(self.object_list)
else:
sql, params, records = get_sql_with_params(
xform, query=query, sort=sort, start_index=start,
limit=limit, fields=fields
)
self.etag_hash = get_etag_hash_from_query(records, sql, params)
except ValueError as e:
raise ParseError(text(e))
except DataError as e:
raise ParseError(text(e))
def test_query_filter_by_datetime_field(self, mock_time):
self._publish_transportation_form()
now = datetime(2014, 1, 1, tzinfo=utc)
times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2),
now + timedelta(seconds=3)]
mock_time.side_effect = times
self._make_submissions()
atime = None
for i in self.xform.instances.all().order_by('-pk'):
i.date_created = times.pop()
i.save()
if atime is None:
atime = i.date_created.strftime(MONGO_STRFTIME)
# mongo $gt
data = [i.get('_submission_time') for i in query_data(
self.xform, query='{"_submission_time": {"$lt": "%s"}}' % (atime),
fields='["_submission_time"]')]
self.assertEqual(self.xform.instances.count(), 4)
self.assertEqual(len(data), 3)
self.assertNotIn(atime, data)
def test_edited_submission(self):
"""
Test submissions that have been edited
"""
# Delete all previous instance history objects
InstanceHistory.objects.all().delete()
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml")
num_instances_history = InstanceHistory.objects.count()
num_instances = Instance.objects.count()
query_args = {
'xform': self.xform,
'query': '{}',
'fields': '[]',
'count': True
}
cursor = [r for r in query_data(**query_args)]
num_data_instances = cursor[0]['count']
# make first submission
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
self.assertEqual(Instance.objects.count(), num_instances + 1)
# Take initial instance from DB
initial_instance = self.xform.instances.first()
# check that '_last_edited' key is not in the json
self.assertIsNone(initial_instance.json.get(LAST_EDITED))
# no new record in instances history
self.assertEqual(InstanceHistory.objects.count(),
num_instances_history)
# check count of mongo instances after first submission
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# edited submission
xml_edit_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited.xml")
client = DigestClient()
client.set_authorization('bob', 'bob', 'Digest')
with catch_signal(process_submission) as handler:
self._make_submission(xml_edit_submission_file_path, client=client)
self.assertEqual(self.response.status_code, 201)
# we must have the same number of instances
self.assertEqual(Instance.objects.count(), num_instances + 1)
# should be a new record in instances history
self.assertEqual(InstanceHistory.objects.count(),
num_instances_history + 1)
instance_history_1 = InstanceHistory.objects.first()
edited_instance = self.xform.instances.first()
self.assertDictEqual(initial_instance.get_dict(),
instance_history_1.get_dict())
handler.assert_called_once_with(instance=edited_instance,
sender=Instance,
signal=ANY)
self.assertNotEqual(edited_instance.uuid, instance_history_1.uuid)
# check that instance history's submission_date is equal to instance's
# date_created - last_edited by default is null for an instance
self.assertEquals(edited_instance.date_created,
instance_history_1.submission_date)
# check that '_last_edited' key is not in the json
self.assertIn(LAST_EDITED, edited_instance.json)
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# make sure we edited the mongo db record and NOT added a new row
query_args['count'] = False
cursor = query_data(**query_args)
record = cursor[0]
with open(xml_edit_submission_file_path, "r") as f:
xml_str = f.read()
xml_str = clean_and_parse_xml(xml_str).toxml()
edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0]
self.assertEqual(record['name'], edited_name)
instance_before_second_edit = edited_instance
xml_edit_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml")
self._make_submission(xml_edit_submission_file_path)
cursor = query_data(**query_args)
record = cursor[0]
edited_instance = self.xform.instances.first()
instance_history_2 = InstanceHistory.objects.last()
self.assertEquals(instance_before_second_edit.last_edited,
instance_history_2.submission_date)
# check that '_last_edited' key is not in the json
self.assertIn(LAST_EDITED, edited_instance.json)
self.assertEqual(record['name'], 'Tom and Jerry')
self.assertEqual(InstanceHistory.objects.count(),
num_instances_history + 2)
# submitting original submission is treated as a duplicate
# does not add a new record
# does not change data
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 202)
self.assertEqual(Instance.objects.count(), num_instances + 1)
self.assertEqual(InstanceHistory.objects.count(),
num_instances_history + 2)
def generate_export(export_type, xform, export_id=None, options=None):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
show_choice_labels: boolean flag
language: language labels as in the XLSForm/XForm
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview, all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform, query=filter_query, start=start,
end=end, count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get(
"group_delimiter", DEFAULT_GROUP_DELIMITER
)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True
)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False
)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
include_reviews = options.get('include_reviews', False)
export_builder.INCLUDE_LABELS_ONLY = options.get(
'include_labels_only', False
)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get(
"repeat_index_tags", DEFAULT_INDEX_TAGS
)
export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels',
False)
export_builder.language = options.get('language')
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.INCLUDE_REVIEWS = include_reviews
export_builder.set_survey(xform.survey, xform,
include_reviews=include_reviews)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(
temp_file.name, records, username, id_string, filter_query,
start=start, end=end, dataview=dataview, xform=xform,
options=options, columns_with_hxl=columns_with_hxl,
total_records=total_records
)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
def generate_export(export_type,
xform,
export_id=None,
options=None,
retries=0):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(user__username__iexact=username,
id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview,
all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform,
query=filter_query,
start=start,
end=end,
count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get("group_delimiter",
DEFAULT_GROUP_DELIMITER)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
export_builder.INCLUDE_LABELS_ONLY = options.get('include_labels_only',
False)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get("repeat_index_tags",
DEFAULT_INDEX_TAGS)
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.set_survey(xform.survey, xform)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(temp_file.name,
records,
username,
id_string,
filter_query,
start=start,
end=end,
dataview=dataview,
xform=xform,
options=options,
columns_with_hxl=columns_with_hxl,
total_records=total_records)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
def generate_external_export(export_type,
username,
id_string,
export_id=None,
options=None,
xform=None):
"""
Generates external export using ONA data through an external service.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
data_id: instance id
query: filter_query for custom queries
meta: metadata associated with external export
token: authentication key required by external service
"""
data_id = options.get("data_id")
filter_query = options.get("query")
meta = options.get("meta")
token = options.get("token")
if xform is None:
xform = XForm.objects.get(user__username__iexact=username,
id_string__iexact=id_string)
user = User.objects.get(username=username)
server, name = _get_server_from_metadata(xform, meta, token)
# dissect the url
parsed_url = urlparse(server)
token = parsed_url.path[5:]
ser = parsed_url.scheme + '://' + parsed_url.netloc
# Get single submission data
if data_id:
inst = Instance.objects.filter(xform__user=user,
xform__id_string=id_string,
deleted_at=None,
pk=data_id)
instances = [inst[0].json if inst else {}]
else:
instances = query_data(xform, query=filter_query)
records = _get_records(instances)
status_code = 0
if records and server:
try:
client = Client(ser)
response = client.xls.create(token, json.dumps(records))
if hasattr(client.xls.conn, 'last_response'):
status_code = client.xls.conn.last_response.status_code
except Exception as e:
raise J2XException(
u"J2X client could not generate report. Server -> {0},"
u" Error-> {1}".format(server, e))
else:
if not server:
raise J2XException(u"External server not set")
elif not records:
raise J2XException(
u"No record to export. Form -> {0}".format(id_string))
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export_options = get_export_options(options)
export = Export.objects.create(xform=xform,
export_type=export_type,
options=export_options)
export.export_url = response
if status_code == 201:
export.internal_status = Export.SUCCESSFUL
export.filename = name + '-' + response[5:] if name else response[5:]
export.export_url = ser + response
else:
export.internal_status = Export.FAILED
export.save()
return export
def test_generate_filtered_attachments_zip_export(self):
"""Test media zip file export filters attachments"""
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(os.path.dirname(api_tests.__file__), 'fixtures',
'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {
"extension": Export.ZIP_EXPORT,
"query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'
}
filter_query = options.get("query")
instance_ids = query_data(self.xform,
fields='["_id"]',
query=filter_query)
export = generate_attachments_zip_export(Export.ZIP_EXPORT,
self.user.username,
self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
filtered_attachments = Attachment.objects.filter(
instance__xform_id=self.xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
self.assertNotEqual(Attachment.objects.count(),
filtered_attachments.count())
for a in filtered_attachments:
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
# export with no query
options.pop('query')
export1 = generate_attachments_zip_export(Export.ZIP_EXPORT,
self.user.username,
self.xform.id_string, None,
options)
self.assertTrue(export1.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
def _get_data(self):
cursor = query_data(**self.data_args)
records = list(record for record in cursor)
return records
def generate_external_export(export_type, username, id_string, export_id=None,
options=None, xform=None):
"""
Generates external export using ONA data through an external service.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
data_id: instance id
query: filter_query for custom queries
meta: metadata associated with external export
token: authentication key required by external service
"""
data_id = options.get("data_id")
filter_query = options.get("query")
meta = options.get("meta")
token = options.get("token")
if xform is None:
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
user = User.objects.get(username=username)
server, name = _get_server_from_metadata(xform, meta, token)
# dissect the url
parsed_url = urlparse(server)
token = parsed_url.path[5:]
ser = parsed_url.scheme + '://' + parsed_url.netloc
# Get single submission data
if data_id:
inst = Instance.objects.filter(xform__user=user,
xform__id_string=id_string,
deleted_at=None,
pk=data_id)
instances = [inst[0].json if inst else {}]
else:
instances = query_data(xform, query=filter_query)
records = _get_records(instances)
status_code = 0
if records and server:
try:
client = Client(ser)
response = client.xls.create(token, json.dumps(records))
if hasattr(client.xls.conn, 'last_response'):
status_code = client.xls.conn.last_response.status_code
except Exception as e:
raise J2XException(
u"J2X client could not generate report. Server -> {0},"
u" Error-> {1}".format(server, e)
)
else:
if not server:
raise J2XException(u"External server not set")
elif not records:
raise J2XException(
u"No record to export. Form -> {0}".format(id_string)
)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export_options = get_export_options(options)
export = Export.objects.create(xform=xform,
export_type=export_type,
options=export_options)
export.export_url = response
if status_code == 201:
export.internal_status = Export.SUCCESSFUL
export.filename = name + '-' + response[5:] if name else response[5:]
export.export_url = ser + response
else:
export.internal_status = Export.FAILED
export.save()
return export
def generate_attachments_zip_export(export_type,
username,
id_string,
export_id=None,
options=None,
xform=None):
"""
Generates zip export of attachments.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
export_type = options.get("extension", export_type)
filter_query = options.get("query")
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
attachments = Attachment.objects.filter(
instance_id__in=[
rec.get('_id') for rec in dataview.query_data(
dataview, all_data=True, filter_query=filter_query)
],
instance__deleted_at__isnull=True)
else:
instance_ids = query_data(xform, fields='["_id"]', query=filter_query)
attachments = Attachment.objects.filter(
instance__deleted_at__isnull=True)
if xform.is_merged_dataset:
attachments = attachments.filter(instance__xform_id__in=[
i for i in xform.mergedxform.xforms.filter(
deleted_at__isnull=True).values_list('id', flat=True)
]).filter(instance_id__in=[i_id['_id'] for i_id in instance_ids])
else:
attachments = attachments.filter(
instance__xform_id=xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
filename = "%s_%s.%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"),
export_type.lower())
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
try:
temp_file = builtins.open(zip_file.name, 'rb')
filename = default_storage.save(file_path,
File(temp_file, file_path))
finally:
temp_file.close()
finally:
if zip_file:
zip_file.close()
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir, export.filename = os.path.split(filename)
export.internal_status = Export.SUCCESSFUL
export.save()
return export
def test_edited_submission_require_auth(self):
"""
Test submissions that have been edited
"""
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid.xml"
)
# require authentication
self.user.profile.require_auth = True
self.user.profile.save()
num_instances_history = InstanceHistory.objects.count()
num_instances = Instance.objects.count()
query_args = {
'xform': self.xform,
'query': '{}',
'fields': '[]',
'count': True
}
cursor = query_data(**query_args)
num_data_instances = cursor[0]['count']
# make first submission
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
self.assertEqual(Instance.objects.count(), num_instances + 1)
# no new record in instances history
self.assertEqual(
InstanceHistory.objects.count(), num_instances_history)
# check count of mongo instances after first submission
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# create a new user
alice = self._create_user('alice', 'alice')
UserProfile.objects.create(user=alice)
auth = DigestAuth('alice', 'alice')
# edited submission
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"
)
self._make_submission(xml_submission_file_path, auth=auth)
self.assertEqual(self.response.status_code, 403)
# assign report perms to user
assign_perm('report_xform', alice, self.xform)
assign_perm('logger.change_xform', alice, self.xform)
self._make_submission(xml_submission_file_path, auth=auth)
self.assertEqual(self.response.status_code, 201)
# we must have the same number of instances
self.assertEqual(Instance.objects.count(), num_instances + 1)
# should be a new record in instances history
self.assertEqual(
InstanceHistory.objects.count(), num_instances_history + 1)
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# make sure we edited the mongo db record and NOT added a new row
query_args['count'] = False
cursor = query_data(**query_args)
record = cursor[0]
with open(xml_submission_file_path, "r") as f:
xml_str = f.read()
xml_str = clean_and_parse_xml(xml_str).toxml()
edited_name = re.match(r"^.+?<name>(.+?)</name>", xml_str).groups()[0]
self.assertEqual(record['name'], edited_name)
def test_generate_filtered_attachments_zip_export(self):
"""Test media zip file export filters attachments"""
filenames = [
'OSMWay234134797.osm',
'OSMWay34298972.osm',
]
osm_fixtures_dir = os.path.realpath(
os.path.join(
os.path.dirname(api_tests.__file__), 'fixtures', 'osm'))
paths = [
os.path.join(osm_fixtures_dir, filename) for filename in filenames
]
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_file_and_set_xform(xlsform_path)
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
count = Attachment.objects.filter(extension='osm').count()
self._make_submission_w_attachment(submission_path, paths)
self._make_submission_w_attachment(submission_path, paths)
self.assertTrue(
Attachment.objects.filter(extension='osm').count() > count)
options = {
"extension": Export.ZIP_EXPORT,
"query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'}
filter_query = options.get("query")
instance_ids = query_data(
self.xform, fields='["_id"]', query=filter_query)
export = generate_attachments_zip_export(
Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None,
options)
self.assertTrue(export.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
filtered_attachments = Attachment.objects.filter(
instance__xform_id=self.xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
self.assertNotEqual(
Attachment.objects.count(), filtered_attachments.count())
for a in filtered_attachments:
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
# export with no query
options.pop('query')
export1 = generate_attachments_zip_export(
Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None,
options)
self.assertTrue(export1.is_successful)
temp_dir = tempfile.mkdtemp()
zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r")
zip_file.extractall(temp_dir)
zip_file.close()
for a in Attachment.objects.all():
self.assertTrue(
os.path.exists(os.path.join(temp_dir, a.media_file.name)))
shutil.rmtree(temp_dir)
def test_edited_submission(self):
"""
Test submissions that have been edited
"""
# Delete all previous instance history objects
InstanceHistory.objects.all().delete()
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid.xml"
)
num_instances_history = InstanceHistory.objects.count()
num_instances = Instance.objects.count()
query_args = {
'xform': self.xform,
'query': '{}',
'fields': '[]',
'count': True
}
cursor = [r for r in query_data(**query_args)]
num_data_instances = cursor[0]['count']
# make first submission
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
self.assertEqual(Instance.objects.count(), num_instances + 1)
# Take initial instance from DB
initial_instance = self.xform.instances.first()
# check that '_last_edited' key is not in the json
self.assertIsNone(initial_instance.json.get(LAST_EDITED))
# no new record in instances history
self.assertEqual(
InstanceHistory.objects.count(), num_instances_history)
# check count of mongo instances after first submission
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# edited submission
xml_edit_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited.xml"
)
client = DigestClient()
client.set_authorization('bob', 'bob', 'Digest')
with catch_signal(process_submission) as handler:
self._make_submission(xml_edit_submission_file_path, client=client)
self.assertEqual(self.response.status_code, 201)
# we must have the same number of instances
self.assertEqual(Instance.objects.count(), num_instances + 1)
# should be a new record in instances history
self.assertEqual(
InstanceHistory.objects.count(), num_instances_history + 1)
instance_history_1 = InstanceHistory.objects.first()
edited_instance = self.xform.instances.first()
self.assertDictEqual(initial_instance.get_dict(),
instance_history_1.get_dict())
handler.assert_called_once_with(instance=edited_instance,
sender=Instance, signal=ANY)
self.assertNotEqual(edited_instance.uuid, instance_history_1.uuid)
# check that instance history's submission_date is equal to instance's
# date_created - last_edited by default is null for an instance
self.assertEquals(edited_instance.date_created,
instance_history_1.submission_date)
# check that '_last_edited' key is not in the json
self.assertIn(LAST_EDITED, edited_instance.json)
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# make sure we edited the mongo db record and NOT added a new row
query_args['count'] = False
cursor = query_data(**query_args)
record = cursor[0]
with open(xml_edit_submission_file_path, "r") as f:
xml_str = f.read()
xml_str = clean_and_parse_xml(xml_str).toxml()
edited_name = re.match(r"^.+?<name>(.+?)</name>", xml_str).groups()[0]
self.assertEqual(record['name'], edited_name)
instance_before_second_edit = edited_instance
xml_edit_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..", "fixtures", "tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml"
)
self._make_submission(xml_edit_submission_file_path)
cursor = query_data(**query_args)
record = cursor[0]
edited_instance = self.xform.instances.first()
instance_history_2 = InstanceHistory.objects.last()
self.assertEquals(instance_before_second_edit.last_edited,
instance_history_2.submission_date)
# check that '_last_edited' key is not in the json
self.assertIn(LAST_EDITED, edited_instance.json)
self.assertEqual(record['name'], 'Tom and Jerry')
self.assertEqual(
InstanceHistory.objects.count(), num_instances_history + 2)
# submitting original submission is treated as a duplicate
# does not add a new record
# does not change data
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 202)
self.assertEqual(Instance.objects.count(), num_instances + 1)
self.assertEqual(
InstanceHistory.objects.count(), num_instances_history + 2)
def test_edited_submission_require_auth(self):
"""
Test submissions that have been edited
"""
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml")
# require authentication
self.user.profile.require_auth = True
self.user.profile.save()
num_instances_history = InstanceHistory.objects.count()
num_instances = Instance.objects.count()
query_args = {
'xform': self.xform,
'query': '{}',
'fields': '[]',
'count': True
}
cursor = query_data(**query_args)
num_data_instances = cursor[0]['count']
# make first submission
self._make_submission(xml_submission_file_path)
self.assertEqual(self.response.status_code, 201)
self.assertEqual(Instance.objects.count(), num_instances + 1)
# no new record in instances history
self.assertEqual(InstanceHistory.objects.count(),
num_instances_history)
# check count of mongo instances after first submission
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# create a new user
alice = self._create_user('alice', 'alice')
UserProfile.objects.create(user=alice)
auth = DigestAuth('alice', 'alice')
# edited submission
xml_submission_file_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "fixtures",
"tutorial", "instances",
"tutorial_2012-06-27_11-27-53_w_uuid_edited.xml")
self._make_submission(xml_submission_file_path, auth=auth)
self.assertEqual(self.response.status_code, 403)
# assign report perms to user
assign_perm('report_xform', alice, self.xform)
assign_perm('logger.change_xform', alice, self.xform)
self._make_submission(xml_submission_file_path, auth=auth)
self.assertEqual(self.response.status_code, 201)
# we must have the same number of instances
self.assertEqual(Instance.objects.count(), num_instances + 1)
# should be a new record in instances history
self.assertEqual(InstanceHistory.objects.count(),
num_instances_history + 1)
cursor = query_data(**query_args)
self.assertEqual(cursor[0]['count'], num_data_instances + 1)
# make sure we edited the mongo db record and NOT added a new row
query_args['count'] = False
cursor = query_data(**query_args)
record = cursor[0]
with open(xml_submission_file_path, "r") as f:
xml_str = f.read()
xml_str = clean_and_parse_xml(xml_str).toxml()
edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0]
self.assertEqual(record['name'], edited_name)
def generate_attachments_zip_export(export_type, username, id_string,
export_id=None, options=None,
xform=None):
"""
Generates zip export of attachments.
param: export_type
params: username: logged in username
params: id_string: xform id_string
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
ext: File extension of the generated export
"""
export_type = options.get("extension", export_type)
filter_query = options.get("query")
if xform is None:
xform = XForm.objects.get(user__username=username, id_string=id_string)
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
attachments = Attachment.objects.filter(
instance_id__in=[
rec.get('_id')
for rec in dataview.query_data(
dataview, all_data=True, filter_query=filter_query)],
instance__deleted_at__isnull=True)
else:
instance_ids = query_data(xform, fields='["_id"]', query=filter_query)
attachments = Attachment.objects.filter(
instance__deleted_at__isnull=True)
if xform.is_merged_dataset:
attachments = attachments.filter(
instance__xform_id__in=[
i for i in xform.mergedxform.xforms.filter(
deleted_at__isnull=True).values_list(
'id', flat=True)]).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
else:
attachments = attachments.filter(
instance__xform_id=xform.pk).filter(
instance_id__in=[i_id['_id'] for i_id in instance_ids])
filename = "%s_%s.%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"),
export_type.lower())
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
zip_file = None
try:
zip_file = create_attachments_zipfile(attachments)
try:
temp_file = builtins.open(zip_file.name, 'rb')
filename = default_storage.save(
file_path,
File(temp_file, file_path))
finally:
temp_file.close()
finally:
if zip_file:
zip_file.close()
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir, export.filename = os.path.split(filename)
export.internal_status = Export.SUCCESSFUL
export.save()
return export
