def should_create_new_export(xform, export_type, options, request=None):
"""
Function that determines whether to create a new export.
param: xform
param: export_type
param: options: additional parameters required for the lookup.
remove_group_name: boolean flag
group_delimiter: "/" or "." with "/" as the default
split_select_multiples: boolean flag
binary_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
params: request: Get params are used to determine if new export is required
"""
split_select_multiples = options.get('split_select_multiples', True)
if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False):
return True
if (request and (frozenset(list(request.GET)) &
frozenset(['start', 'end', 'data_id']))) or\
not split_select_multiples:
return True
export_options_kwargs = get_export_options_query_kwargs(options)
export_query = Export.objects.filter(xform=xform,
export_type=export_type,
**export_options_kwargs)
if options.get(EXPORT_QUERY_KEY) is None:
export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY)
if export_query.count() == 0 or\
Export.exports_outdated(xform, export_type, options=options):
return True
return False
def create_export_object(xform, export_type, options):
"""
Return an export object that has not been saved to the database.
"""
export_options = get_export_options(options)
return Export(xform=xform, export_type=export_type, options=export_options,
created_on=timezone.now())
def should_create_new_export(xform, export_type):
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
if Export.objects.filter(
xform=xform, export_type=export_type).count() == 0\
or Export.exports_outdated(xform, export_type=export_type):
return True
return False
def should_create_new_export(xform, export_type):
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
if Export.objects.filter(
xform=xform, export_type=export_type).count() == 0\
or Export.exports_outdated(xform, export_type=export_type):
return True
return False
def test_last_submission_time_on_export(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
generate_export(
Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string)
num_exports = Export.objects.filter(
xform=self.xform, export_type=Export.XLS_EXPORT).count()
# check that our function knows there are no more submissions
self.assertFalse(
Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
sleep(1)
# force new last submission date on xform
last_submission = self.xform.instances.order_by('-date_created')[0]
last_submission.date_created += datetime.timedelta(hours=1)
last_submission.save()
# check that our function knows data has changed
self.assertTrue(
Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
# check that requesting list url will generate a new export
export_list_url = reverse(export_list, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': Export.XLS_EXPORT
})
self.client.get(export_list_url)
self.assertEqual(
Export.objects.filter(xform=self.xform,
export_type=Export.XLS_EXPORT).count(),
num_exports + 1)
# make sure another export type causes auto-generation
num_exports = Export.objects.filter(
xform=self.xform, export_type=Export.CSV_EXPORT).count()
export_list_url = reverse(export_list, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': Export.CSV_EXPORT
})
self.client.get(export_list_url)
self.assertEqual(
Export.objects.filter(xform=self.xform,
export_type=Export.CSV_EXPORT).count(),
num_exports + 1)
def test_retrieving_pending_export(self):
self._create_user_and_login()
self._publish_transportation_form()
export = Export(xform=self.xform,
export_type=Export.CSV_EXPORT,
options={},
task_id="abcsde")
export.save()
test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {})
self.assertEqual(export, test_export)
test_export = check_pending_export(self.xform, Export.XLS_EXPORT, {})
self.assertIsNone(test_export)
export.created_on = export.created_on - timedelta(minutes=6)
export.save()
test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {})
self.assertIsNone(test_export)
def _create_old_export(self, xform, export_type, options, filename=None):
options = OrderedDict(sorted(options.items()))
Export(xform=xform,
export_type=export_type,
options=options,
filename=filename,
internal_status=Export.SUCCESSFUL).save()
self.export = Export.objects.filter(xform=xform,
export_type=export_type)[0]
def test_exports_outdated_doesnt_consider_failed_exports(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create a bad export
export = Export.objects.create(
xform=self.xform, export_type=Export.XLS_EXPORT,
internal_status=Export.FAILED)
self.assertTrue(
Export.exports_outdated(self.xform, export.export_type))
def test_exports_outdated_considers_pending_exports(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create a pending export
export = Export.objects.create(
xform=self.xform, export_type=Export.XLS_EXPORT,
internal_status=Export.PENDING)
self.assertFalse(
Export.exports_outdated(self.xform, export.export_type))
def _create_old_export(self, xform, export_type, options, filename=None):
options = OrderedDict(sorted(options.items()))
Export(xform=xform,
export_type=export_type,
options=options,
filename=filename,
internal_status=Export.SUCCESSFUL).save()
# pylint: disable=attribute-defined-outside-init
self.export = Export.objects.filter(xform=xform,
export_type=export_type)[0]
def test_last_submission_time_empty(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
# set time of last submission to None
export.time_of_last_submission = None
export.save()
self.assertTrue(Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
def test_export_not_found(self):
export_type = "csv"
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._publish_transportation_form_and_submit_instance()
self._create_old_export(self.xform, export_type, options)
export = Export(
xform=self.xform, export_type=export_type, options=options)
export.save()
export_id = export.pk
export.delete()
export = generate_export(export_type, self.xform, export_id, options)
self.assertIsNotNone(export)
self.assertTrue(export.is_successful)
self.assertNotEqual(export_id, export.pk)
def test_retrieving_pending_export(self):
self._create_user_and_login()
self._publish_transportation_form()
export = Export(
xform=self.xform,
export_type=Export.CSV_EXPORT,
options={},
task_id="abcsde")
export.save()
test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {})
self.assertEqual(export, test_export)
test_export = check_pending_export(self.xform, Export.XLS_EXPORT, {})
self.assertIsNone(test_export)
export.created_on = export.created_on - timedelta(minutes=6)
export.save()
test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {})
self.assertIsNone(test_export)
def should_create_new_export(xform,
export_type,
options,
request=None):
"""
Function that determines whether to create a new export.
param: xform
param: export_type
param: options: additional parameters required for the lookup.
remove_group_name: boolean flag
group_delimiter: "/" or "." with "/" as the default
split_select_multiples: boolean flag
binary_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
params: request: Get params are used to determine if new export is required
"""
split_select_multiples = options.get('split_select_multiples', True)
if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False):
return True
if (request and (frozenset(list(request.GET)) &
frozenset(['start', 'end', 'data_id']))) or\
not split_select_multiples:
return True
export_options_kwargs = get_export_options_query_kwargs(options)
export_query = Export.objects.filter(
xform=xform,
export_type=export_type,
**export_options_kwargs
)
if options.get(EXPORT_QUERY_KEY) is None:
export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY)
if export_query.count() == 0 or\
Export.exports_outdated(xform, export_type, options=options):
return True
return False
def test_export_not_found(self):
export_type = "csv"
options = {
"group_delimiter": "/",
"remove_group_name": False,
"split_select_multiples": True
}
self._publish_transportation_form_and_submit_instance()
self._create_old_export(self.xform, export_type, options)
export = Export(
xform=self.xform, export_type=export_type, options=options)
export.save()
export_id = export.pk
export.delete()
export = generate_export(export_type, self.xform, export_id, options)
self.assertIsNotNone(export)
self.assertTrue(export.is_successful)
self.assertNotEqual(export_id, export.pk)
def generate_export(export_type,
xform,
export_id=None,
options=None,
retries=0):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(user__username__iexact=username,
id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview,
all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform,
query=filter_query,
start=start,
end=end,
count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get("group_delimiter",
DEFAULT_GROUP_DELIMITER)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
export_builder.INCLUDE_LABELS_ONLY = options.get('include_labels_only',
False)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get("repeat_index_tags",
DEFAULT_INDEX_TAGS)
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.set_survey(xform.survey, xform)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(temp_file.name,
records,
username,
id_string,
filter_query,
start=start,
end=end,
dataview=dataview,
xform=xform,
options=options,
columns_with_hxl=columns_with_hxl,
total_records=total_records)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
def generate_export(export_type, extension, username, id_string,
export_id=None, filter_query=None, group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
"""
Create appropriate export object given the export type
"""
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
}
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(
temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
return export
def create_export_object(xform, export_type, options):
export_options = get_export_options(options)
return Export(xform=xform,
export_type=export_type,
options=export_options,
created_on=timezone.now())
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
show_label=False):
"""
Create appropriate export object given the export type
"""
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.DB_EXPORT: 'to_postgres_sav',
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.SHOW_LABEL = show_label
export_builder.set_survey(xform.data_dictionary().survey)
print "xform.data_dictionary().survey"
print xform.data_dictionary().survey
export_builder.FORM_OWNER = xform.user.username
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# start: original code commented out
# dont persist exports that have a filter
# if filter_query is None:
# export.save()
# end: original code commented out
export.save()
return export
def create_export_object(xform, export_type, options):
export_options = get_export_options(options)
return Export(xform=xform, export_type=export_type, options=export_options)
def should_create_new_export(xform, export_type):
if (not Export.objects.filter(xform=xform,
export_type=export_type).exists()
or Export.exports_outdated(xform, export_type=export_type)):
return True
return False
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
start=None,
end=None,
remove_group_name=False):
"""
Create appropriate export object given the export type
"""
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__iexact=id_string)
records = ParsedInstance.query_data(xform,
query=filter_query,
start=start,
end=end)
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = remove_group_name
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(temp_file.name,
records,
username,
id_string,
filter_query,
start=start,
end=end)
except NoRecordsFoundError:
pass
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None and start is None and end is None:
export.save()
return export
def should_create_new_export(xform, export_type):
if Export.objects.filter(
xform=xform, export_type=export_type).count() == 0\
or Export.exports_outdated(xform, export_type=export_type):
return True
return False
def generate_export(export_type, extension, username, id_string,
export_id=None, filter_query=None, group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
sync_to_gsuit=False, user=None):
"""
Create appropriate export object given the export type
"""
time.sleep(5)
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.ANALYSER_EXPORT: 'to_analyser_export'
}
xform = XForm.objects.get(
user__username__iexact=username, id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
__version__ = "0"
try:
__version__ = filter_query['$and'][0]['__version__']
except Exception as e:
print(str(e))
if __version__:
survey = build_survey_from_history(xform, __version__)
if not survey:
export_builder.set_survey(xform.data_dictionary().survey)
else:
export_builder.set_survey(survey)
else:
export_builder.set_survey(xform.data_dictionary().survey)
prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string))
temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(
temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
if export_type == Export.ANALYSER_EXPORT:
# Analyser exports should be distinguished by more than just their file extension.
basename= '{}_ANALYSER_{}'.format(id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
print sync_to_gsuit, 'file_url--------->', temp_file, filter_query
try:
if sync_to_gsuit == True and '__version__' not in filter_query['$and'][0]:
if not os.path.exists("media/forms/"):
os.makedirs("media/forms/")
temporarylocation="media/forms/submissions_{}.xls".format(id_string)
import shutil
shutil.copy(temp_file.name, temporarylocation)
fxf_form = FieldSightXF.objects.get(pk=filter_query['$and'][0]['fs_project_uuid'])
upload_to_drive(temporarylocation, str(fxf_form.id) + '_' +id_string, None, fxf_form.project, user)
os.remove(temporarylocation)
except Exception as e:
print e.__dict__
# get or create export object
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
dir_name, basename = os.path.split(export_filename)
temp_file.close()
if export_id:
export = Export.objects.get(id=export_id)
else:
fsxf = filter_query.values()[0]
# print("fsxf", fsxf)
export = Export(xform=xform, export_type=export_type, fsxf_id=fsxf)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
export.save()
return export
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
"""
Create appropriate export object given the export type
"""
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.ANALYSER_EXPORT: 'to_analyser_export'
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
prefix = slugify('{}_export__{}__{}'.format(export_type, username,
id_string))
temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
if export_type == Export.ANALYSER_EXPORT:
# Analyser exports should be distinguished by more than just their file extension.
basename = '{}_ANALYSER_{}'.format(
id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
return export
def generate_export(export_type, xform, export_id=None, options=None):
"""
Create appropriate export object given the export type.
param: export_type
param: xform
params: export_id: ID of export object associated with the request
param: options: additional parameters required for the lookup.
binary_select_multiples: boolean flag
end: end offset
ext: export extension type
dataview_pk: dataview pk
group_delimiter: "/" or "."
query: filter_query for custom queries
remove_group_name: boolean flag
split_select_multiples: boolean flag
index_tag: ('[', ']') or ('_', '_')
show_choice_labels: boolean flag
language: language labels as in the XLSForm/XForm
"""
username = xform.user.username
id_string = xform.id_string
end = options.get("end")
extension = options.get("extension", export_type)
filter_query = options.get("query")
remove_group_name = options.get("remove_group_name", False)
start = options.get("start")
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets',
}
if xform is None:
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
dataview = None
if options.get("dataview_pk"):
dataview = DataView.objects.get(pk=options.get("dataview_pk"))
records = dataview.query_data(dataview, all_data=True,
filter_query=filter_query)
total_records = dataview.query_data(dataview,
count=True)[0].get('count')
else:
records = query_data(xform, query=filter_query, start=start, end=end)
if filter_query:
total_records = query_data(xform, query=filter_query, start=start,
end=end, count=True)[0].get('count')
else:
total_records = xform.num_of_submissions
if isinstance(records, QuerySet):
records = records.iterator()
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = True \
if export_type == Export.SAV_ZIP_EXPORT else remove_group_name
export_builder.GROUP_DELIMITER = options.get(
"group_delimiter", DEFAULT_GROUP_DELIMITER
)
export_builder.SPLIT_SELECT_MULTIPLES = options.get(
"split_select_multiples", True
)
export_builder.BINARY_SELECT_MULTIPLES = options.get(
"binary_select_multiples", False
)
export_builder.INCLUDE_LABELS = options.get('include_labels', False)
include_reviews = options.get('include_reviews', False)
export_builder.INCLUDE_LABELS_ONLY = options.get(
'include_labels_only', False
)
export_builder.INCLUDE_HXL = options.get('include_hxl', False)
export_builder.INCLUDE_IMAGES \
= options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT)
export_builder.VALUE_SELECT_MULTIPLES = options.get(
'value_select_multiples', False)
export_builder.REPEAT_INDEX_TAGS = options.get(
"repeat_index_tags", DEFAULT_INDEX_TAGS
)
export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels',
False)
export_builder.language = options.get('language')
# 'win_excel_utf8' is only relevant for CSV exports
if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT:
del options['win_excel_utf8']
export_builder.INCLUDE_REVIEWS = include_reviews
export_builder.set_survey(xform.survey, xform,
include_reviews=include_reviews)
temp_file = NamedTemporaryFile(suffix=("." + extension))
columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl(
xform.survey_elements)
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(
temp_file.name, records, username, id_string, filter_query,
start=start, end=end, dataview=dataview, xform=xform,
options=options, columns_with_hxl=columns_with_hxl,
total_records=total_records
)
except NoRecordsFoundError:
pass
except SPSSIOError as e:
export = get_or_create_export(export_id, xform, export_type, options)
export.error_message = str(e)
export.internal_status = Export.FAILED
export.save()
report_exception("SAV Export Failure", e, sys.exc_info())
return export
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f"))
if remove_group_name:
# add 'remove group name' flag to filename
basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG)
if dataview:
basename = "{}-{}".format(basename, DATAVIEW_EXPORT)
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = default_storage.save(file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
export = get_or_create_export(export_id, xform, export_type, options)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# do not persist exports that have a filter
# Get URL of the exported sheet.
if export_type == Export.GOOGLE_SHEETS_EXPORT:
export.export_url = export_builder.url
# if we should create a new export is true, we should not save it
if start is None and end is None:
export.save()
return export
def should_create_new_export(xform, export_type):
if Export.objects.filter(
xform=xform, export_type=export_type).count() == 0\
or Export.exports_outdated(xform, export_type=export_type):
return True
return False
def _create_old_export(self, xform, export_type, options):
Export(xform=xform, export_type=export_type, options=options).save()
self.export = Export.objects.filter(xform=xform,
export_type=export_type)
