def should_create_new_export(xform, export_type, options, request=None): """ Function that determines whether to create a new export. param: xform param: export_type param: options: additional parameters required for the lookup. remove_group_name: boolean flag group_delimiter: "/" or "." with "/" as the default split_select_multiples: boolean flag binary_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') params: request: Get params are used to determine if new export is required """ split_select_multiples = options.get('split_select_multiples', True) if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False): return True if (request and (frozenset(list(request.GET)) & frozenset(['start', 'end', 'data_id']))) or\ not split_select_multiples: return True export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter(xform=xform, export_type=export_type, **export_options_kwargs) if options.get(EXPORT_QUERY_KEY) is None: export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY) if export_query.count() == 0 or\ Export.exports_outdated(xform, export_type, options=options): return True return False
def create_export_object(xform, export_type, options): """ Return an export object that has not been saved to the database. """ export_options = get_export_options(options) return Export(xform=xform, export_type=export_type, options=export_options, created_on=timezone.now())
def should_create_new_export(xform, export_type): # TODO resolve circular import from onadata.apps.viewer.models.export import Export if Export.objects.filter( xform=xform, export_type=export_type).count() == 0\ or Export.exports_outdated(xform, export_type=export_type): return True return False
def test_last_submission_time_on_export(self): self._publish_transportation_form() self._submit_transport_instance() # create export generate_export( Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) num_exports = Export.objects.filter( xform=self.xform, export_type=Export.XLS_EXPORT).count() # check that our function knows there are no more submissions self.assertFalse( Export.exports_outdated(xform=self.xform, export_type=Export.XLS_EXPORT)) sleep(1) # force new last submission date on xform last_submission = self.xform.instances.order_by('-date_created')[0] last_submission.date_created += datetime.timedelta(hours=1) last_submission.save() # check that our function knows data has changed self.assertTrue( Export.exports_outdated(xform=self.xform, export_type=Export.XLS_EXPORT)) # check that requesting list url will generate a new export export_list_url = reverse(export_list, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': Export.XLS_EXPORT }) self.client.get(export_list_url) self.assertEqual( Export.objects.filter(xform=self.xform, export_type=Export.XLS_EXPORT).count(), num_exports + 1) # make sure another export type causes auto-generation num_exports = Export.objects.filter( xform=self.xform, export_type=Export.CSV_EXPORT).count() export_list_url = reverse(export_list, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': Export.CSV_EXPORT }) self.client.get(export_list_url) self.assertEqual( Export.objects.filter(xform=self.xform, export_type=Export.CSV_EXPORT).count(), num_exports + 1)
def test_retrieving_pending_export(self): self._create_user_and_login() self._publish_transportation_form() export = Export(xform=self.xform, export_type=Export.CSV_EXPORT, options={}, task_id="abcsde") export.save() test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {}) self.assertEqual(export, test_export) test_export = check_pending_export(self.xform, Export.XLS_EXPORT, {}) self.assertIsNone(test_export) export.created_on = export.created_on - timedelta(minutes=6) export.save() test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {}) self.assertIsNone(test_export)
def _create_old_export(self, xform, export_type, options, filename=None): options = OrderedDict(sorted(options.items())) Export(xform=xform, export_type=export_type, options=options, filename=filename, internal_status=Export.SUCCESSFUL).save() self.export = Export.objects.filter(xform=xform, export_type=export_type)[0]
def test_exports_outdated_doesnt_consider_failed_exports(self): self._publish_transportation_form() self._submit_transport_instance() # create a bad export export = Export.objects.create( xform=self.xform, export_type=Export.XLS_EXPORT, internal_status=Export.FAILED) self.assertTrue( Export.exports_outdated(self.xform, export.export_type))
def test_exports_outdated_considers_pending_exports(self): self._publish_transportation_form() self._submit_transport_instance() # create a pending export export = Export.objects.create( xform=self.xform, export_type=Export.XLS_EXPORT, internal_status=Export.PENDING) self.assertFalse( Export.exports_outdated(self.xform, export.export_type))
def _create_old_export(self, xform, export_type, options, filename=None): options = OrderedDict(sorted(options.items())) Export(xform=xform, export_type=export_type, options=options, filename=filename, internal_status=Export.SUCCESSFUL).save() # pylint: disable=attribute-defined-outside-init self.export = Export.objects.filter(xform=xform, export_type=export_type)[0]
def test_last_submission_time_empty(self): self._publish_transportation_form() self._submit_transport_instance() # create export export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) # set time of last submission to None export.time_of_last_submission = None export.save() self.assertTrue(Export.exports_outdated(xform=self.xform, export_type=Export.XLS_EXPORT))
def test_export_not_found(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) export = Export( xform=self.xform, export_type=export_type, options=options) export.save() export_id = export.pk export.delete() export = generate_export(export_type, self.xform, export_id, options) self.assertIsNotNone(export) self.assertTrue(export.is_successful) self.assertNotEqual(export_id, export.pk)
def test_retrieving_pending_export(self): self._create_user_and_login() self._publish_transportation_form() export = Export( xform=self.xform, export_type=Export.CSV_EXPORT, options={}, task_id="abcsde") export.save() test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {}) self.assertEqual(export, test_export) test_export = check_pending_export(self.xform, Export.XLS_EXPORT, {}) self.assertIsNone(test_export) export.created_on = export.created_on - timedelta(minutes=6) export.save() test_export = check_pending_export(self.xform, Export.CSV_EXPORT, {}) self.assertIsNone(test_export)
def should_create_new_export(xform, export_type, options, request=None): """ Function that determines whether to create a new export. param: xform param: export_type param: options: additional parameters required for the lookup. remove_group_name: boolean flag group_delimiter: "/" or "." with "/" as the default split_select_multiples: boolean flag binary_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') params: request: Get params are used to determine if new export is required """ split_select_multiples = options.get('split_select_multiples', True) if getattr(settings, 'SHOULD_ALWAYS_CREATE_NEW_EXPORT', False): return True if (request and (frozenset(list(request.GET)) & frozenset(['start', 'end', 'data_id']))) or\ not split_select_multiples: return True export_options_kwargs = get_export_options_query_kwargs(options) export_query = Export.objects.filter( xform=xform, export_type=export_type, **export_options_kwargs ) if options.get(EXPORT_QUERY_KEY) is None: export_query = export_query.exclude(options__has_key=EXPORT_QUERY_KEY) if export_query.count() == 0 or\ Export.exports_outdated(xform, export_type, options=options): return True return False
def generate_export(export_type, xform, export_id=None, options=None, retries=0): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get("group_delimiter", DEFAULT_GROUP_DELIMITER) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False) export_builder.INCLUDE_LABELS = options.get('include_labels', False) export_builder.INCLUDE_LABELS_ONLY = options.get('include_labels_only', False) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get("repeat_index_tags", DEFAULT_INDEX_TAGS) # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.set_survey(xform.survey, xform) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__(temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', } xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) # query mongo for the cursor records = query_mongo(username, id_string, filter_query) export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) func.__call__( temp_file.name, records, username, id_string, filter_query) # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save( file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None: export.save() return export
def create_export_object(xform, export_type, options): export_options = get_export_options(options) return Export(xform=xform, export_type=export_type, options=export_options, created_on=timezone.now())
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, show_label=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.DB_EXPORT: 'to_postgres_sav', } xform = XForm.objects.get(user__username__iexact=username, id_string__exact=id_string) # query mongo for the cursor records = query_mongo(username, id_string, filter_query) export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.SHOW_LABEL = show_label export_builder.set_survey(xform.data_dictionary().survey) print "xform.data_dictionary().survey" print xform.data_dictionary().survey export_builder.FORM_OWNER = xform.user.username temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) func.__call__(temp_file.name, records, username, id_string, filter_query) # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # start: original code commented out # dont persist exports that have a filter # if filter_query is None: # export.save() # end: original code commented out export.save() return export
def create_export_object(xform, export_type, options): export_options = get_export_options(options) return Export(xform=xform, export_type=export_type, options=export_options)
def should_create_new_export(xform, export_type): if (not Export.objects.filter(xform=xform, export_type=export_type).exists() or Export.exports_outdated(xform, export_type=export_type)): return True return False
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, start=None, end=None, remove_group_name=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', } xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) records = ParsedInstance.query_data(xform, query=filter_query, start=start, end=end) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = remove_group_name export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__(temp_file.name, records, username, id_string, filter_query, start=start, end=end) except NoRecordsFoundError: pass # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None and start is None and end is None: export.save() return export
def should_create_new_export(xform, export_type): if Export.objects.filter( xform=xform, export_type=export_type).count() == 0\ or Export.exports_outdated(xform, export_type=export_type): return True return False
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, sync_to_gsuit=False, user=None): """ Create appropriate export object given the export type """ time.sleep(5) export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.ANALYSER_EXPORT: 'to_analyser_export' } xform = XForm.objects.get( user__username__iexact=username, id_string__exact=id_string) # query mongo for the cursor records = query_mongo(username, id_string, filter_query) export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples __version__ = "0" try: __version__ = filter_query['$and'][0]['__version__'] except Exception as e: print(str(e)) if __version__: survey = build_survey_from_history(xform, __version__) if not survey: export_builder.set_survey(xform.data_dictionary().survey) else: export_builder.set_survey(survey) else: export_builder.set_survey(xform.data_dictionary().survey) prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string)) temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) func.__call__( temp_file.name, records, username, id_string, filter_query) # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) if export_type == Export.ANALYSER_EXPORT: # Analyser exports should be distinguished by more than just their file extension. basename= '{}_ANALYSER_{}'.format(id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes print sync_to_gsuit, 'file_url--------->', temp_file, filter_query try: if sync_to_gsuit == True and '__version__' not in filter_query['$and'][0]: if not os.path.exists("media/forms/"): os.makedirs("media/forms/") temporarylocation="media/forms/submissions_{}.xls".format(id_string) import shutil shutil.copy(temp_file.name, temporarylocation) fxf_form = FieldSightXF.objects.get(pk=filter_query['$and'][0]['fs_project_uuid']) upload_to_drive(temporarylocation, str(fxf_form.id) + '_' +id_string, None, fxf_form.project, user) os.remove(temporarylocation) except Exception as e: print e.__dict__ # get or create export object temp_file.seek(0) export_filename = storage.save( file_path, File(temp_file, file_path)) dir_name, basename = os.path.split(export_filename) temp_file.close() if export_id: export = Export.objects.get(id=export_id) else: fsxf = filter_query.values()[0] # print("fsxf", fsxf) export = Export(xform=xform, export_type=export_type, fsxf_id=fsxf) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None: export.save() export.save() return export
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): """ Create appropriate export object given the export type """ export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.ANALYSER_EXPORT: 'to_analyser_export' } xform = XForm.objects.get(user__username__iexact=username, id_string__exact=id_string) # query mongo for the cursor records = query_mongo(username, id_string, filter_query) export_builder = ExportBuilder() export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string)) temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) func.__call__(temp_file.name, records, username, id_string, filter_query) # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) if export_type == Export.ANALYSER_EXPORT: # Analyser exports should be distinguished by more than just their file extension. basename = '{}_ANALYSER_{}'.format( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None: export.save() return export
def generate_export(export_type, xform, export_id=None, options=None): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') show_choice_labels: boolean flag language: language labels as in the XLSForm/XForm """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True ) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) export_builder.INCLUDE_LABELS = options.get('include_labels', False) include_reviews = options.get('include_reviews', False) export_builder.INCLUDE_LABELS_ONLY = options.get( 'include_labels_only', False ) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels', False) export_builder.language = options.get('language') # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.INCLUDE_REVIEWS = include_reviews export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records ) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export
def _create_old_export(self, xform, export_type, options): Export(xform=xform, export_type=export_type, options=options).save() self.export = Export.objects.filter(xform=xform, export_type=export_type)