def test_delete_oldest_export_on_limit(self): self._publish_transportation_form() self._submit_transport_instance() # create first export first_export = generate_export( Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) self.assertIsNotNone(first_export.pk) # create exports that exceed set limit for i in range(Export.MAX_EXPORTS): generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) # first export should be deleted exports = Export.objects.filter(id=first_export.id) self.assertEqual(len(exports), 0)
def test_dotted_fields_csv_export(self): """ Test CSV export with dotted field names """ path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone', 'userone_with_dot_name_fields.xls') self._publish_xls_file_and_set_xform(path) path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone', 'userone_with_dot_name_fields.xml') self._make_submission( path, forced_submission_time=self._submission_time) # test csv self.options['id_string'] = 'userone' export = generate_export( Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') test_file_path = os.path.join( os.path.dirname(__file__), 'fixtures', 'userone', 'userone_with_dot_name_fields.csv') with storage.open(export.filepath, 'r') as csv_file: self._test_csv_files(csv_file, test_file_path)
def create_csv_export(username, id_string, export_id, **options): """ CSV export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.CSV_EXPORT, export.xform, export_id, options) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records export.internal_status = Export.FAILED export.save() except Exception as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "CSV Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_csv_nested_repeat_output(self): path = os.path.join(self.fixture_dir, 'double_repeat.xls') self._publish_xls_file(path) path = os.path.join(self.fixture_dir, 'instance.xml') self._make_submission( path, forced_submission_time=self._submission_time) self.maxDiff = None dd = DataDictionary.objects.all()[0] xpaths = [ '/double_repeat/bed_net[1]/member[1]/name', '/double_repeat/bed_net[1]/member[2]/name', '/double_repeat/bed_net[2]/member[1]/name', '/double_repeat/bed_net[2]/member[2]/name', '/double_repeat/meta/instanceID' ] self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths) # test csv export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, 'double_repeat') storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with open(os.path.join(self.fixture_dir, 'export.csv')) as f1: with storage.open(export.filepath) as f2: expected_content = f1.read() actual_content = f2.read() self.assertEquals(actual_content, expected_content)
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export( export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta') ) else: export = generate_export( export_type, extension, xform.user.username, xform.id_string, None, query ) audit = { "xform": xform.id_string, "export_type": export_type } log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def create_sav_zip_export(username, id_string, export_id, query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): export = Export.objects.get(id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.SAV_ZIP_EXPORT, 'zip', username, id_string, export_id, query, group_delimiter, split_select_multiples, binary_select_multiples) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception( "SAV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_csv_export(username, id_string, export_id, query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = Export.objects.get(id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export( Export.CSV_EXPORT, 'csv', username, id_string, export_id, query, group_delimiter, split_select_multiples, binary_select_multiples) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records export.internal_status = Export.FAILED export.save() except Exception as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("CSV Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_duplicate_export_filename_is_renamed(self): self._publish_transportation_form() self._submit_transport_instance() # TODO: mock the time # only works if the time we time we generate the basename # is exact to the second with the time the 2nd export is created # create an export object in the db basename = "%s_%s" % ( self.xform.id_string, datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + ".csv" export = Export.objects.create( xform=self.xform, export_type=Export.CSV_EXPORT, filename=filename) # 2nd export export_2 = generate_export( Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string) if export.created_on.timetuple() == export_2.created_on.timetuple(): new_filename = increment_index_in_filename(filename) self.assertEqual(new_filename, export_2.filename) else: self.skipTest("duplicate export filename test skipped " "because export times differ.")
def create_xls_export(username, id_string, export_id, query=None, force_xlsx=True, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state ext = 'xls' if not force_xlsx else 'xlsx' export = Export.objects.get(id=export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export( Export.XLS_EXPORT, ext, username, id_string, export_id, query, group_delimiter, split_select_multiples, binary_select_multiples) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("XLS Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) #raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def test_dotted_fields_csv_export(self): """ Test CSV export with dotted field names """ path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone', 'userone_with_dot_name_fields.xls') self._publish_xls_file_and_set_xform(path) path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone', 'userone_with_dot_name_fields.xml') self._make_submission( path, forced_submission_time=self._submission_time) # test csv self.options['id_string'] = 'userone' export = generate_export( Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') test_file_path = os.path.join( os.path.dirname(__file__), 'fixtures', 'userone', 'userone_with_dot_name_fields.csv') with storage.open(export.filepath, 'r') as csv_file: self._test_csv_files(csv_file, test_file_path)
def test_csv_repeat_with_note(self): """ Test that note field in repeat is not in csv export """ path = os.path.join(self.fixture_dir, 'repeat_w_note.xlsx') self._publish_xls_file_and_set_xform(path) path = os.path.join(self.fixture_dir, 'repeat_w_note.xml') self._make_submission( path, forced_submission_time=self._submission_time) export = generate_export( Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with storage.open(export.filepath, 'r') as csv_file: reader = csv.reader(csv_file) rows = [row for row in reader] actual_headers = [h for h in rows[0]] expected_headers = [ 'chnum', 'chrepeat[1]/chname', 'chrepeat[2]/chname', 'meta/instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received'] self.assertEqual(sorted(expected_headers), sorted(actual_headers))
def create_xls_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) options["extension"] = 'xlsx' if force_xlsx else 'xls' try: export = _get_export_object(id=export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.XLS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "XLS Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export( export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta') ) else: export = generate_export( export_type, extension, xform.user.username, xform.id_string, None, query ) audit = { "xform": xform.id_string, "export_type": export_type } log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def create_csv_export(username, id_string, export_id, query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, remove_group_name=False): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = Export.objects.get(id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export( Export.CSV_EXPORT, 'csv', username, id_string, export_id, query, group_delimiter, split_select_multiples, binary_select_multiples, remove_group_name=remove_group_name ) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records export.internal_status = Export.FAILED export.save() except Exception as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("CSV Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_csv_repeat_with_note(self): """ Test that note field in repeat is not in csv export """ path = os.path.join(self.fixture_dir, 'repeat_w_note.xlsx') self._publish_xls_file_and_set_xform(path) path = os.path.join(self.fixture_dir, 'repeat_w_note.xml') self._make_submission( path, forced_submission_time=self._submission_time) export = generate_export( Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with storage.open(export.filepath, 'r') as csv_file: reader = csv.reader(csv_file) rows = [row for row in reader] actual_headers = [h for h in rows[0]] expected_headers = [ 'chnum', 'chrepeat[1]/chname', 'chrepeat[2]/chname', 'meta/instanceID', '_id', '_uuid', '_submission_time', '_tags', '_notes', '_version', '_duration', '_submitted_by', '_total_media', '_media_count', '_media_all_received'] self.assertEqual(sorted(expected_headers), sorted(actual_headers))
def create_xls_export(username, id_string, export_id, **options): """ XLS export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state force_xlsx = options.get("force_xlsx", True) options["extension"] = 'xlsx' if force_xlsx else 'xls' try: export = _get_export_object(export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.XLS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "XLS Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def test_csv_nested_repeat_output(self): """ Test CSV export with nested repeats """ path = os.path.join(self.fixture_dir, 'double_repeat.xls') self._publish_xls_file(path) self.xform = XForm.objects.get(id_string='double_repeat') path = os.path.join(self.fixture_dir, 'instance.xml') self._make_submission(path, forced_submission_time=self._submission_time) self.maxDiff = None data_dictionary = DataDictionary.objects.all()[0] xpaths = [ u'/double_repeat/bed_net[1]/member[1]/name', u'/double_repeat/bed_net[1]/member[2]/name', u'/double_repeat/bed_net[2]/member[1]/name', u'/double_repeat/bed_net[2]/member[2]/name', u'/double_repeat/meta/instanceID' ] self.assertEquals(data_dictionary.xpaths(repeat_iterations=2), xpaths) # test csv export = generate_export(Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') test_file_path = os.path.join(self.fixture_dir, 'export.csv') with storage.open(export.filepath) as csv_file: self._test_csv_files(csv_file, test_file_path)
def create_sav_zip_export(username, id_string, export_id, query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False): export = Export.objects.get(id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export( Export.SAV_ZIP_EXPORT, 'zip', username, id_string, export_id, query, group_delimiter, split_select_multiples, binary_select_multiples ) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("SAV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_csv_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.CSV_EXPORT, export.xform, export_id, options) except NoRecordsFoundError: # not much we can do but we don't want to report this as the user # should not even be on this page if the survey has no records export.internal_status = Export.FAILED export.save() except Exception as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "CSV Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_csv_nested_repeat_output(self): path = os.path.join(self.fixture_dir, 'double_repeat.xls') self._publish_xls_file(path) path = os.path.join(self.fixture_dir, 'instance.xml') self._make_submission( path, forced_submission_time=self._submission_time) self.maxDiff = None dd = DataDictionary.objects.all()[0] xpaths = [ u'/double_repeat/bed_net[1]/member[1]/name', u'/double_repeat/bed_net[1]/member[2]/name', u'/double_repeat/bed_net[2]/member[1]/name', u'/double_repeat/bed_net[2]/member[2]/name', u'/double_repeat/meta/instanceID' ] self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths) # test csv export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, 'double_repeat') storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with open(os.path.join(self.fixture_dir, 'export.csv')) as f1: with storage.open(export.filepath) as f2: expected_content = f1.read() actual_content = f2.read() self.assertEquals(actual_content, expected_content)
def test_csv_nested_repeat_output(self): """ Test CSV export with nested repeats """ path = os.path.join(self.fixture_dir, 'double_repeat.xls') self._publish_xls_file(path) self.xform = XForm.objects.get(id_string='double_repeat') path = os.path.join(self.fixture_dir, 'instance.xml') self._make_submission( path, forced_submission_time=self._submission_time) self.maxDiff = None data_dictionary = DataDictionary.objects.all()[0] xpaths = [ u'/double_repeat/bed_net[1]/member[1]/name', u'/double_repeat/bed_net[1]/member[2]/name', u'/double_repeat/bed_net[2]/member[1]/name', u'/double_repeat/bed_net[2]/member[2]/name', u'/double_repeat/meta/instanceID' ] self.assertEquals(data_dictionary.xpaths(repeat_iterations=2), xpaths) # test csv export = generate_export( Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') test_file_path = os.path.join(self.fixture_dir, 'export.csv') with storage.open(export.filepath, 'r') as csv_file: self._test_csv_files(csv_file, test_file_path)
def test_last_submission_time_on_export(self): self._publish_transportation_form() self._submit_transport_instance() # create export generate_export( Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) num_exports = Export.objects.filter( xform=self.xform, export_type=Export.XLS_EXPORT).count() # check that our function knows there are no more submissions self.assertFalse( Export.exports_outdated(xform=self.xform, export_type=Export.XLS_EXPORT)) sleep(1) # force new last submission date on xform last_submission = self.xform.instances.order_by('-date_created')[0] last_submission.date_created += datetime.timedelta(hours=1) last_submission.save() # check that our function knows data has changed self.assertTrue( Export.exports_outdated(xform=self.xform, export_type=Export.XLS_EXPORT)) # check that requesting list url will generate a new export export_list_url = reverse(export_list, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': Export.XLS_EXPORT }) self.client.get(export_list_url) self.assertEqual( Export.objects.filter(xform=self.xform, export_type=Export.XLS_EXPORT).count(), num_exports + 1) # make sure another export type causes auto-generation num_exports = Export.objects.filter( xform=self.xform, export_type=Export.CSV_EXPORT).count() export_list_url = reverse(export_list, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': Export.CSV_EXPORT }) self.client.get(export_list_url) self.assertEqual( Export.objects.filter(xform=self.xform, export_type=Export.CSV_EXPORT).count(), num_exports + 1)
def test_delete_file_on_export_delete(self): self._publish_transportation_form() self._submit_transport_instance() export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) # delete export object export.delete() self.assertFalse(storage.exists(export.filepath))
def test_export_progress_output(self): self._publish_transportation_form() self._submit_transport_instance() # create exports for i in range(2): generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) self.assertEqual(Export.objects.count(), 2) # progress for multiple exports progress_url = reverse(export_progress, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': 'xls' }) get_data = {'export_ids': [e.id for e in Export.objects.all()]} response = self.client.get(progress_url, get_data) content = json.loads(response.content) self.assertEqual(len(content), 2) self.assertEqual(sorted(['url', 'export_id', 'complete', 'filename']), sorted(content[0].keys()))
def test_last_submission_time_empty(self): self._publish_transportation_form() self._submit_transport_instance() # create export export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) # set time of last submission to None export.time_of_last_submission = None export.save() self.assertTrue(Export.exports_outdated(xform=self.xform, export_type=Export.XLS_EXPORT))
def test_generate_csv_zip_export(self): # publish xls form self._publish_transportation_form_and_submit_instance() # create export db object export = generate_export( Export.CSV_ZIP_EXPORT, "zip", self.user.username, self.xform.id_string, group_delimiter='/', split_select_multiples=True) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.zip')
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export(export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta'), request.GET.get('data_id')) elif export_type == Export.OSM_EXPORT: export = generate_osm_export(export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) elif export_type == Export.KML_EXPORT: export = generate_kml_export(export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) else: remove_group_name = False if "remove_group_name" in request.QUERY_PARAMS: remove_group_name = \ str_to_bool(request.QUERY_PARAMS["remove_group_name"]) export = generate_export(export_type, extension, xform.user.username, xform.id_string, None, query, remove_group_name=remove_group_name) audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def test_export_public_project(self): """ Test export of a public form for anonymous users. """ self._create_user_and_login() self._publish_transportation_form() self.xform.shared_data = True self.xform.save() export = generate_export(Export.CSV_EXPORT, self.xform, None, {"extension": "csv"}) request = self.factory.get('/export') response = self.view(request, pk=export.pk) self.assertEqual(status.HTTP_200_OK, response.status_code)
def test_export_non_public_export(self): """ Test export of a private form for anonymous users results in HTTP_404_NOT_FOUND response. """ self._create_user_and_login() self._publish_transportation_form() self.xform.shared_data = False self.xform.save() export = generate_export(Export.CSV_EXPORT, self.xform, None, {"extension": "csv"}) request = self.factory.get('/export') response = self.view(request, pk=export.pk) self.assertEqual(status.HTTP_404_NOT_FOUND, response.status_code)
def test_create_export(self): self._publish_transportation_form_and_submit_instance() storage = get_storage_class()() # test xls export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.xls') # test csv export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string) self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') # test xls with existing export_id existing_export = Export.objects.create(xform=self.xform, export_type=Export.XLS_EXPORT) export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string, existing_export.id) self.assertEqual(existing_export.id, export.id)
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export( export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta'), request.GET.get('data_id') ) elif export_type == Export.OSM_EXPORT: export = generate_osm_export( export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) elif export_type == Export.KML_EXPORT: export = generate_kml_export( export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) else: remove_group_name = False if "remove_group_name" in request.QUERY_PARAMS: remove_group_name = \ str_to_bool(request.QUERY_PARAMS["remove_group_name"]) export = generate_export( export_type, extension, xform.user.username, xform.id_string, None, query, remove_group_name=remove_group_name ) audit = { "xform": xform.id_string, "export_type": export_type } log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def test_export_download_url(self): self._publish_transportation_form() self._submit_transport_instance() export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string) csv_export_url = reverse(export_download, kwargs={ "username": self.user.username, "id_string": self.xform.id_string, "export_type": Export.CSV_EXPORT, "filename": export.filename }) response = self.client.get(csv_export_url) self.assertEqual(response.status_code, 200) # test xls export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) xls_export_url = reverse(export_download, kwargs={ "username": self.user.username, "id_string": self.xform.id_string, "export_type": Export.XLS_EXPORT, "filename": export.filename }) response = self.client.get(xls_export_url) self.assertEqual(response.status_code, 200)
def create_db_export(username, id_string, export_id, query=None, force_xlsx=False, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, show_label=False): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state ext = 'txt' if not force_xlsx else 'txt' try: logger = create_db_export.get_logger() logger.info("Enter to get export_id") # print 'Enter to get export_id' export = Export.objects.get(id=export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.DB_EXPORT, ext, username, id_string, export_id, query, group_delimiter, split_select_multiples, binary_select_multiples, show_label) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception( "DB Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def test_allow_export_download_for_basic_auth(self): extra = { 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, self.login_password) } # create export export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string) self.assertTrue(isinstance(export, Export)) url = reverse(export_download, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': export.export_type, 'filename': export.filename }) response = self.anon.get(url, **extra) self.assertEqual(response.status_code, 200)
def test_allow_export_download_for_basic_auth(self): extra = { 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, self.login_password) } # create export export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string) self.assertTrue(isinstance(export, Export)) url = reverse(export_download, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': export.export_type, 'filename': export.filename }) response = self.anon.get(url, **extra) self.assertEqual(response.status_code, 200)
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: export = generate_export(export_type, extension, xform.user.username, xform.id_string, None, query) audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, t("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(t("No records found to export")) else: return export
def test_delete_export_url(self): self._publish_transportation_form() self._submit_transport_instance() # create export export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) exports = Export.objects.filter(id=export.id) self.assertEqual(len(exports), 1) delete_url = reverse(delete_export, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': 'xls' }) post_data = {'export_id': export.id} response = self.client.post(delete_url, post_data) self.assertEqual(response.status_code, 302) exports = Export.objects.filter(id=export.id) self.assertEqual(len(exports), 0)
def create_analyser_export(username, id_string, export_id, query=None): # Mostly a serving of copy pasta based on the above `create_xls_export()`. Enjoy. # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state ext = 'xlsx' try: export = Export.objects.get(id=export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.ANALYSER_EXPORT, ext, username, id_string, export_id, query, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception( "Analyser Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def create_sav_zip_export(username, id_string, export_id, **options): export = _get_export_object(id=export_id) options["extension"] = Export.ZIP_EXPORT try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception("SAV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_csv_export_output(self): path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls') self._publish_xls_file_and_set_xform(path) path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml') self._make_submission( path, forced_submission_time=self._submission_time) # test csv export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, 'tutorial_w_repeats') storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with open(os.path.join( self.fixture_dir, 'tutorial_w_repeats.csv')) as f1: with storage.open(export.filepath) as f2: expected_content = f1.read() actual_content = f2.read() self.assertEquals(actual_content, expected_content)
def test_csv_export_output(self): path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls') self._publish_xls_file_and_set_xform(path) path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml') self._make_submission( path, forced_submission_time=self._submission_time) # test csv export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, 'tutorial_w_repeats') storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with open(os.path.join( self.fixture_dir, 'tutorial_w_repeats.csv'), 'rb') as f1: with storage.open(export.filepath) as f2: expected_content = f1.read() actual_content = f2.read() self.assertEqual(actual_content, expected_content)
def test_404_on_export_io_error(self): """ Test that we return a 404 when the response_with_mimetype_and_name encounters an IOError """ self._publish_transportation_form() self._submit_transport_instance() export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username, self.xform.id_string) export_url = reverse(export_download, kwargs={ "username": self.user.username, "id_string": self.xform.id_string, "export_type": Export.CSV_EXPORT, "filename": export.filename }) # delete the export export.delete() # access the export response = self.client.get(export_url) self.assertEqual(response.status_code, 404)
def create_google_sheet_export(username, id_string, export_id, **options): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state try: export = _get_export_object(id=export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception("Google Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_csv_export_truncated_titles(self): path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls') self._publish_xls_file_and_set_xform(path) path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml') self._make_submission(path, forced_submission_time=self._submission_time) # test csv self.options['remove_group_name'] = True export = generate_export(Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') with open(os.path.join( self.fixture_dir, 'tutorial_w_repeats_truncate_titles.csv')) \ as f1: with storage.open(export.filepath) as f2: expected_content = f1.read() actual_content = f2.read() self.assertEquals(actual_content, expected_content)
def test_csv_export_output(self): """ Test CSV export output """ path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls') self._publish_xls_file_and_set_xform(path) path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml') self._make_submission(path, forced_submission_time=self._submission_time) # test csv export = generate_export(Export.CSV_EXPORT, self.xform, None, self.options) storage = get_storage_class()() self.assertTrue(storage.exists(export.filepath)) path, ext = os.path.splitext(export.filename) self.assertEqual(ext, '.csv') test_file_path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.csv') with storage.open(export.filepath, 'r') as csv_file: self._test_csv_files(csv_file, test_file_path)
def test_export_not_found(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) export = Export( xform=self.xform, export_type=export_type, options=options) export.save() export_id = export.pk export.delete() export = generate_export(export_type, self.xform, export_id, options) self.assertIsNotNone(export) self.assertTrue(export.is_successful) self.assertNotEqual(export_id, export.pk)
def test_export_not_found(self): export_type = "csv" options = { "group_delimiter": "/", "remove_group_name": False, "split_select_multiples": True } self._publish_transportation_form_and_submit_instance() self._create_old_export(self.xform, export_type, options) export = Export( xform=self.xform, export_type=export_type, options=options) export.save() export_id = export.pk export.delete() export = generate_export(export_type, self.xform, export_id, options) self.assertIsNotNone(export) self.assertTrue(export.is_successful) self.assertNotEqual(export_id, export.pk)
def test_graceful_exit_on_export_delete_if_file_doesnt_exist(self): self._publish_transportation_form() self._submit_transport_instance() export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username, self.xform.id_string) storage = get_storage_class()() # delete file storage.delete(export.filepath) self.assertFalse(storage.exists(export.filepath)) # clear filename, like it would be in an incomplete export export.filename = None export.filedir = None export.save() # delete export record, which should try to delete file as well delete_url = reverse(delete_export, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string, 'export_type': 'xls' }) post_data = {'export_id': export.id} response = self.client.post(delete_url, post_data) self.assertEqual(response.status_code, 302)
def create_sav_zip_export(username, id_string, export_id, **options): """ SPSS sav export task. """ export = _get_export_object(export_id) options["extension"] = Export.ZIP_EXPORT try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.SAV_ZIP_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, TypeError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "SAV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_google_sheet_export(username, id_string, export_id, **options): """ Google Sheets export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state try: export = _get_export_object(export_id) # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_export(Export.GOOGLE_SHEETS_EXPORT, export.xform, export_id, options) except (Exception, NoRecordsFoundError, ConnectionError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "Google Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def create_analyser_export(username, id_string, export_id, query=None): # Mostly a serving of copy pasta based on the above `create_xls_export()`. Enjoy. # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state ext = 'xlsx' try: export = Export.objects.get(id=export_id) except Export.DoesNotExist: # no export for this ID return None. return None # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery try: gen_export = generate_export(Export.ANALYSER_EXPORT, ext, username, id_string, export_id, query, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("Analyser Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) # Raise for now to let celery know we failed # - doesnt seem to break celery` raise else: return gen_export.id
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: export = generate_export( export_type, extension, xform.user.username, xform.id_string, None, query ) audit = { "xform": xform.id_string, "export_type": export_type } log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) else: return export