def create_osm_export(username, id_string, export_id, **options): """ OSM export task. """ # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = _get_export_object(export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_osm_export( Export.OSM_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "OSM Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_generate_osm_export(self): filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join(os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = {"extension": Attachment.OSM} export = generate_osm_export(Attachment.OSM, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) with open(combined_osm_path) as f: osm = f.read() with default_storage.open(export.filepath) as f2: content = f2.read() self.assertMultiLineEqual(content.strip(), osm.strip())
def test_generate_osm_export(self): filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath(os.path.join( os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) export = generate_osm_export(Attachment.OSM, Attachment.OSM, self.user.username, self.xform.id_string) self.assertTrue(export.is_successful) with open(combined_osm_path) as f: osm = f.read() with default_storage.open(export.filepath) as f2: content = f2.read() self.assertMultiLineEqual(content, osm)
def create_osm_export(username, id_string, export_id, query=None): # we re-query the db instead of passing model objects according to # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state export = Export.objects.get(id=export_id) try: # though export is not available when for has 0 submissions, we # catch this since it potentially stops celery gen_export = generate_osm_export( Export.OSM_EXPORT, 'osm', username, id_string, export_id, query) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("OSM Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e, sys.exc_info()) raise else: return gen_export.id
def test_generate_osm_export(self): filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join( os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = {"extension": Attachment.OSM} export = generate_osm_export(Attachment.OSM, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) with open(combined_osm_path, encoding='utf-8') as f: osm = f.read() with default_storage.open(export.filepath) as f2: content = f2.read().decode('utf-8') self.assertMultiLineEqual(content.strip(), osm.strip()) # delete submission and check that content is no longer in export submission = self.xform.instances.filter().first() submission.deleted_at = timezone.now() submission.save() export = generate_osm_export(Attachment.OSM, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) with default_storage.open(export.filepath) as f2: content = f2.read() self.assertEqual(content, b'')
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export(export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta'), request.GET.get('data_id')) elif export_type == Export.OSM_EXPORT: export = generate_osm_export(export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) elif export_type == Export.KML_EXPORT: export = generate_kml_export(export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) else: remove_group_name = False if "remove_group_name" in request.QUERY_PARAMS: remove_group_name = \ str_to_bool(request.QUERY_PARAMS["remove_group_name"]) export = generate_export(export_type, extension, xform.user.username, xform.id_string, None, query, remove_group_name=remove_group_name) audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def _generate_new_export(request, xform, query, export_type): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) try: if export_type == Export.EXTERNAL_EXPORT: export = generate_external_export( export_type, xform.user.username, xform.id_string, None, request.GET.get('token'), query, request.GET.get('meta'), request.GET.get('data_id') ) elif export_type == Export.OSM_EXPORT: export = generate_osm_export( export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) elif export_type == Export.KML_EXPORT: export = generate_kml_export( export_type, extension, xform.user.username, xform.id_string, export_id=None, filter_query=None) else: remove_group_name = False if "remove_group_name" in request.QUERY_PARAMS: remove_group_name = \ str_to_bool(request.QUERY_PARAMS["remove_group_name"]) export = generate_export( export_type, extension, xform.user.username, xform.id_string, None, query, remove_group_name=remove_group_name ) audit = { "xform": xform.id_string, "export_type": export_type } log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return {'error': str(e)} else: return export
def _generate_new_export(request, xform, query, export_type, dataview_pk=False): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) options = { "extension": extension, "username": xform.user.username, "id_string": xform.id_string, } if query: options['query'] = query options["dataview_pk"] = dataview_pk if export_type == Export.GOOGLE_SHEETS_EXPORT: options['google_credentials'] = \ _get_google_credential(request).to_json() try: if export_type == Export.EXTERNAL_EXPORT: options['token'] = request.GET.get('token') options['data_id'] = request.GET.get('data_id') options['meta'] = request.GET.get('meta') export = generate_external_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) elif export_type == Export.OSM_EXPORT: export = generate_osm_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) elif export_type == Export.ZIP_EXPORT: export = generate_attachments_zip_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) elif export_type == Export.KML_EXPORT: export = generate_kml_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) else: options.update(parse_request_export_options(request.query_params)) export = generate_export(export_type, xform, None, options) audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return async_status(FAILED, str(e)) except SPSSIOError as e: raise exceptions.ParseError(str(e)) else: return export