def test_generate_attachments_zip_export(self): filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join(os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = {"extension": Export.ZIP_EXPORT} export = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # deleted submission submission = self.xform.instances.filter().first() submission.deleted_at = timezone.now() submission.save() export = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertFalse( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def test_generate_attachments_zip_export(self): filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join( os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = {"extension": Export.ZIP_EXPORT} export = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # deleted submission submission = self.xform.instances.filter().first() submission.deleted_at = timezone.now() submission.save() export = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertFalse( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def create_zip_export(username, id_string, export_id, **options): """ Attachments zip export task. """ export = _get_export_object(export_id) try: gen_export = generate_attachments_zip_export( Export.ZIP_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e) raise else: if not settings.TESTING_MODE: delete_export.apply_async( (), {'export_id': gen_export.id}, countdown=settings.ZIP_EXPORT_COUNTDOWN) return gen_export.id
def create_zip_export(username, id_string, export_id, **options): """ Attachments zip export task. """ export = _get_export_object(export_id) try: gen_export = generate_attachments_zip_export( Export.ZIP_EXPORT, username, id_string, export_id, options, xform=export.xform) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.error_message = str(e) export.save() # mail admins details = _get_export_details(username, id_string, export_id) report_exception( "Zip Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e) raise else: if not settings.TESTING_MODE: delete_export.apply_async( (), {'export_id': gen_export.id}, countdown=settings.ZIP_EXPORT_COUNTDOWN) return gen_export.id
def create_zip_export(username, id_string, export_id, query=None): export = Export.objects.get(id=export_id) try: gen_export = generate_attachments_zip_export( Export.ZIP_EXPORT, 'zip', username, id_string, export_id, query) except (Exception, NoRecordsFoundError) as e: export.internal_status = Export.FAILED export.save() # mail admins details = { 'export_id': export_id, 'username': username, 'id_string': id_string } report_exception("Zip Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details, e) raise else: if not settings.TESTING_MODE: delete_export.apply_async( (), {'export_id': gen_export.id}, countdown=settings.ZIP_EXPORT_COUNTDOWN) return gen_export.id
def _generate_new_export(request, xform, query, export_type, dataview_pk=False): query = _set_start_end_params(request, query) extension = _get_extension_from_export_type(export_type) options = { "extension": extension, "username": xform.user.username, "id_string": xform.id_string, } if query: options['query'] = query options["dataview_pk"] = dataview_pk if export_type == Export.GOOGLE_SHEETS_EXPORT: options['google_credentials'] = \ _get_google_credential(request).to_json() try: if export_type == Export.EXTERNAL_EXPORT: options['token'] = request.GET.get('token') options['data_id'] = request.GET.get('data_id') options['meta'] = request.GET.get('meta') export = generate_external_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) elif export_type == Export.OSM_EXPORT: export = generate_osm_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) elif export_type == Export.ZIP_EXPORT: export = generate_attachments_zip_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) elif export_type == Export.KML_EXPORT: export = generate_kml_export(export_type, xform.user.username, xform.id_string, None, options, xform=xform) else: options.update(parse_request_export_options(request.query_params)) export = generate_export(export_type, xform, None, options) audit = {"xform": xform.id_string, "export_type": export_type} log.audit_log( log.Actions.EXPORT_CREATED, request.user, xform.user, _("Created %(export_type)s export on '%(id_string)s'.") % { 'id_string': xform.id_string, 'export_type': export_type.upper() }, audit, request) except NoRecordsFoundError: raise Http404(_("No records found to export")) except J2XException as e: # j2x exception return async_status(FAILED, str(e)) except SPSSIOError as e: raise exceptions.ParseError(str(e)) else: return export
def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join(os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = { "extension": Export.ZIP_EXPORT, "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}' } filter_query = options.get("query") instance_ids = query_data(self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() filtered_attachments = Attachment.objects.filter( instance__xform_id=self.xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) self.assertNotEqual(Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query options.pop('query') export1 = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export1.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join( os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = { "extension": Export.ZIP_EXPORT, "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'} filter_query = options.get("query") instance_ids = query_data( self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() filtered_attachments = Attachment.objects.filter( instance__xform_id=self.xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) self.assertNotEqual( Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query options.pop('query') export1 = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export1.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)