def render(self, data, accepted_media_type=None, renderer_context=None): view = renderer_context['view'] # `AssetNestedObjectViewsetMixin` provides the asset asset = view.asset if renderer_context['response'].status_code != status.HTTP_200_OK: # We're ending up with stuff like `{u'detail': u'Not found.'}` in # `data`. Is this the best way to handle that? return None pack, submission_stream = build_formpack(asset, data) # Right now, we're more-or-less mirroring the JSON renderer. In the # future, we could expose more export options (e.g. label language) export = pack.export( versions=pack.versions.keys(), group_sep='/', lang=formpack.constants.UNSPECIFIED_TRANSLATION, hierarchy_in_labels=True, ) geo_question_name = view.request.query_params.get('geo_question_name') if not geo_question_name: # No geo question specified; use the first one in the latest # version of the form latest_version = next(reversed(list(pack.versions.values()))) first_section = next(iter(latest_version.sections.values())) geo_questions = (field for field in first_section.fields.values() if field.data_type in GEO_QUESTION_TYPES) try: geo_question_name = next(geo_questions).name except StopIteration: # formpack will gracefully return an empty `features` array geo_question_name = None return ''.join( export.to_geojson( submission_stream, geo_question_name=geo_question_name, ))
def setUp(self): self.maxDiff = None self.user = User.objects.get(username='******') self.asset = Asset.objects.get(uid='axD3Wc8ZnfgLXBcURRt5fM') # To avoid cluttering the fixture, assign permissions here self.asset.assign_perm(self.user, PERM_VIEW_SUBMISSIONS) self.submissions = self.asset.deployment.get_submissions( self.asset.owner.id) self.submission_id_field = '_id' self.formpack, self.submission_stream = report_data.build_formpack( self.asset, submission_stream=self.submissions, ) self.fields_to_inspect = [ 'created_in_first_version', 'created_in_second_version', 'created_in_third_version', 'created_in_fourth_version', ] self.expected_results = {} for sub in self.submissions: fields_values = {} for field in self.fields_to_inspect: try: value = sub[field] except KeyError: value = '' fields_values[field] = value self.expected_results[str( sub[self.submission_id_field])] = fields_values
def setUp(self): self.user = User.objects.get(username='******') self.asset = Asset.objects.create(name='Identificación de animales', content=self.form_content, owner=self.user) self.asset.deploy(backend='mock', active=True) self.asset.save() v_uid = self.asset.latest_deployed_version.uid for submission in self.submissions: submission.update({'__version__': v_uid}) self.asset.deployment.mock_submissions(self.submissions) self.formpack, self.submission_stream = report_data.build_formpack( self.asset, submission_stream=self.asset.deployment.get_submissions())
def get_export_object( self, source: Optional[Asset] = None ) -> Tuple[formpack.reporting.Export, Generator]: """ Get the formpack Export object and submission stream for processing. """ fields = self.data.get('fields', []) query = self.data.get('query', {}) submission_ids = self.data.get('submission_ids', []) if source is None: source_url = self.data.get('source', False) if not source_url: raise Exception('no source specified for the export') try: source = resolve_url_to_asset(source_url) except Asset.DoesNotExist: raise self.InaccessibleData source_perms = source.get_perms(self.user) if (PERM_VIEW_SUBMISSIONS not in source_perms and PERM_PARTIAL_SUBMISSIONS not in source_perms): raise self.InaccessibleData if not source.has_deployment: raise Exception('the source must be deployed prior to export') # Include the group name in `fields` for Mongo to correctly filter # for repeat groups fields = self._get_fields_and_groups(fields) submission_stream = source.deployment.get_submissions( user=self.user, fields=fields, submission_ids=submission_ids, query=query, ) pack, submission_stream = build_formpack( source, submission_stream, self._fields_from_all_versions) # Wrap the submission stream in a generator that records the most # recent timestamp submission_stream = self._record_last_submission_time( submission_stream) options = self._build_export_options(pack) return pack.export(**options), submission_stream
def setUp(self): super().setUp() self.anotheruser = User.objects.get(username='******') partial_perms = { PERM_VIEW_SUBMISSIONS: [{ '_submitted_by': self.anotheruser.username }] } self.asset.assign_perm( self.anotheruser, PERM_PARTIAL_SUBMISSIONS, partial_perms=partial_perms, ) self.formpack, self.submission_stream = report_data.build_formpack( self.asset, submission_stream=self.asset.deployment.get_submissions( self.asset.owner.id), )
def validate_data_sharing(self, data_sharing: dict) -> dict: """ Validates `data_sharing`. It is really basic. Only the type of each property is validated. No data is validated. It is consistent with partial permissions and REST services. The client bears the responsibility of providing valid data. """ errors = {} if not self.instance or not data_sharing: return data_sharing if 'enabled' not in data_sharing: errors['enabled'] = t('The property is required') if 'fields' in data_sharing: if not isinstance(data_sharing['fields'], list): errors['fields'] = t('The property must be an array') else: asset = self.instance fields = data_sharing['fields'] form_pack, _unused = build_formpack(asset, submission_stream=[]) valid_fields = [ f.path for f in form_pack.get_fields_for_versions( form_pack.versions.keys()) ] unknown_fields = set(fields) - set(valid_fields) if unknown_fields and valid_fields: errors['fields'] = t( 'Some fields are invalid, ' 'choices are: `{valid_fields}`').format( valid_fields='`,`'.join(valid_fields)) else: data_sharing['fields'] = [] if errors: raise serializers.ValidationError(errors) return data_sharing
def _run_task(self, messages): ''' Generate the export and store the result in the `self.result` `PrivateFileField`. Should be called by the `run()` method of the superclass. The `submission_stream` method is provided for testing ''' source_url = self.data.get('source', False) if not source_url: raise Exception('no source specified for the export') source_type, source = _resolve_url_to_asset_or_collection(source_url) if source_type != 'asset': raise NotImplementedError( 'only an `Asset` may be exported at this time') if not source.has_perm(self.user, 'view_submissions'): # Unsure if DRF exceptions make sense here since we're not # returning a HTTP response raise exceptions.PermissionDenied( u'{user} cannot export {source}'.format(user=self.user, source=source)) if not source.has_deployment: raise Exception('the source must be deployed prior to export') export_type = self.data.get('type', '').lower() if export_type not in ('xls', 'csv'): raise NotImplementedError( 'only `xls` and `csv` are valid export types') # Take this opportunity to do some housekeeping self.log_and_mark_stuck_as_errored(self.user, source_url) if hasattr(source.deployment, '_get_submissions'): # Currently used only for unit testing (`MockDeploymentBackend`) # TODO: Have the KC backend also implement `_get_submissions()`? submission_stream = source.deployment._get_submissions() else: submission_stream = None pack, submission_stream = build_formpack( source, submission_stream, self._fields_from_all_versions) # Wrap the submission stream in a generator that records the most # recent timestamp submission_stream = self._record_last_submission_time( submission_stream) options = self._build_export_options(pack) export = pack.export(**options) extension = 'xlsx' if export_type == 'xls' else export_type filename = self._build_export_filename(export, extension) self.result.save(filename, ContentFile('')) # FileField files are opened read-only by default and must be # closed and reopened to allow writing # https://code.djangoproject.com/ticket/13809 self.result.close() self.result.file.close() with self.result.storage.open(self.result.name, 'wb') as output_file: if export_type == 'csv': for line in export.to_csv(submission_stream): output_file.write((line + u"\r\n").encode('utf-8')) elif export_type == 'xls': # XLSX export actually requires a filename (limitation of # pyexcelerate?) with tempfile.NamedTemporaryFile( prefix='export_xlsx', mode='rb') as xlsx_output_file: export.to_xlsx(xlsx_output_file.name, submission_stream) # TODO: chunk again once # https://github.com/jschneier/django-storages/issues/449 # is fixed ''' while True: chunk = xlsx_output_file.read(5 * 1024 * 1024) if chunk: output_file.write(chunk) else: break ''' output_file.write(xlsx_output_file.read()) # Restore the FileField to its typical state self.result.open('rb') self.save(update_fields=['last_submission_time']) # Now that a new export has completed successfully, remove any old # exports in excess of the per-user, per-form limit self.remove_excess(self.user, source_url)
def _run_task(self, messages): """ Generate the export and store the result in the `self.result` `PrivateFileField`. Should be called by the `run()` method of the superclass. The `submission_stream` method is provided for testing """ source_url = self.data.get('source', False) fields = self.data.get('fields', []) flatten = self.data.get('flatten', True) if not source_url: raise Exception('no source specified for the export') source = _resolve_url_to_asset(source_url) source_perms = source.get_perms(self.user) if (PERM_VIEW_SUBMISSIONS not in source_perms and PERM_PARTIAL_SUBMISSIONS not in source_perms): # Unsure if DRF exceptions make sense here since we're not # returning a HTTP response raise exceptions.PermissionDenied( '{user} cannot export {source}'.format(user=self.user, source=source)) if not source.has_deployment: raise Exception('the source must be deployed prior to export') export_type = self.data.get('type', '').lower() if export_type not in ('xls', 'csv', 'geojson', 'spss_labels'): raise NotImplementedError( 'only `xls`, `csv`, `geojson`, and `spss_labels` ' 'are valid export types') # Take this opportunity to do some housekeeping self.log_and_mark_stuck_as_errored(self.user, source_url) submission_stream = source.deployment.get_submissions( requesting_user_id=self.user.id, fields=fields) pack, submission_stream = build_formpack( source, submission_stream, self._fields_from_all_versions) # Wrap the submission stream in a generator that records the most # recent timestamp submission_stream = self._record_last_submission_time( submission_stream) options = self._build_export_options(pack) export = pack.export(**options) filename = self._build_export_filename(export, export_type) self.result.save(filename, ContentFile('')) # FileField files are opened read-only by default and must be # closed and reopened to allow writing # https://code.djangoproject.com/ticket/13809 self.result.close() self.result.file.close() with self.result.storage.open(self.result.name, 'wb') as output_file: if export_type == 'csv': for line in export.to_csv(submission_stream): output_file.write((line + "\r\n").encode('utf-8')) elif export_type == 'geojson': for line in export.to_geojson(submission_stream, flatten=flatten): output_file.write(line.encode('utf-8')) elif export_type == 'xls': # XLSX export actually requires a filename (limitation of # pyexcelerate?) with tempfile.NamedTemporaryFile( prefix='export_xlsx', mode='rb') as xlsx_output_file: export.to_xlsx(xlsx_output_file.name, submission_stream) # TODO: chunk again once # https://github.com/jschneier/django-storages/issues/449 # is fixed # TODO: Check if monkey-patch (line 57) can restore writing # by chunk """ while True: chunk = xlsx_output_file.read(5 * 1024 * 1024) if chunk: output_file.write(chunk) else: break """ output_file.write(xlsx_output_file.read()) elif export_type == 'spss_labels': export.to_spss_labels(output_file) # Restore the FileField to its typical state self.result.open('rb') self.save(update_fields=['last_submission_time']) # Now that a new export has completed successfully, remove any old # exports in excess of the per-user, per-form limit self.remove_excess(self.user, source_url)