def render(self, data, accepted_media_type=None, renderer_context=None):
view = renderer_context['view']
# `AssetNestedObjectViewsetMixin` provides the asset
asset = view.asset
if renderer_context['response'].status_code != status.HTTP_200_OK:
# We're ending up with stuff like `{u'detail': u'Not found.'}` in
# `data`. Is this the best way to handle that?
return None
pack, submission_stream = build_formpack(asset, data)
# Right now, we're more-or-less mirroring the JSON renderer. In the
# future, we could expose more export options (e.g. label language)
export = pack.export(
versions=pack.versions.keys(),
group_sep='/',
lang=formpack.constants.UNSPECIFIED_TRANSLATION,
hierarchy_in_labels=True,
)
geo_question_name = view.request.query_params.get('geo_question_name')
if not geo_question_name:
# No geo question specified; use the first one in the latest
# version of the form
latest_version = next(reversed(list(pack.versions.values())))
first_section = next(iter(latest_version.sections.values()))
geo_questions = (field for field in first_section.fields.values()
if field.data_type in GEO_QUESTION_TYPES)
try:
geo_question_name = next(geo_questions).name
except StopIteration:
# formpack will gracefully return an empty `features` array
geo_question_name = None
return ''.join(
export.to_geojson(
submission_stream,
geo_question_name=geo_question_name,
))
def setUp(self):
self.maxDiff = None
self.user = User.objects.get(username='******')
self.asset = Asset.objects.get(uid='axD3Wc8ZnfgLXBcURRt5fM')
# To avoid cluttering the fixture, assign permissions here
self.asset.assign_perm(self.user, PERM_VIEW_SUBMISSIONS)
self.submissions = self.asset.deployment.get_submissions(
self.asset.owner.id)
self.submission_id_field = '_id'
self.formpack, self.submission_stream = report_data.build_formpack(
self.asset,
submission_stream=self.submissions,
)
self.fields_to_inspect = [
'created_in_first_version',
'created_in_second_version',
'created_in_third_version',
'created_in_fourth_version',
]
self.expected_results = {}
for sub in self.submissions:
fields_values = {}
for field in self.fields_to_inspect:
try:
value = sub[field]
except KeyError:
value = ''
fields_values[field] = value
self.expected_results[str(
sub[self.submission_id_field])] = fields_values
def setUp(self):
self.user = User.objects.get(username='******')
self.asset = Asset.objects.create(name='Identificación de animales',
content=self.form_content,
owner=self.user)
self.asset.deploy(backend='mock', active=True)
self.asset.save()
v_uid = self.asset.latest_deployed_version.uid
for submission in self.submissions:
submission.update({'__version__': v_uid})
self.asset.deployment.mock_submissions(self.submissions)
self.formpack, self.submission_stream = report_data.build_formpack(
self.asset,
submission_stream=self.asset.deployment.get_submissions())
def get_export_object(
self,
source: Optional[Asset] = None
) -> Tuple[formpack.reporting.Export, Generator]:
"""
Get the formpack Export object and submission stream for processing.
"""
fields = self.data.get('fields', [])
query = self.data.get('query', {})
submission_ids = self.data.get('submission_ids', [])
if source is None:
source_url = self.data.get('source', False)
if not source_url:
raise Exception('no source specified for the export')
try:
source = resolve_url_to_asset(source_url)
except Asset.DoesNotExist:
raise self.InaccessibleData
source_perms = source.get_perms(self.user)
if (PERM_VIEW_SUBMISSIONS not in source_perms
and PERM_PARTIAL_SUBMISSIONS not in source_perms):
raise self.InaccessibleData
if not source.has_deployment:
raise Exception('the source must be deployed prior to export')
# Include the group name in `fields` for Mongo to correctly filter
# for repeat groups
fields = self._get_fields_and_groups(fields)
submission_stream = source.deployment.get_submissions(
user=self.user,
fields=fields,
submission_ids=submission_ids,
query=query,
)
pack, submission_stream = build_formpack(
source, submission_stream, self._fields_from_all_versions)
# Wrap the submission stream in a generator that records the most
# recent timestamp
submission_stream = self._record_last_submission_time(
submission_stream)
options = self._build_export_options(pack)
return pack.export(**options), submission_stream
def setUp(self):
super().setUp()
self.anotheruser = User.objects.get(username='******')
partial_perms = {
PERM_VIEW_SUBMISSIONS: [{
'_submitted_by': self.anotheruser.username
}]
}
self.asset.assign_perm(
self.anotheruser,
PERM_PARTIAL_SUBMISSIONS,
partial_perms=partial_perms,
)
self.formpack, self.submission_stream = report_data.build_formpack(
self.asset,
submission_stream=self.asset.deployment.get_submissions(
self.asset.owner.id),
)
def validate_data_sharing(self, data_sharing: dict) -> dict:
"""
Validates `data_sharing`. It is really basic.
Only the type of each property is validated. No data is validated.
It is consistent with partial permissions and REST services.
The client bears the responsibility of providing valid data.
"""
errors = {}
if not self.instance or not data_sharing:
return data_sharing
if 'enabled' not in data_sharing:
errors['enabled'] = t('The property is required')
if 'fields' in data_sharing:
if not isinstance(data_sharing['fields'], list):
errors['fields'] = t('The property must be an array')
else:
asset = self.instance
fields = data_sharing['fields']
form_pack, _unused = build_formpack(asset,
submission_stream=[])
valid_fields = [
f.path for f in form_pack.get_fields_for_versions(
form_pack.versions.keys())
]
unknown_fields = set(fields) - set(valid_fields)
if unknown_fields and valid_fields:
errors['fields'] = t(
'Some fields are invalid, '
'choices are: `{valid_fields}`').format(
valid_fields='`,`'.join(valid_fields))
else:
data_sharing['fields'] = []
if errors:
raise serializers.ValidationError(errors)
return data_sharing
def _run_task(self, messages):
'''
Generate the export and store the result in the `self.result`
`PrivateFileField`. Should be called by the `run()` method of the
superclass. The `submission_stream` method is provided for testing
'''
source_url = self.data.get('source', False)
if not source_url:
raise Exception('no source specified for the export')
source_type, source = _resolve_url_to_asset_or_collection(source_url)
if source_type != 'asset':
raise NotImplementedError(
'only an `Asset` may be exported at this time')
if not source.has_perm(self.user, 'view_submissions'):
# Unsure if DRF exceptions make sense here since we're not
# returning a HTTP response
raise exceptions.PermissionDenied(
u'{user} cannot export {source}'.format(user=self.user,
source=source))
if not source.has_deployment:
raise Exception('the source must be deployed prior to export')
export_type = self.data.get('type', '').lower()
if export_type not in ('xls', 'csv'):
raise NotImplementedError(
'only `xls` and `csv` are valid export types')
# Take this opportunity to do some housekeeping
self.log_and_mark_stuck_as_errored(self.user, source_url)
if hasattr(source.deployment, '_get_submissions'):
# Currently used only for unit testing (`MockDeploymentBackend`)
# TODO: Have the KC backend also implement `_get_submissions()`?
submission_stream = source.deployment._get_submissions()
else:
submission_stream = None
pack, submission_stream = build_formpack(
source, submission_stream, self._fields_from_all_versions)
# Wrap the submission stream in a generator that records the most
# recent timestamp
submission_stream = self._record_last_submission_time(
submission_stream)
options = self._build_export_options(pack)
export = pack.export(**options)
extension = 'xlsx' if export_type == 'xls' else export_type
filename = self._build_export_filename(export, extension)
self.result.save(filename, ContentFile(''))
# FileField files are opened read-only by default and must be
# closed and reopened to allow writing
# https://code.djangoproject.com/ticket/13809
self.result.close()
self.result.file.close()
with self.result.storage.open(self.result.name, 'wb') as output_file:
if export_type == 'csv':
for line in export.to_csv(submission_stream):
output_file.write((line + u"\r\n").encode('utf-8'))
elif export_type == 'xls':
# XLSX export actually requires a filename (limitation of
# pyexcelerate?)
with tempfile.NamedTemporaryFile(
prefix='export_xlsx', mode='rb') as xlsx_output_file:
export.to_xlsx(xlsx_output_file.name, submission_stream)
# TODO: chunk again once
# https://github.com/jschneier/django-storages/issues/449
# is fixed
'''
while True:
chunk = xlsx_output_file.read(5 * 1024 * 1024)
if chunk:
output_file.write(chunk)
else:
break
'''
output_file.write(xlsx_output_file.read())
# Restore the FileField to its typical state
self.result.open('rb')
self.save(update_fields=['last_submission_time'])
# Now that a new export has completed successfully, remove any old
# exports in excess of the per-user, per-form limit
self.remove_excess(self.user, source_url)
def _run_task(self, messages):
"""
Generate the export and store the result in the `self.result`
`PrivateFileField`. Should be called by the `run()` method of the
superclass. The `submission_stream` method is provided for testing
"""
source_url = self.data.get('source', False)
fields = self.data.get('fields', [])
flatten = self.data.get('flatten', True)
if not source_url:
raise Exception('no source specified for the export')
source = _resolve_url_to_asset(source_url)
source_perms = source.get_perms(self.user)
if (PERM_VIEW_SUBMISSIONS not in source_perms
and PERM_PARTIAL_SUBMISSIONS not in source_perms):
# Unsure if DRF exceptions make sense here since we're not
# returning a HTTP response
raise exceptions.PermissionDenied(
'{user} cannot export {source}'.format(user=self.user,
source=source))
if not source.has_deployment:
raise Exception('the source must be deployed prior to export')
export_type = self.data.get('type', '').lower()
if export_type not in ('xls', 'csv', 'geojson', 'spss_labels'):
raise NotImplementedError(
'only `xls`, `csv`, `geojson`, and `spss_labels` '
'are valid export types')
# Take this opportunity to do some housekeeping
self.log_and_mark_stuck_as_errored(self.user, source_url)
submission_stream = source.deployment.get_submissions(
requesting_user_id=self.user.id, fields=fields)
pack, submission_stream = build_formpack(
source, submission_stream, self._fields_from_all_versions)
# Wrap the submission stream in a generator that records the most
# recent timestamp
submission_stream = self._record_last_submission_time(
submission_stream)
options = self._build_export_options(pack)
export = pack.export(**options)
filename = self._build_export_filename(export, export_type)
self.result.save(filename, ContentFile(''))
# FileField files are opened read-only by default and must be
# closed and reopened to allow writing
# https://code.djangoproject.com/ticket/13809
self.result.close()
self.result.file.close()
with self.result.storage.open(self.result.name, 'wb') as output_file:
if export_type == 'csv':
for line in export.to_csv(submission_stream):
output_file.write((line + "\r\n").encode('utf-8'))
elif export_type == 'geojson':
for line in export.to_geojson(submission_stream,
flatten=flatten):
output_file.write(line.encode('utf-8'))
elif export_type == 'xls':
# XLSX export actually requires a filename (limitation of
# pyexcelerate?)
with tempfile.NamedTemporaryFile(
prefix='export_xlsx', mode='rb') as xlsx_output_file:
export.to_xlsx(xlsx_output_file.name, submission_stream)
# TODO: chunk again once
# https://github.com/jschneier/django-storages/issues/449
# is fixed
# TODO: Check if monkey-patch (line 57) can restore writing
# by chunk
"""
while True:
chunk = xlsx_output_file.read(5 * 1024 * 1024)
if chunk:
output_file.write(chunk)
else:
break
"""
output_file.write(xlsx_output_file.read())
elif export_type == 'spss_labels':
export.to_spss_labels(output_file)
# Restore the FileField to its typical state
self.result.open('rb')
self.save(update_fields=['last_submission_time'])
# Now that a new export has completed successfully, remove any old
# exports in excess of the per-user, per-form limit
self.remove_excess(self.user, source_url)
