def test_corrupted_submission(self):
"""Test xml submissions that contain unicode characters.
"""
xml = 'v\xee\xf3\xc0k\x91\x91\xae\xff\xff\xff\xff\xcf[$b\xd0\xc9\'uW\x80RP\xff\xff\xff\xff7\xd0\x03%F\xa7p\xa2\x87\xb6f\xb1\xff\xff\xff\xffg~\xf3O\xf3\x9b\xbc\xf6ej_$\xff\xff\xff\xff\x13\xe8\xa9D\xed\xfb\xe7\xa4d\x96>\xfa\xff\xff\xff\xff\xc7h"\x86\x14\\.\xdb\x8aoF\xa4\xff\xff\xff\xff\xcez\xff\x01\x0c\x9a\x94\x18\xe1\x03\x8e\xfa\xff\xff\xff\xff39P|\xf9n\x18F\xb1\xcb\xacd\xff\xff\xff\xff\xce>\x97i;1u\xcfI*\xf2\x8e\xff\xff\xff\xffFg\x9d\x0fR:\xcd*\x14\x85\xf0e\xff\xff\xff\xff\xd6\xdc\xda\x8eM\x06\xf1\xfc\xc1\xe8\xd6\xe0\xff\xff\xff\xff\xe7G\xe1\xa1l\x02T\n\xde\x1boJ\xff\xff\xff\xffz \x92\xbc\tR{#\xbb\x9f\xa6s\xff\xff\xff\xff\xa2\x8f(\xb6=\xe11\xfcV\xcf\xef\x0b\xff\xff\xff\xff\xa3\x83\x7ft\xd7\x05+)\xeb9\\*\xff\xff\xff\xff\xfe\x93\xb2\xa2\x06n;\x1b4\xaf\xa6\x93\xff\xff\xff\xff\xe7\xf7\x12Q\x83\xbb\x9a\xc8\xc8q34\xff\xff\xff\xffT2\xa5\x07\x9a\xc9\x89\xf8\x14Y\xab\x19\xff\xff\xff\xff\x16\xd0R\x1d\x06B\x95\xea\\\x1ftP\xff\xff\xff\xff\x94^\'\x01#oYV\xc5\\\xb7@\xff\xff\xff\xff !\x11\x00\x8b\xf3[\xde\xa2\x01\x9dl\xff\xff\xff\xff\xe7z\x92\xc3\x03\xd3\xb5B5 \xaa7\xff\xff\xff\xff\xff\xc3Q:\xa6\xb3\xa3\x1e\x90 \xa0\\\xff\xff\xff\xff\xff\x14<\x03Vr\xe8Z.Ql\xf5\xff\xff\xff\xffEx\xf7\x0b_\xa1\x7f\xfcG\xa4\x18\xcd\xff\xff\xff\xff1|~i\x00\xb3. ,1Q\x0e\xff\xff\xff\xff\x87a\x933Y\xd7\xe1B#\xa7a\xee\xff\xff\xff\xff\r\tJ\x18\xd0\xdb\x0b\xbe\x00\x91\x95\x9e\xff\xff\xff\xffHfW\xcd\x8f\xa9z6|\xc5\x171\xff\xff\xff\xff\xf5tP7\x93\x02Q|x\x17\xb1\xcb\xff\xff\xff\xffVb\x11\xa0*\xd9;\x0b\xf8\x1c\xd3c\xff\xff\xff\xff\x84\x82\xcer\x15\x99`5LmA\xd5\xff\xff\xff\xfft\xce\x8e\xcbw\xee\xf3\xc0w\xca\xb3\xfd\xff\xff\xff\xff\xb0\xaab\x92\xd4\x02\x84H3\x94\xa9~\xff\xff\xff\xff\xfe7\x18\xcaW=\x94\xbc|\x0f{\x84\xff\xff\xff\xff\xe8\xdf\xde?\x8b\xb7\x9dH3\xc1\xf2\xaa\xff\xff\xff\xff\xbe\x00\xba\xd7\xba6!\x95g\xb01\xf9\xff\xff\xff\xff\x93\xe3\x90YH9g\xf7\x97nhv\xff\xff\xff\xff\x82\xc7`\xaebn\x9d\x1e}\xba\x1e/\xff\xff\xff\xff\xbd\xe5\xa1\x05\x03\xf26\xa0\xe2\xc1*\x07\xff\xff\xff\xffny\x88\x9f\x19\xd2\xd0\xf7\x1de\xa7\xe0\xff\xff\xff\xff\xc4O&\x14\x8dVH\x90\x8b+\x03\xf9\xff\xff\xff\xff\xf69\xc2\xabo%\xcc/\xc9\xe4dP\xff\xff\xff\xff (\x08G\xebM\x03\x99Y\xb4\xb3\x1f\xff\xff\xff\xffzH\xd2\x19p#\xc5\xa4)\xfd\x05\x9a\xff\xff\xff\xffd\x86\xb2F\x15\x0f\xf4.\xfd\\\xd4#\xff\xff\xff\xff\xaf\xbe\xc6\x9di\xa0\xbc\xd5>cp\xe2\xff\xff\xff\xff&h\x91\xe9\xa0H\xdd\xaer\x87\x18E\xff\xff\xff\xffjg\x08E\x8f\xa4&\xab\xff\x98\x0ei\xff\xff\xff\xff\x01\xfd{"\xed\\\xa3M\x9e\xc3\xf8K\xff\xff\xff\xff\x87Y\x98T\xf0\xa6\xec\x98\xb3\xef\xa7\xaa\xff\xff\xff\xffA\xced\xfal\xd3\xd9\x06\xc6~\xee}\xff\xff\xff\xff:\x7f\xa2\x10\xc7\xadB,}PF%\xff\xff\xff\xff\xb2\xbc\n\x17%\x98\x904\x89\tF\x1f\xff\xff\xff\xff\xdc\xd8\xc6@#M\x87uf\x02\xc6g\xff\xff\xff\xffK\xaf\xb0-=l\x07\xe1Nv\xe4\xf4\xff\xff\xff\xff\xdb\x13\'Ne\xb2UT\x9a#\xb1^\xff\xff\xff\xff\xb2\rne\xd1\x9d\x88\xda\xbb!\xfa@\xff\xff\xff\xffflq\x0f\x01z]uh\'|?\xff\xff\xff\xff\xd5\'\x19\x865\xba\xf2\xe7\x8fR-\xcc\xff\xff\xff\xff\xce\xd6\xfdi\x04\x9b\xa7\tu\x05\xb7\xc8\xff\xff\xff\xff\xc3\xd0)\x11\xdd\xb1\xa5kp\xc9\xd5\xf7\xff\xff\xff\xff\xffU\x9f \xb7\xa1#3rup[\xff\xff\xff\xff\xfc=' # noqa
request = RequestFactory().post('/')
request.user = self.user
error, instance = safe_create_instance(
self.user.username, TempFileProxy(xml), None, None, request)
text = 'File likely corrupted during transmission'
self.assertContains(error, text, status_code=400)
def create_instance_from_json(username, request):
request.accepted_renderer = JSONRenderer()
request.accepted_media_type = JSONRenderer.media_type
dict_form = request.DATA
submission = dict_form.get('submission')
if submission is None:
# return an error
return [_(u"No submission key provided."), None]
# convert lists in submission dict to joined strings
submission_joined = dict_lists2strings(submission)
xml_string = dict2xform(submission_joined, dict_form.get('id'))
xml_file = StringIO.StringIO(xml_string)
return safe_create_instance(username, xml_file, [], None, request)
def create_instance_from_json(username, request):
request.accepted_renderer = JSONRenderer()
request.accepted_media_type = JSONRenderer.media_type
dict_form = request.DATA
submission = dict_form.get('submission')
if submission is None:
# return an error
return [_(u"No submission key provided."), None]
# convert lists in submission dict to joined strings
submission_joined = dict_lists2strings(submission)
xml_string = dict2xform(submission_joined, dict_form.get('id'))
xml_file = StringIO.StringIO(xml_string)
return safe_create_instance(username, xml_file, [], None, request)
def create_instance_from_xml(username, request):
xml_file_list = request.FILES.pop('xml_submission_file', [])
xml_file = xml_file_list[0] if len(xml_file_list) else None
media_files = request.FILES.values()
return safe_create_instance(username, xml_file, media_files, None, request)
def submit_csv(username, xform, csv_file):
""" Imports CSV data to an existing form
Takes a csv formatted file or string containing rows of submission/instance
and converts those to xml submissions and finally submits them by calling
:py:func:`formshare.libs.utils.logger_tools.safe_create_instance`
:param str username: the subission user
:param formshare.apps.logger.models.XForm xfrom: The submission's XForm.
:param (str or file): A CSV formatted file with submission rows.
:return: If sucessful, a dict with import summary else dict with error str.
:rtype: Dict
"""
if isinstance(csv_file, unicode):
csv_file = cStringIO.StringIO(csv_file)
elif csv_file is None or not hasattr(csv_file, 'read'):
return {'error': (u'Invalid param type for `csv_file`. '
'Expected utf-8 encoded file or unicode string '
'got {} instead.'.format(type(csv_file).__name__))}
num_rows = sum(1 for row in csv_file) - 1
csv_file.seek(0)
csv_reader = ucsv.DictReader(csv_file)
# check for spaces in headers
if any(' ' in header for header in csv_reader.fieldnames):
return {'error': u'CSV file fieldnames should not contain spaces'}
rollback_uuids = []
submission_time = datetime.utcnow().isoformat()
ona_uuid = {'formhub': {'uuid': xform.uuid}}
error = None
additions = inserts = 0
try:
for row in csv_reader:
# fetch submission uuid before purging row metadata
row_uuid = row.get('_uuid')
submitted_by = row.get('_submitted_by')
submission_date = row.get('_submission_time', submission_time)
location_data = {}
for key in row.keys(): # seems faster than a comprehension
# remove metadata (keys starting with '_')
if key.startswith('_'):
del row[key]
# Collect row location data into separate location_data dict
if key.endswith(('.latitude', '.longitude',
'.altitude', '.precision')):
location_key, location_prop = key.rsplit(u'.', 1)
location_data.setdefault(location_key, {}).update(
{location_prop: row.get(key, '0')})
# collect all location K-V pairs into single geopoint field(s)
# in location_data dict
for location_key in location_data.keys():
location_data.update(
{location_key:
(u'%(latitude)s %(longitude)s '
'%(altitude)s %(precision)s') % defaultdict(
lambda: '', location_data.get(location_key))})
row = dict_pathkeys_to_nested_dicts(row)
location_data = dict_pathkeys_to_nested_dicts(location_data)
row = dict_merge(row, location_data)
# inject our form's uuid into the submission
row.update(ona_uuid)
old_meta = row.get('meta', {})
new_meta, update = get_submission_meta_dict(xform, row_uuid)
inserts += update
old_meta.update(new_meta)
row.update({'meta': old_meta})
row_uuid = row.get('meta').get('instanceID')
rollback_uuids.append(row_uuid.replace('uuid:', ''))
xml_file = cStringIO.StringIO(
dict2xmlsubmission(row, xform, row_uuid, submission_date))
try:
error, instance = safe_create_instance(username, xml_file, [],
xform.uuid, None)
except ValueError as e:
error = e
if error:
Instance.objects.filter(uuid__in=rollback_uuids,
xform=xform).delete()
return {'error': str(error)}
else:
additions += 1
try:
current_task.update_state(state='PROGRESS',
meta={'progress': additions,
'total': num_rows})
except:
pass
users = User.objects.filter(
username=submitted_by) if submitted_by else []
if users:
instance.user = users[0]
instance.save()
except UnicodeDecodeError:
Instance.objects.filter(uuid__in=rollback_uuids,
xform=xform).delete()
return {'error': u'CSV file must be utf-8 encoded'}
except Exception as e:
Instance.objects.filter(uuid__in=rollback_uuids,
xform=xform).delete()
return {'error': str(e)}
return {'additions': additions - inserts, 'updates': inserts}
def submission(request, username=None):
if username:
formlist_user = get_object_or_404(User, username__iexact=username)
profile, created = UserProfile.objects.get_or_create(
user=formlist_user)
if profile.require_auth:
authenticator = HttpDigestAuthenticator()
if not authenticator.authenticate(request):
return authenticator.build_challenge_response()
if request.method == 'HEAD':
response = OpenRosaResponse(status=204)
if username:
response['Location'] = request.build_absolute_uri().replace(
request.get_full_path(), '/%s/submission' % username)
else:
response['Location'] = request.build_absolute_uri().replace(
request.get_full_path(), '/submission')
return response
xml_file_list = []
media_files = []
# request.FILES is a django.utils.datastructures.MultiValueDict
# for each key we have a list of values
try:
xml_file_list = request.FILES.pop("xml_submission_file", [])
if len(xml_file_list) != 1:
return OpenRosaResponseBadRequest(
_(u"There should be a single XML submission file."))
# save this XML file and media files as attachments
media_files = request.FILES.values()
# get uuid from post request
uuid = request.POST.get('uuid')
error, instance = safe_create_instance(username, xml_file_list[0],
media_files, uuid, request)
if error:
return error
elif instance is None:
return OpenRosaResponseBadRequest(
_(u"Unable to create submission."))
audit = {"xform": instance.xform.id_string}
audit_log(
Actions.SUBMISSION_CREATED, request.user, instance.xform.user,
_("Created submission on form %(id_string)s.") %
{"id_string": instance.xform.id_string}, audit, request)
# response as html if posting with a UUID
if not username and uuid:
response = _html_submission_response(request, instance)
else:
response = _submission_response(request, instance)
# ODK needs two things for a form to be considered successful
# 1) the status code needs to be 201 (created)
# 2) The location header needs to be set to the host it posted to
response.status_code = 201
response['Location'] = request.build_absolute_uri(request.path)
return response
except IOError as e:
if _bad_request(e):
return OpenRosaResponseBadRequest(
_(u"File transfer interruption."))
else:
raise
finally:
if len(xml_file_list):
[_file.close() for _file in xml_file_list]
if len(media_files):
[_file.close() for _file in media_files]
def submission(request, username=None):
if username:
formlist_user = get_object_or_404(User, username__iexact=username)
profile, created = UserProfile.objects.get_or_create(
user=formlist_user)
if profile.require_auth:
authenticator = HttpDigestAuthenticator()
if not authenticator.authenticate(request):
return authenticator.build_challenge_response()
if request.method == 'HEAD':
response = OpenRosaResponse(status=204)
if username:
response['Location'] = request.build_absolute_uri().replace(
request.get_full_path(), '/%s/submission' % username)
else:
response['Location'] = request.build_absolute_uri().replace(
request.get_full_path(), '/submission')
return response
xml_file_list = []
media_files = []
# request.FILES is a django.utils.datastructures.MultiValueDict
# for each key we have a list of values
try:
xml_file_list = request.FILES.pop("xml_submission_file", [])
if len(xml_file_list) != 1:
return OpenRosaResponseBadRequest(
_(u"There should be a single XML submission file.")
)
# save this XML file and media files as attachments
media_files = request.FILES.values()
# get uuid from post request
uuid = request.POST.get('uuid')
error, instance = safe_create_instance(
username, xml_file_list[0], media_files, uuid, request)
if error:
return error
elif instance is None:
return OpenRosaResponseBadRequest(
_(u"Unable to create submission."))
audit = {
"xform": instance.xform.id_string
}
audit_log(
Actions.SUBMISSION_CREATED, request.user, instance.xform.user,
_("Created submission on form %(id_string)s.") %
{
"id_string": instance.xform.id_string
}, audit, request)
# response as html if posting with a UUID
if not username and uuid:
response = _html_submission_response(request, instance)
else:
response = _submission_response(request, instance)
# ODK needs two things for a form to be considered successful
# 1) the status code needs to be 201 (created)
# 2) The location header needs to be set to the host it posted to
response.status_code = 201
response['Location'] = request.build_absolute_uri(request.path)
return response
except IOError as e:
if _bad_request(e):
return OpenRosaResponseBadRequest(
_(u"File transfer interruption."))
else:
raise
finally:
if len(xml_file_list):
[_file.close() for _file in xml_file_list]
if len(media_files):
[_file.close() for _file in media_files]
def submit_csv(username, xform, csv_file):
""" Imports CSV data to an existing form
Takes a csv formatted file or string containing rows of submission/instance
and converts those to xml submissions and finally submits them by calling
:py:func:`formshare.libs.utils.logger_tools.safe_create_instance`
:param str username: the subission user
:param formshare.apps.logger.models.XForm xfrom: The submission's XForm.
:param (str or file): A CSV formatted file with submission rows.
:return: If sucessful, a dict with import summary else dict with error str.
:rtype: Dict
"""
if isinstance(csv_file, unicode):
csv_file = cStringIO.StringIO(csv_file)
elif csv_file is None or not hasattr(csv_file, 'read'):
return {
'error': (u'Invalid param type for `csv_file`. '
'Expected utf-8 encoded file or unicode string '
'got {} instead.'.format(type(csv_file).__name__))
}
num_rows = sum(1 for row in csv_file) - 1
csv_file.seek(0)
csv_reader = ucsv.DictReader(csv_file)
# check for spaces in headers
if any(' ' in header for header in csv_reader.fieldnames):
return {'error': u'CSV file fieldnames should not contain spaces'}
rollback_uuids = []
submission_time = datetime.utcnow().isoformat()
ona_uuid = {'formhub': {'uuid': xform.uuid}}
error = None
additions = inserts = 0
try:
for row in csv_reader:
# fetch submission uuid before purging row metadata
row_uuid = row.get('_uuid')
submitted_by = row.get('_submitted_by')
submission_date = row.get('_submission_time', submission_time)
location_data = {}
for key in row.keys(): # seems faster than a comprehension
# remove metadata (keys starting with '_')
if key.startswith('_'):
del row[key]
# Collect row location data into separate location_data dict
if key.endswith(
('.latitude', '.longitude', '.altitude', '.precision')):
location_key, location_prop = key.rsplit(u'.', 1)
location_data.setdefault(location_key, {}).update(
{location_prop: row.get(key, '0')})
# collect all location K-V pairs into single geopoint field(s)
# in location_data dict
for location_key in location_data.keys():
location_data.update({
location_key: (u'%(latitude)s %(longitude)s '
'%(altitude)s %(precision)s') %
defaultdict(lambda: '', location_data.get(location_key))
})
row = dict_pathkeys_to_nested_dicts(row)
location_data = dict_pathkeys_to_nested_dicts(location_data)
row = dict_merge(row, location_data)
# inject our form's uuid into the submission
row.update(ona_uuid)
old_meta = row.get('meta', {})
new_meta, update = get_submission_meta_dict(xform, row_uuid)
inserts += update
old_meta.update(new_meta)
row.update({'meta': old_meta})
row_uuid = row.get('meta').get('instanceID')
rollback_uuids.append(row_uuid.replace('uuid:', ''))
xml_file = cStringIO.StringIO(
dict2xmlsubmission(row, xform, row_uuid, submission_date))
try:
error, instance = safe_create_instance(username, xml_file, [],
xform.uuid, None)
except ValueError as e:
error = e
if error:
Instance.objects.filter(uuid__in=rollback_uuids,
xform=xform).delete()
return {'error': str(error)}
else:
additions += 1
try:
current_task.update_state(state='PROGRESS',
meta={
'progress': additions,
'total': num_rows
})
except:
pass
users = User.objects.filter(
username=submitted_by) if submitted_by else []
if users:
instance.user = users[0]
instance.save()
except UnicodeDecodeError:
Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete()
return {'error': u'CSV file must be utf-8 encoded'}
except Exception as e:
Instance.objects.filter(uuid__in=rollback_uuids, xform=xform).delete()
return {'error': str(e)}
return {'additions': additions - inserts, 'updates': inserts}
def create_instance_from_xml(username, request):
xml_file_list = request.FILES.pop('xml_submission_file', [])
xml_file = xml_file_list[0] if len(xml_file_list) else None
media_files = request.FILES.values()
return safe_create_instance(username, xml_file, media_files, None, request)