def test_archive_forms_wrong_domain(self):
uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX))
response = archive_forms('wrong_domain', self.user, list(uploaded_file.get_worksheet()))
self.assertEqual(len(response['errors']), len(self.xforms), "Error when wrong domain")
def upload_fixture_api(request, domain, **kwargs):
"""
Use following curl-command to test.
> curl -v --digest http://127.0.0.1:8000/a/gsid/fixtures/fixapi/ -u [email protected]:password
-F "file-to-upload=@hqtest_fixtures.xlsx"
-F "replace=true"
"""
response_codes = {"fail": 405, "warning": 402, "success": 200}
error_messages = {
"invalid_post_req":
"Invalid post request. Submit the form with field 'file-to-upload' and POST parameter 'replace'",
"has_no_permission":
"User {attr} doesn't have permission to upload fixtures",
"invalid_file":
"Error processing your file. Submit a valid (.xlsx) file",
"has_no_sheet":
"Workbook does not have a sheet called {attr}",
"unknown_fail":
"Fixture upload couldn't succeed due to the following error: {attr}",
}
def _return_response(code, message):
resp_json = {}
resp_json["code"] = code
resp_json["message"] = message
return HttpResponse(json.dumps(resp_json), mimetype="application/json")
try:
upload_file = request.FILES["file-to-upload"]
replace = request.POST["replace"]
if replace.lower() == "true":
replace = True
elif replace.lower() == "false":
replace = False
except Exception:
return _return_response(response_codes["fail"],
error_messages["invalid_post_req"])
if not request.couch_user.has_permission(domain,
Permissions.edit_data.name):
error_message = error_messages["has_no_permission"].format(
attr=request.couch_user.username)
return _return_response(response_codes["fail"], error_message)
try:
workbook = WorkbookJSONReader(upload_file)
except Exception:
return _return_response(response_codes["fail"],
error_messages["invalid_file"])
try:
upload_resp = run_upload(
request, domain, workbook,
replace=replace) # error handle for other files
except WorksheetNotFound as e:
error_message = error_messages["has_no_sheet"].format(attr=e.title)
return _return_response(response_codes["fail"], error_message)
except ExcelMalformatException as e:
notify_exception(request)
return _return_response(response_codes["fail"], str(e))
except DuplicateFixtureTagException as e:
return _return_response(response_codes["fail"], str(e))
except FixtureAPIException as e:
return _return_response(response_codes["fail"], str(e))
except Exception as e:
notify_exception(request, message=e)
error_message = error_messages["unknown_fail"].format(attr=e)
return _return_response(response_codes["fail"], error_message)
num_unknown_groups = len(upload_resp["unknown_groups"])
num_unknown_users = len(upload_resp["unknown_users"])
resp_json = {}
if not num_unknown_users and not num_unknown_groups:
num_uploads = upload_resp["number_of_fixtures"]
success_message = "Successfully uploaded %d fixture%s." % (
num_uploads, 's' if num_uploads > 1 else '')
return _return_response(response_codes["success"], success_message)
else:
resp_json["code"] = response_codes["warning"]
warn_groups = "%d group%s unknown" % (num_unknown_groups, 's are'
if num_unknown_groups > 1 else ' is')
warn_users = "%d user%s unknown" % (num_unknown_users, 's are'
if num_unknown_users > 1 else ' is')
resp_json["message"] = "Fixtures have been uploaded. But following "
if num_unknown_groups:
resp_json["message"] += "%s %s" % (warn_groups,
upload_resp["unknown_groups"])
if num_unknown_users:
resp_json["message"] += "%s%s%s" % (
("and following " if num_unknown_groups else ""), warn_users,
upload_resp["unknown_users"])
return HttpResponse(json.dumps(resp_json), mimetype="application/json")
def post(self, request, *args, **kwargs):
"""View's dispatch method automatically calls this"""
upload = request.FILES.get('bulk_upload_file')
try:
self.workbook = WorkbookJSONReader(upload)
except InvalidFileException:
try:
csv.DictReader(
io.StringIO(upload.read().decode('ascii'), newline=None))
return HttpResponseBadRequest(
"CommCare HQ no longer supports CSV upload. "
"Please convert to Excel 2007 or higher (.xlsx) "
"and try again.")
except UnicodeDecodeError:
return HttpResponseBadRequest("Unrecognized format")
except JSONReaderError as e:
messages.error(request,
'Your upload was unsuccessful. %s' % e.message)
return self.get(request, *args, **kwargs)
except HeaderValueError as e:
return HttpResponseBadRequest(
"Upload encountered a data type error: %s" % e.message)
try:
self.user_specs = self.workbook.get_worksheet(title='users')
except WorksheetNotFound:
try:
self.user_specs = self.workbook.get_worksheet()
except WorksheetNotFound:
return HttpResponseBadRequest("Workbook has no worksheets")
try:
self.group_specs = self.workbook.get_worksheet(title='groups')
except WorksheetNotFound:
self.group_specs = []
self.location_specs = []
if Domain.get_by_name(self.domain).commtrack_enabled:
try:
self.location_specs = self.workbook.get_worksheet(
title='locations')
except WorksheetNotFound:
# if there is no sheet for locations (since this was added
# later and is optional) we don't error
pass
try:
check_headers(self.user_specs)
except UserUploadError as e:
messages.error(request, _(e.message))
return HttpResponseRedirect(
reverse(UploadCommCareUsers.urlname, args=[self.domain]))
task_ref = expose_cached_download(None, expiry=1 * 60 * 60)
task = bulk_upload_async.delay(self.domain, list(self.user_specs),
list(self.group_specs),
list(self.location_specs))
task_ref.set_task(task)
return HttpResponseRedirect(
reverse(UserUploadStatusView.urlname,
args=[self.domain, task_ref.download_id]))
def post(self, request):
"""View's dispatch method automatically calls this"""
def error_redirect():
return HttpResponseRedirect(
reverse("fixture_interface_dispatcher",
args=[],
kwargs={
'domain': self.domain,
'report_slug': 'edit_lookup_tables'
}))
try:
replace = request.POST["replace"]
replace = True
except KeyError:
replace = False
try:
workbook = WorkbookJSONReader(request.file)
except AttributeError:
messages.error(request,
_("Error processing your Excel (.xlsx) file"))
return error_redirect()
except Exception as e:
messages.error(
request,
_("Invalid file-format. Please upload a valid xlsx file."))
return error_redirect()
try:
upload_result = run_upload(request,
self.domain,
workbook,
replace=replace)
if upload_result["unknown_groups"]:
for group_name in upload_result["unknown_groups"]:
messages.error(
request,
_("Unknown group: '%(name)s'") % {'name': group_name})
if upload_result["unknown_users"]:
for user_name in upload_result["unknown_users"]:
messages.error(
request,
_("Unknown user: '******'") % {'name': user_name})
except WorksheetNotFound as e:
messages.error(
request,
_("Workbook does not contain a sheet called '%(title)s'") %
{'title': e.title})
return error_redirect()
except ExcelMalformatException as e:
messages.error(
request,
_("Uploaded excel file has following formatting-problems: '%(e)s'"
) % {'e': e})
return error_redirect()
except DuplicateFixtureTagException as e:
messages.error(request, e)
except FixtureAPIException as e:
messages.error(request, _(str(e)))
return error_redirect()
except Exception as e:
notify_exception(request, message=str(e))
messages.error(request, str(e))
messages.error(
request,
_("Fixture upload failed for some reason and we have noted this failure. "
"Please make sure the excel file is correctly formatted and try again."
))
return error_redirect()
return HttpResponseRedirect(
reverse("fixture_interface_dispatcher",
args=[],
kwargs={
'domain': self.domain,
'report_slug': 'edit_lookup_tables'
}))
def clean_message_bank_file(self):
value = self.cleaned_data.get("message_bank_file")
if not value:
raise ValidationError(_("Please choose a file."))
try:
workbook = WorkbookJSONReader(value)
except InvalidFileException:
raise ValidationError(
_("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again."
))
try:
worksheet = workbook.get_worksheet()
except WorksheetNotFound:
raise ValidationError(_("Workbook has no worksheets."))
message_ids = {}
messages = []
row_num = 2
for row in worksheet:
if "ID" not in row:
raise ValidationError(_("Column 'ID' not found."))
if "Message" not in row:
raise ValidationError(_("Column 'Message' not found."))
msg_id = row.get("ID")
text = row.get("Message")
try:
assert isinstance(msg_id, basestring)
msg_id = msg_id.strip()
assert len(msg_id) > 1
assert msg_id[0].upper() in "ABCDEFGH"
except Exception:
raise ValidationError(
_("Invalid ID at row %(row_num)s") % {"row_num": row_num})
if msg_id in message_ids:
raise ValidationError(
_("Duplicate ID at row %(row_num)s") %
{"row_num": row_num})
try:
assert isinstance(text, basestring)
text = text.strip()
assert len(text) > 0
except Exception:
raise ValidationError(
_("Invalid Message at row %(row_num)s") %
{"row_num": row_num})
try:
msg_id.encode("ascii")
except Exception:
raise ValidationError(
_("ID at row %(row_num)s contains invalid character(s)") %
{"row_num": row_num})
try:
text.encode("ascii")
except Exception:
raise ValidationError(
_("Message at row %(row_num)s contains invalid character(s)"
) % {"row_num": row_num})
if len(text) > 160:
raise ValidationError(
_("Message at row %(row_num)s is longer than 160 characters."
) % {"row_num": row_num})
messages.append({
"msg_id": msg_id,
"text": text,
})
message_ids[msg_id] = True
row_num += 1
return messages
