def update_domains(request):
if request.method == "POST":
try:
workbook = WorkbookJSONReader(request.file)
domains = workbook.get_worksheet(title='domains')
success_count = 0
fail_count = 0
for row in domains:
try:
name = row["name"]
domain = Domain.get_by_name(name)
if domain:
for k, v in row.items():
setattr(domain, k, v)
domain.save()
success_count += 1
else:
messages.warning(request, "No domain with name %s found" % name)
fail_count += 1
except Exception, e:
messages.warning(request, "Update for %s failed: %s" % (row.get("name", '<No Name>'), e))
fail_count += 1
if success_count:
messages.success(request, "%s domains successfully updated" % success_count)
if fail_count:
messages.error(request, "%s domains had errors. details above." % fail_count)
except Exception, e:
messages.error(request, "Something went wrong! Update failed. Here's your error: %s" % e)
def clean_contact_upload_file(self):
value = self.cleaned_data.get("contact_upload_file", None)
if self.cleaned_data.get("use_contact_upload_file", "N") == "Y":
if value is None:
raise ValidationError("Please choose a file.")
try:
workbook = WorkbookJSONReader(value)
except InvalidFileException:
raise ValidationError(
"Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again."
)
try:
worksheet = workbook.get_worksheet()
except WorksheetNotFound:
raise ValidationError("Workbook has no worksheets.")
contacts = []
for row in worksheet:
if "PhoneNumber" not in row:
raise ValidationError("Column 'PhoneNumber' not found.")
contacts.append({
"phone_number":
validate_phone_number(row.get("PhoneNumber"))
})
if len(contacts) == 0:
raise ValidationError(_("Please add at least one contact."))
return contacts
else:
return None
def test_archive_forms_basic(self):
uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX))
response = archive_forms(self.domain_name, self.user, list(uploaded_file.get_worksheet()))
# Need to re-get instance from DB to get updated attributes
for key, _id in self.XFORMS.iteritems():
self.assertEqual(XFormInstance.get(_id).doc_type, 'XFormArchived')
self.assertEqual(len(response['success']), len(self.xforms))
class FixtureWorkbook(object):
"""
Helper class for working with the fixture workbook
"""
def __init__(self, file_or_filename):
try:
self.workbook = WorkbookJSONReader(file_or_filename)
except AttributeError:
raise FixtureUploadError(
_("Error processing your Excel (.xlsx) file"))
except Exception:
raise FixtureUploadError(
_("Invalid file-format. Please upload a valid xlsx file."))
def get_types_sheet(self):
try:
return self.workbook.get_worksheet(title='types')
except WorksheetNotFound as e:
raise FixtureUploadError(
_("Workbook does not contain a sheet called '%(title)s'") %
{'title': e.title})
def get_data_sheet(self, data_type):
return self.workbook.get_worksheet(data_type.tag)
def get_all_type_sheets(self):
type_sheets = []
seen_tags = set()
for number_of_fixtures, dt in enumerate(self.get_types_sheet()):
table_definition = FixtureTableDefinition.from_row(dt)
if table_definition.table_id in seen_tags:
raise DuplicateFixtureTagException(
_(FAILURE_MESSAGES['duplicate_tag']).format(
tag=table_definition.table_id))
seen_tags.add(table_definition.table_id)
type_sheets.append(table_definition)
return type_sheets
def validate(self):
self.get_types_sheet()
self.get_all_type_sheets()
def test_archive_forms_missing(self):
uploaded_file = WorkbookJSONReader(join(BASE_PATH, MISSING_XLSX))
response = archive_forms(self.domain_name, self.user, list(uploaded_file.get_worksheet()))
for key, _id in self.XFORMS.iteritems():
self.assertEqual(XFormInstance.get(_id).doc_type, 'XFormArchived')
self.assertEqual(len(response['success']), len(self.xforms))
self.assertEqual(len(response['errors']), 1,
"One error for trying to archive a missing form")
class UploadCommCareUsers(TemplateView):
template_name = 'users/upload_commcare_users.html'
def get_context_data(self, **kwargs):
"""TemplateView automatically calls this to render the view (on a get)"""
context = _users_context(self.request, self.domain)
context["show_secret_settings"] = self.request.REQUEST.get("secret", False)
return context
@method_decorator(get_file)
def post(self, request):
"""View's dispatch method automatically calls this"""
redirect = request.POST.get('redirect')
try:
self.workbook = WorkbookJSONReader(request.file)
except InvalidFileException:
try:
csv.DictReader(io.StringIO(request.file.read().decode('ascii'),
newline=None))
return HttpResponseBadRequest(
"CommCare HQ no longer supports CSV upload. "
"Please convert to Excel 2007 or higher (.xlsx) "
"and try again."
)
except UnicodeDecodeError:
return HttpResponseBadRequest("Unrecognized format")
except JSONReaderError as e:
messages.error(request,
'Your upload was unsuccessful. %s' % e.message)
return HttpResponseRedirect(redirect)
try:
self.user_specs = self.workbook.get_worksheet(title='users')
except WorksheetNotFound:
try:
self.user_specs = self.workbook.get_worksheet()
except WorksheetNotFound:
return HttpResponseBadRequest("Workbook has no worksheets")
try:
self.group_specs = self.workbook.get_worksheet(title='groups')
except WorksheetNotFound:
self.group_specs = []
try:
check_headers(self.user_specs)
except UserUploadError as e:
return HttpResponseBadRequest(e)
response = HttpResponse()
response_rows = []
async = request.REQUEST.get("async", False)
if async:
download_id = uuid.uuid4().hex
bulk_upload_async.delay(download_id, self.domain,
list(self.user_specs),
list(self.group_specs))
messages.success(request,
'Your upload is in progress. You can check the progress <a href="%s">here</a>.' %\
reverse('hq_soil_download', kwargs={'domain': self.domain, 'download_id': download_id}),
extra_tags="html")
else:
ret = create_or_update_users_and_groups(self.domain, self.user_specs, self.group_specs)
for error in ret["errors"]:
messages.error(request, error)
for row in ret["rows"]:
response_rows.append(row)
if redirect:
if not async:
messages.success(request,
_('Your bulk user upload is complete!'))
problem_rows = []
for row in response_rows:
if row['flag'] not in ('updated', 'created'):
problem_rows.append(row)
if problem_rows:
messages.error(
request,
_('However, we ran into problems with the following users:')
)
for row in problem_rows:
if row['flag'] == 'missing-data':
messages.error(request,
_('A row with no username was skipped'))
else:
messages.error(request,
'{username}: {flag}'.format(**row))
return HttpResponseRedirect(redirect)
else:
return response
@method_decorator(require_can_edit_commcare_users)
def dispatch(self, request, domain, *args, **kwargs):
self.domain = domain
return super(UploadCommCareUsers, self).dispatch(request, *args, **kwargs)
class UploadCommCareUsers(BaseManageCommCareUserView):
template_name = 'users/upload_commcare_users.html'
urlname = 'upload_commcare_users'
page_title = ugettext_noop("Bulk Upload Mobile Workers")
@method_decorator(requires_privilege_with_fallback(privileges.BULK_USER_MANAGEMENT))
def dispatch(self, request, *args, **kwargs):
return super(UploadCommCareUsers, self).dispatch(request, *args, **kwargs)
@property
def page_context(self):
context = {
'bulk_upload': {
"help_site": {
"address": BULK_MOBILE_HELP_SITE,
"name": _("CommCare Help Site"),
},
"download_url": reverse(
"download_commcare_users", args=(self.domain,)),
"adjective": _("mobile worker"),
"plural_noun": _("mobile workers"),
},
'show_secret_settings': self.request.REQUEST.get("secret", False),
}
context.update({
'bulk_upload_form': get_bulk_upload_form(context),
})
return context
def post(self, request, *args, **kwargs):
upload = request.FILES.get('bulk_upload_file')
"""View's dispatch method automatically calls this"""
try:
self.workbook = WorkbookJSONReader(upload)
except InvalidFileException:
try:
csv.DictReader(io.StringIO(upload.read().decode('ascii'),
newline=None))
return HttpResponseBadRequest(
"CommCare HQ no longer supports CSV upload. "
"Please convert to Excel 2007 or higher (.xlsx) "
"and try again."
)
except UnicodeDecodeError:
return HttpResponseBadRequest("Unrecognized format")
except JSONReaderError as e:
messages.error(request,
'Your upload was unsuccessful. %s' % e.message)
return self.get(request, *args, **kwargs)
except HeaderValueError as e:
return HttpResponseBadRequest("Upload encountered a data type error: %s"
% e.message)
try:
self.user_specs = self.workbook.get_worksheet(title='users')
except WorksheetNotFound:
try:
self.user_specs = self.workbook.get_worksheet()
except WorksheetNotFound:
return HttpResponseBadRequest("Workbook has no worksheets")
try:
self.group_specs = self.workbook.get_worksheet(title='groups')
except WorksheetNotFound:
self.group_specs = []
self.location_specs = []
if Domain.get_by_name(self.domain).commtrack_enabled:
try:
self.location_specs = self.workbook.get_worksheet(title='locations')
except WorksheetNotFound:
# if there is no sheet for locations (since this was added
# later and is optional) we don't error
pass
try:
check_headers(self.user_specs)
except UserUploadError as e:
return HttpResponseBadRequest(e)
task_ref = expose_download(None, expiry=1*60*60)
task = bulk_upload_async.delay(
self.domain,
list(self.user_specs),
list(self.group_specs),
list(self.location_specs)
)
task_ref.set_task(task)
return HttpResponseRedirect(
reverse(
UserUploadStatusView.urlname,
args=[self.domain, task_ref.download_id]
)
)
def test_archive_forms_wrong_domain(self):
uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX))
response = archive_forms('wrong_domain', self.user, list(uploaded_file.get_worksheet()))
self.assertEqual(len(response['errors']), len(self.xforms), "Error when wrong domain")
def clean_message_bank_file(self):
value = self.cleaned_data.get("message_bank_file")
if not value:
raise ValidationError(_("Please choose a file."))
try:
workbook = WorkbookJSONReader(value)
except InvalidFileException:
raise ValidationError(
_("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again."
))
try:
worksheet = workbook.get_worksheet()
except WorksheetNotFound:
raise ValidationError(_("Workbook has no worksheets."))
message_ids = {}
messages = []
row_num = 2
for row in worksheet:
if "ID" not in row:
raise ValidationError(_("Column 'ID' not found."))
if "Message" not in row:
raise ValidationError(_("Column 'Message' not found."))
msg_id = row.get("ID")
text = row.get("Message")
try:
assert isinstance(msg_id, basestring)
msg_id = msg_id.strip()
assert len(msg_id) > 1
assert msg_id[0].upper() in "ABCDEFGH"
except Exception:
raise ValidationError(
_("Invalid ID at row %(row_num)s") % {"row_num": row_num})
if msg_id in message_ids:
raise ValidationError(
_("Duplicate ID at row %(row_num)s") %
{"row_num": row_num})
try:
assert isinstance(text, basestring)
text = text.strip()
assert len(text) > 0
except Exception:
raise ValidationError(
_("Invalid Message at row %(row_num)s") %
{"row_num": row_num})
try:
msg_id.encode("ascii")
except Exception:
raise ValidationError(
_("ID at row %(row_num)s contains invalid character(s)") %
{"row_num": row_num})
try:
text.encode("ascii")
except Exception:
raise ValidationError(
_("Message at row %(row_num)s contains invalid character(s)"
) % {"row_num": row_num})
if len(text) > 160:
raise ValidationError(
_("Message at row %(row_num)s is longer than 160 characters."
) % {"row_num": row_num})
messages.append({
"msg_id": msg_id,
"text": text,
})
message_ids[msg_id] = True
row_num += 1
return messages
