def process_ui_translation_upload(app, trans_file): workbook = WorkbookJSONReader(trans_file) translations = workbook.get_worksheet(title='translations') commcare_ui_strings = load_translations('en', 2).keys() default_trans = get_default_translations_for_download(app) lang_with_defaults = app.langs[get_index_for_defaults(app.langs)] trans_dict = defaultdict(dict) # Use this to hard fail and not update any translations error_properties = [] # Use this to pass warnings without failing hard warnings = [] for row in translations: if row["property"] not in commcare_ui_strings: # Add a warning for unknown properties, but still add them to the translation dict warnings.append(row["property"] + " is not a known CommCare UI string, but we added it anyway") for lang in app.langs: if row.get(lang): all_parameters = re.findall("\$.*?}", row[lang]) for param in all_parameters: if not re.match("\$\{[0-9]+}", param): error_properties.append(row["property"] + ' - ' + row[lang]) if not (lang_with_defaults == lang and row[lang] == default_trans.get(row["property"], "")): trans_dict[lang].update({row["property"]: row[lang]}) return trans_dict, error_properties, warnings
def test_archive_forms_wrong_domain(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX)) response = archive_forms_old('wrong_domain', 'user1', self.username, list(uploaded_file.get_worksheet())) self.assertEqual(len(response['errors']), len(self.xforms), "Error when wrong domain")
def test_archive_forms_basic(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX)) response = archive_forms_old(self.domain_name, self.user, list(uploaded_file.get_worksheet())) # Need to re-get instance from DB to get updated attributes for key, _id in self.XFORMS.iteritems(): self.assertEqual(XFormInstance.get(_id).doc_type, 'XFormArchived') self.assertEqual(len(response['success']), len(self.xforms))
class FixtureWorkbook(object): """ Helper class for working with the fixture workbook """ def __init__(self, file_or_filename): try: self.workbook = WorkbookJSONReader(file_or_filename) except AttributeError: raise FixtureUploadError( _("Error processing your Excel (.xlsx) file")) except InvalidFileException: raise FixtureUploadError( _("Invalid file-format. Please upload a valid xlsx file.")) def get_types_sheet(self): try: return self.workbook.get_worksheet(title='types') except WorksheetNotFound as e: raise FixtureUploadError( _("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title}) def get_data_sheet(self, data_type): return self.workbook.get_worksheet(data_type.tag) def get_all_type_sheets(self): type_sheets = [] seen_tags = set() for number_of_fixtures, dt in enumerate(self.get_types_sheet()): table_definition = FixtureTableDefinition.from_row(dt) if table_definition.table_id in seen_tags: raise DuplicateFixtureTagException( _(FAILURE_MESSAGES['duplicate_tag']).format( tag=table_definition.table_id)) seen_tags.add(table_definition.table_id) type_sheets.append(table_definition) return type_sheets def validate(self): self.get_types_sheet() self.get_all_type_sheets()
def test_archive_forms_missing(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, MISSING_XLSX)) response = archive_forms_old(self.domain_name, self.user, list(uploaded_file.get_worksheet())) for key, _id in self.XFORMS.iteritems(): self.assertEqual(XFormInstance.get(_id).doc_type, 'XFormArchived') self.assertEqual(len(response['success']), len(self.xforms)) self.assertEqual(len(response['errors']), 1, "One error for trying to archive a missing form")
class _FixtureWorkbook(object): """ Helper class for working with the fixture workbook """ def __init__(self, file_or_filename): try: self.workbook = WorkbookJSONReader(file_or_filename) except AttributeError: # todo: I don't know what would cause this error and it's a bad message raise FixtureUploadError( [_("Error processing your Excel (.xlsx) file")]) except InvalidExcelFileException: raise FixtureUploadError([FAILURE_MESSAGES['not_excel_file']]) except HeaderValueError as e: raise FixtureUploadError([unicode(e)]) except JSONReaderError as e: raise FixtureUploadError([unicode(e)]) def get_types_sheet(self): try: return self.workbook.get_worksheet(title='types') except WorksheetNotFound: raise FixtureUploadError([FAILURE_MESSAGES['no_types_sheet']]) def get_data_sheet(self, data_type_tag): return self.workbook.get_worksheet(data_type_tag) def get_all_type_sheets(self): type_sheets = [] seen_tags = set() for number_of_fixtures, dt in enumerate(self.get_types_sheet()): table_definition = _FixtureTableDefinition.from_row(dt) if table_definition.table_id in seen_tags: raise FixtureUploadError([ _(FAILURE_MESSAGES['duplicate_tag']).format( tag=table_definition.table_id) ]) seen_tags.add(table_definition.table_id) type_sheets.append(table_definition) return type_sheets
def test_archive_forms_basic(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX)) with drop_connected_signals(xform_archived): response = archive_forms_old(DOMAIN_NAME, 'user1', self.username, list(uploaded_file.get_worksheet())) print response # Need to re-get instance from DB to get updated attributes for key, _id in self.XFORMS.iteritems(): self.assertTrue(FormAccessors(DOMAIN_NAME).get_form(_id).is_archived) self.assertEqual(len(response['success']), len(self.xforms))
def test_archive_forms_missing(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, MISSING_XLSX)) with drop_connected_signals(xform_archived): response = archive_forms_old(DOMAIN_NAME, 'user1', self.username, list(uploaded_file.get_worksheet())) for key, _id in self.XFORMS.iteritems(): self.assertTrue(FormAccessors(DOMAIN_NAME).get_form(_id).is_archived) self.assertEqual(len(response['success']), len(self.xforms)) self.assertEqual(len(response['errors']), 1, "One error for trying to archive a missing form")
class FixtureWorkbook(object): """ Helper class for working with the fixture workbook """ def __init__(self, file_or_filename): try: self.workbook = WorkbookJSONReader(file_or_filename) except AttributeError: raise FixtureUploadError(_("Error processing your Excel (.xlsx) file")) except InvalidFileException: raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file.")) def get_types_sheet(self): try: return self.workbook.get_worksheet(title="types") except WorksheetNotFound as e: raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {"title": e.title}) def get_data_sheet(self, data_type): return self.workbook.get_worksheet(data_type.tag) def get_all_type_sheets(self): type_sheets = [] seen_tags = set() for number_of_fixtures, dt in enumerate(self.get_types_sheet()): table_definition = FixtureTableDefinition.from_row(dt) if table_definition.table_id in seen_tags: raise DuplicateFixtureTagException( _(FAILURE_MESSAGES["duplicate_tag"]).format(tag=table_definition.table_id) ) seen_tags.add(table_definition.table_id) type_sheets.append(table_definition) return type_sheets def validate(self): self.get_types_sheet() self.get_all_type_sheets()
def test_archive_forms_basic(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX)) with drop_connected_signals(xform_archived): response = archive_forms_old(DOMAIN_NAME, 'user1', self.username, list(uploaded_file.get_worksheet())) print response # Need to re-get instance from DB to get updated attributes for key, _id in self.XFORMS.iteritems(): self.assertTrue( FormAccessors(DOMAIN_NAME).get_form(_id).is_archived) self.assertEqual(len(response['success']), len(self.xforms))
def test_archive_forms_missing(self): uploaded_file = WorkbookJSONReader(join(BASE_PATH, MISSING_XLSX)) with drop_connected_signals(xform_archived): response = archive_forms_old(DOMAIN_NAME, 'user1', self.username, list(uploaded_file.get_worksheet())) for key, _id in self.XFORMS.iteritems(): self.assertTrue( FormAccessors(DOMAIN_NAME).get_form(_id).is_archived) self.assertEqual(len(response['success']), len(self.xforms)) self.assertEqual(len(response['errors']), 1, "One error for trying to archive a missing form")
def process_ui_translation_upload(app, trans_file): workbook = WorkbookJSONReader(trans_file) translations = workbook.get_worksheet(title='translations') default_trans = get_default_translations_for_download(app) lang_with_defaults = app.langs[get_index_for_defaults(app.langs)] trans_dict = defaultdict(dict) error_properties = [] for row in translations: for lang in app.langs: if row.get(lang): all_parameters = re.findall("\$.*?}", row[lang]) for param in all_parameters: if not re.match("\$\{[0-9]+}", param): error_properties.append(row["property"] + ' - ' + row[lang]) if not (lang_with_defaults == lang and row[lang] == default_trans.get(row["property"], "")): trans_dict[lang].update({row["property"]: row[lang]}) return trans_dict, error_properties
class UploadCommCareUsers(BaseManageCommCareUserView): template_name = 'users/upload_commcare_users.html' urlname = 'upload_commcare_users' page_title = ugettext_noop("Bulk Upload Mobile Workers") @method_decorator( requires_privilege_with_fallback(privileges.BULK_USER_MANAGEMENT)) def dispatch(self, request, *args, **kwargs): return super(UploadCommCareUsers, self).dispatch(request, *args, **kwargs) @property def page_context(self): request_params = self.request.GET if self.request.method == 'GET' else self.request.POST context = { 'bulk_upload': { "help_site": { "address": BULK_MOBILE_HELP_SITE, "name": _("CommCare Help Site"), }, "download_url": reverse("download_commcare_users", args=(self.domain, )), "adjective": _("mobile worker"), "plural_noun": _("mobile workers"), }, 'show_secret_settings': request_params.get("secret", False), } context.update({ 'bulk_upload_form': get_bulk_upload_form(context), }) return context def post(self, request, *args, **kwargs): """View's dispatch method automatically calls this""" upload = request.FILES.get('bulk_upload_file') try: self.workbook = WorkbookJSONReader(upload) except (InvalidFileException, BadZipfile): try: csv.DictReader( io.StringIO(upload.read().decode('ascii'), newline=None)) return HttpResponseBadRequest( "CommCare HQ no longer supports CSV upload. " "Please convert to Excel 2007 or higher (.xlsx) " "and try again.") except UnicodeDecodeError: return HttpResponseBadRequest("Unrecognized format") except JSONReaderError as e: messages.error(request, 'Your upload was unsuccessful. %s' % e.message) return self.get(request, *args, **kwargs) except HeaderValueError as e: return HttpResponseBadRequest( "Upload encountered a data type error: %s" % e.message) try: self.user_specs = self.workbook.get_worksheet(title='users') except WorksheetNotFound: try: self.user_specs = self.workbook.get_worksheet() except WorksheetNotFound: return HttpResponseBadRequest("Workbook has no worksheets") try: self.group_specs = self.workbook.get_worksheet(title='groups') except WorksheetNotFound: self.group_specs = [] self.location_specs = [] if Domain.get_by_name(self.domain).commtrack_enabled: try: self.location_specs = self.workbook.get_worksheet( title='locations') except WorksheetNotFound: # if there is no sheet for locations (since this was added # later and is optional) we don't error pass try: check_headers(self.user_specs) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect( reverse(UploadCommCareUsers.urlname, args=[self.domain])) # convert to list here because iterator destroys the row once it has # been read the first time self.user_specs = list(self.user_specs) for user_spec in self.user_specs: try: user_spec['username'] = enforce_string_type( user_spec['username']) except StringTypeRequiredError: messages.error( request, _("Error: Expected username to be a Text type for username {0}" ).format(user_spec['username'])) return HttpResponseRedirect( reverse(UploadCommCareUsers.urlname, args=[self.domain])) try: check_existing_usernames(self.user_specs, self.domain) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect( reverse(UploadCommCareUsers.urlname, args=[self.domain])) try: check_duplicate_usernames(self.user_specs) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect( reverse(UploadCommCareUsers.urlname, args=[self.domain])) task_ref = expose_cached_download(payload=None, expiry=1 * 60 * 60, file_extension=None) task = bulk_upload_async.delay(self.domain, self.user_specs, list(self.group_specs), list(self.location_specs)) task_ref.set_task(task) return HttpResponseRedirect( reverse(UserUploadStatusView.urlname, args=[self.domain, task_ref.download_id]))
class UploadCommCareUsers(BaseManageCommCareUserView): template_name = 'users/upload_commcare_users.html' urlname = 'upload_commcare_users' page_title = ugettext_noop("Bulk Upload Mobile Workers") @method_decorator(requires_privilege_with_fallback(privileges.BULK_USER_MANAGEMENT)) def dispatch(self, request, *args, **kwargs): return super(UploadCommCareUsers, self).dispatch(request, *args, **kwargs) @property def page_context(self): request_params = expect_GET(self.request) context = { 'bulk_upload': { "help_site": { "address": BULK_MOBILE_HELP_SITE, "name": _("CommCare Help Site"), }, "download_url": reverse( "download_commcare_users", args=(self.domain,)), "adjective": _("mobile worker"), "plural_noun": _("mobile workers"), }, 'show_secret_settings': request_params.get("secret", False), } context.update({ 'bulk_upload_form': get_bulk_upload_form(context), }) return context def post(self, request, *args, **kwargs): """View's dispatch method automatically calls this""" upload = request.FILES.get('bulk_upload_file') try: self.workbook = WorkbookJSONReader(upload) except (InvalidFileException, BadZipfile): try: csv.DictReader(io.StringIO(upload.read().decode('ascii'), newline=None)) return HttpResponseBadRequest( "CommCare HQ no longer supports CSV upload. " "Please convert to Excel 2007 or higher (.xlsx) " "and try again." ) except UnicodeDecodeError: return HttpResponseBadRequest("Unrecognized format") except JSONReaderError as e: messages.error(request, 'Your upload was unsuccessful. %s' % e.message) return self.get(request, *args, **kwargs) except HeaderValueError as e: return HttpResponseBadRequest("Upload encountered a data type error: %s" % e.message) try: self.user_specs = self.workbook.get_worksheet(title='users') except WorksheetNotFound: try: self.user_specs = self.workbook.get_worksheet() except WorksheetNotFound: return HttpResponseBadRequest("Workbook has no worksheets") try: self.group_specs = self.workbook.get_worksheet(title='groups') except WorksheetNotFound: self.group_specs = [] self.location_specs = [] if Domain.get_by_name(self.domain).commtrack_enabled: try: self.location_specs = self.workbook.get_worksheet(title='locations') except WorksheetNotFound: # if there is no sheet for locations (since this was added # later and is optional) we don't error pass try: check_headers(self.user_specs) except UserUploadError as e: messages.error(request, _(e.message)) return HttpResponseRedirect(reverse(UploadCommCareUsers.urlname, args=[self.domain])) task_ref = expose_cached_download(payload=None, expiry=1*60*60, file_extension=None) task = bulk_upload_async.delay( self.domain, list(self.user_specs), list(self.group_specs), list(self.location_specs) ) task_ref.set_task(task) return HttpResponseRedirect( reverse( UserUploadStatusView.urlname, args=[self.domain, task_ref.download_id] ) )
def clean_message_bank_file(self): value = self.cleaned_data.get("message_bank_file") if not value: raise ValidationError(_("Please choose a file.")) try: workbook = WorkbookJSONReader(value) except InvalidFileException: raise ValidationError(_("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again.")) try: worksheet = workbook.get_worksheet() except WorksheetNotFound: raise ValidationError(_("Workbook has no worksheets.")) message_ids = {} messages = [] row_num = 2 for row in worksheet: if "ID" not in row: raise ValidationError(_("Column 'ID' not found.")) if "Message" not in row: raise ValidationError(_("Column 'Message' not found.")) msg_id = row.get("ID") text = row.get("Message") try: assert isinstance(msg_id, basestring) msg_id = msg_id.strip() assert len(msg_id) > 1 assert msg_id[0].upper() in "ABCDEFGH" except Exception: raise ValidationError(_("Invalid ID at row %(row_num)s") % {"row_num" : row_num}) if msg_id in message_ids: raise ValidationError(_("Duplicate ID at row %(row_num)s") % {"row_num" : row_num}) try: assert isinstance(text, basestring) text = text.strip() assert len(text) > 0 except Exception: raise ValidationError(_("Invalid Message at row %(row_num)s") % {"row_num" : row_num}) try: msg_id.encode("ascii") except Exception: raise ValidationError(_("ID at row %(row_num)s contains invalid character(s)") % {"row_num" : row_num}) try: text.encode("ascii") except Exception: raise ValidationError(_("Message at row %(row_num)s contains invalid character(s)") % {"row_num" : row_num}) if len(text) > 160: raise ValidationError(_("Message at row %(row_num)s is longer than 160 characters.") % {"row_num" : row_num}) messages.append({ "msg_id" : msg_id, "text" : text, }) message_ids[msg_id] = True row_num += 1 return messages
def test_excel_formula_reading(self): formula_filepath = self.get_path('formula_sheet', 'xlsx') workbook = WorkbookJSONReader(formula_filepath) results = list(workbook.get_worksheet('Sheet1')) self.assertEqual(results[0]['formula'], 2) # Instead of =SUM(1,1)
def clean_message_bank_file(self): value = self.cleaned_data.get("message_bank_file") if not value: raise ValidationError(_("Please choose a file.")) try: workbook = WorkbookJSONReader(value) except InvalidExcelFileException: raise ValidationError(_("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again.")) try: worksheet = workbook.get_worksheet() except WorksheetNotFound: raise ValidationError(_("Workbook has no worksheets.")) message_ids = {} messages = [] row_num = 2 for row in worksheet: if "ID" not in row: raise ValidationError(_("Column 'ID' not found.")) if "Message" not in row: raise ValidationError(_("Column 'Message' not found.")) msg_id = row.get("ID") text = row.get("Message") try: assert isinstance(msg_id, basestring) msg_id = msg_id.strip() assert len(msg_id) > 1 assert msg_id[0].upper() in "ABCDEFGH" except Exception: raise ValidationError(_("Invalid ID at row %(row_num)s") % {"row_num" : row_num}) if msg_id in message_ids: raise ValidationError(_("Duplicate ID at row %(row_num)s") % {"row_num" : row_num}) try: assert isinstance(text, basestring) text = text.strip() assert len(text) > 0 except Exception: raise ValidationError(_("Invalid Message at row %(row_num)s") % {"row_num" : row_num}) try: msg_id.encode("ascii") except Exception: raise ValidationError(_("ID at row %(row_num)s contains invalid character(s)") % {"row_num" : row_num}) try: text.encode("ascii") except Exception: raise ValidationError(_("Message at row %(row_num)s contains invalid character(s)") % {"row_num" : row_num}) if len(text) > 160: raise ValidationError(_("Message at row %(row_num)s is longer than 160 characters.") % {"row_num" : row_num}) messages.append({ "msg_id" : msg_id, "text" : text, }) message_ids[msg_id] = True row_num += 1 return messages