def test_validate_call_columns(self): with get_spreadsheet( self.get_path('missing_call_columns_case_upload', 'xlsx')) as spreadsheet: errors = CallColumnsValidator.run(spreadsheet) self.assertEqual(errors[0], 'Need at least one Call column for Calls 1-6')
def test_unexpected_sheet(self): with get_spreadsheet( self.get_path('unexpected_sheet_case_upload', 'xlsx')) as spreadsheet: errors = RequiredColumnsValidator.run(spreadsheet) self.assertEqual(errors, [ 'Unexpected sheet uploaded. Either Rch_id or admission_id should be present' ])
def post(self, request, *args, **kwargs): handle = request.FILES['bulk_upload_file'] extension = os.path.splitext(handle.name)[1][1:].strip().lower() if extension not in valid_extensions: messages.error(request, _("Please choose a file with one of the following extensions: " "{}").format(", ".join(valid_extensions))) return self.get(request, *args, **kwargs) meta = transient_file_store.write_file(handle, handle.name, self.domain) file_id = meta.identifier f = transient_file_store.get_tempfile_ref_for_contents(file_id) try: open_spreadsheet_download_ref(f) except SpreadsheetFileExtError: messages.error(request, _("File does not appear to be an Excel file. Please choose another file.")) return self.get(request, *args, **kwargs) from corehq.apps.app_manager.views.media_utils import interpolate_media_path from corehq.apps.hqmedia.view_helpers import validate_multimedia_paths_rows, update_multimedia_paths # Get rows, filtering out header, no-ops, and any extra "Usages" columns rows = [] with get_spreadsheet(f) as spreadsheet: for row in list(spreadsheet.iter_rows())[1:]: if row[1]: rows.append(row[:2]) (errors, warnings) = validate_multimedia_paths_rows(self.app, rows) if len(errors): for msg in errors: messages.error(request, msg, extra_tags='html') return self.get(request, *args, **kwargs) paths = { row[0]: interpolate_media_path(row[1]) for row in rows if row[1] } successes = update_multimedia_paths(self.app, paths) self.app.save() # Force all_media to reset self.app.all_media.reset_cache(self.app) self.app.all_media_paths.reset_cache(self.app) # Warn if any old paths remain in app (because they're used in a place this function doesn't know about) warnings = [] self.app.remove_unused_mappings() app_paths = {m.path: True for m in self.app.all_media()} for old_path, new_path in paths.items(): if old_path in app_paths: warnings.append(_("Could not completely update path <code>{}</code>, " "please check app for remaining references.").format(old_path)) for msg in successes: messages.success(request, msg, extra_tags='html') for msg in warnings: messages.warning(request, msg, extra_tags='html') return self.get(request, *args, **kwargs)
def handle(self, export_file, config_file, domain, **options): start = datetime.utcnow() with open(config_file, 'r', encoding='utf-8') as f: config = ImporterConfig.from_json(f.read()) with get_spreadsheet(export_file) as spreadsheet: print( json.dumps(do_import(spreadsheet, config, domain), default=json_handler)) print('finished in %s seconds' % (datetime.utcnow() - start).seconds)
def test_validate_required_columns(self): with get_spreadsheet( self.get_path('missing_columns_rch_case_upload', 'xlsx')) as spreadsheet: errors = RequiredColumnsValidator.run(spreadsheet) # extract missing columns from message missing_columns = set( errors[0].removeprefix('Missing columns ').split(', ')) expected_missing_columns = set(REQUIRED_COLUMNS) self.assertEqual(missing_columns, expected_missing_columns) with get_spreadsheet( self.get_path('missing_columns_sncu_case_upload', 'xlsx')) as spreadsheet: errors = RequiredColumnsValidator.run(spreadsheet) # extract missing columns from message missing_columns = set( errors[0].removeprefix('Missing columns ').split(', ')) expected_missing_columns = set(REQUIRED_COLUMNS) expected_missing_columns.add(NEWBORN_WEIGHT_COLUMN) self.assertEqual(missing_columns, expected_missing_columns)
def handle(self, export_file, config_file, domain, user_id, **options): start = datetime.utcnow() if '@' in user_id: user = WebUser.get_by_username(user_id) else: user = WebUser.get(user_id) if not user.is_member_of(domain): raise CommandError("%s can't access %s" % (user, domain)) with open(config_file, 'r', encoding='utf-8') as f: config = ImporterConfig.from_json(f.read()) config.couch_user_id = user._id with get_spreadsheet(export_file) as spreadsheet: print(json.dumps(do_import(spreadsheet, config, domain), default=json_handler)) print('finished in %s seconds' % (datetime.utcnow() - start).seconds)
def _assert_missing_values_for_sheet(self, spreadsheet_name, required_values): with get_spreadsheet(self.get_path(spreadsheet_name, 'xlsx')) as spreadsheet: for row_num, raw_row in enumerate(spreadsheet.iter_row_dicts()): if row_num == 0: continue # skip first row (header row) fields_to_update = raw_row.copy() if RCH_BENEFICIARY_IDENTIFIER in fields_to_update: fields_to_update['external_id'] = fields_to_update.pop( RCH_BENEFICIARY_IDENTIFIER) elif SNCU_BENEFICIARY_IDENTIFIER in fields_to_update: fields_to_update['external_id'] = fields_to_update.pop( SNCU_BENEFICIARY_IDENTIFIER) fields_to_update, errors = RequiredValueValidator.run( row_num, raw_row, fields_to_update, {}) self.assertEqual([error.title for error in errors], ['Missing required column(s)']) self.assertEqual( [error.message for error in errors], [f"Required columns are {', '.join(required_values)}"])
def handle(self, *args, **options): if len(args) != 4: raise CommandError('Usage is import_cases %s' % self.args) start = datetime.utcnow() export_file, config_file, domain, user_id = args if '@' in user_id: user = WebUser.get_by_username(user_id) else: user = WebUser.get(user_id) if not user.is_member_of(domain): raise CommandError("%s can't access %s" % (user, domain)) with open(config_file, 'r') as f: config = ImporterConfig.from_json(f.read()) config.couch_user_id = user._id with get_spreadsheet(export_file) as spreadsheet: print json.dumps(do_import(spreadsheet, config, domain), default=json_handler) print 'finished in %s seconds' % (datetime.utcnow() - start).seconds
def update_multimedia_paths(request, domain, app_id): if not request.FILES: return json_response( {'error': _("Please choose an Excel file to import.")}) handle = request.FILES['file'] extension = os.path.splitext(handle.name)[1][1:].strip().lower() if extension not in ALLOWED_EXTENSIONS: return json_response({ 'error': _("Please choose a file with one of the following extensions: " "{}").format(", ".join(ALLOWED_EXTENSIONS)) }) meta = transient_file_store.write_file(handle, handle.name, domain) file_id = meta.identifier f = transient_file_store.get_tempfile_ref_for_contents(file_id) try: open_spreadsheet_download_ref(f) except SpreadsheetFileExtError: return json_response({ 'error': _("File does not appear to be an Excel file. Please choose another file." ) }) app = get_app(domain, app_id) from corehq.apps.app_manager.views.media_utils import interpolate_media_path from corehq.apps.hqmedia.view_helpers import validate_multimedia_paths_rows, update_multimedia_paths # Get rows, filtering out header, no-ops, and any extra "Usages" columns rows = [] with get_spreadsheet(f) as spreadsheet: for row in list(spreadsheet.iter_rows())[1:]: if row[1]: rows.append(row[:2]) (errors, warnings) = validate_multimedia_paths_rows(app, rows) if len(errors): return json_response({ 'complete': 1, 'errors': errors, }) paths = {row[0]: interpolate_media_path(row[1]) for row in rows if row[1]} successes = update_multimedia_paths(app, paths) app.save() # Force all_media to reset app.all_media.reset_cache(app) app.all_media_paths.reset_cache(app) # Warn if any old paths remain in app (because they're used in a place this function doesn't know about) warnings = [] app.remove_unused_mappings() app_paths = {m.path: True for m in app.all_media()} for old_path, new_path in paths.items(): if old_path in app_paths: warnings.append( _("Could not completely update path <code>{}</code>, " "please check app for remaining references.").format( old_path)) return json_response({ 'complete': 1, 'successes': successes, 'warnings': warnings, })
def get_spreadsheet(self): return get_spreadsheet(self.get_tempfile())
def update_multimedia_paths(request, domain, app_id): if not request.FILES: return json_response({ 'error': _("Please choose an Excel file to import.") }) handle = request.FILES['file'] extension = os.path.splitext(handle.name)[1][1:].strip().lower() if extension not in ALLOWED_EXTENSIONS: return json_response({ 'error': _("Please choose a file with one of the following extensions: " "{}").format(", ".join(ALLOWED_EXTENSIONS)) }) meta = transient_file_store.write_file(handle, handle.name, domain) file_id = meta.identifier f = transient_file_store.get_tempfile_ref_for_contents(file_id) try: open_spreadsheet_download_ref(f) except SpreadsheetFileExtError: return json_response({ 'error': _("File does not appear to be an Excel file. Please choose another file.") }) app = get_app(domain, app_id) from corehq.apps.app_manager.views.media_utils import interpolate_media_path from corehq.apps.hqmedia.view_helpers import validate_multimedia_paths_rows, update_multimedia_paths # Get rows, filtering out header, no-ops, and any extra "Usages" columns rows = [] with get_spreadsheet(f) as spreadsheet: for row in list(spreadsheet.iter_rows())[1:]: if row[1]: rows.append(row[:2]) (errors, warnings) = validate_multimedia_paths_rows(app, rows) if len(errors): return json_response({ 'complete': 1, 'errors': errors, }) paths = { row[0]: interpolate_media_path(row[1]) for row in rows if row[1] } successes = update_multimedia_paths(app, paths) app.save() # Force all_media to reset app.all_media.reset_cache(app) app.all_media_paths.reset_cache(app) # Warn if any old paths remain in app (because they're used in a place this function doesn't know about) warnings = [] app.remove_unused_mappings() app_paths = {m.path: True for m in app.all_media()} for old_path, new_path in six.iteritems(paths): if old_path in app_paths: warnings.append(_("Could not completely update path <code>{}</code>, " "please check app for remaining references.").format(old_path)) return json_response({ 'complete': 1, 'successes': successes, 'warnings': warnings, })