def as_render_errors(self) -> List[RenderError]: """Build RenderError(s) that describe this error. Render errors will include a QuickFix that would resolve this error.""" if self.should_be_text: message = I18nMessage.trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.shouldBeText", default="Convert to Text.", ) else: # i18n: The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "datetime"; however, including an (possibly empty) "other" case is mandatory. message = I18nMessage.trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.general", default= "Convert { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.", args={ "found_type": self.found_type, "best_wanted_type": self.best_wanted_type_id, }, ) params = {"colnames": self.column_names} if "text" in self.wanted_types: module_id = "converttotext" elif "number" in self.wanted_types: module_id = "converttexttonumber" elif "datetime" in self.wanted_types: module_id = "convert-date" else: raise RuntimeError( f"Unhandled wanted_types: {self.wanted_types}") return [ RenderError( self._as_i18n_message(), [ QuickFix(message, QuickFixAction.PrependStep(module_id, params)) ], ) ]
def _as_i18n_message(self) -> I18nMessage: """Build a message to prompt the user to use a quick fix.""" # TODO make each quick fix get its own paragraph. (For now, quick # fixes are nothing but buttons.) icu_args = { "columns": len(self.column_names), **{ str(i): self.column_names[i] for i in range(0, len(self.column_names)) }, } if self.should_be_text: # Convert to Text # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names as {0}, {1}, {2}, etc. return I18nMessage.trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.shouldBeText", default="{ columns, plural, offset:2" " =1 {The column “{0}” must be converted to Text.}" " =2 {The columns “{0}” and “{1}” must be converted to Text.}" " =3 {The columns “{0}”, “{1}” and “{2}” must be converted to Text.}" " other {The columns “{0}”, “{1}” and # others must be converted to Text.}}", args=icu_args, ) else: icu_args["found_type"] = self.found_type icu_args["best_wanted_type"] = self.best_wanted_type_id # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "datetime"; however, including a (possibly empty) "other" case is mandatory. return I18nMessage.trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.general", default="{ columns, plural, offset:2" " =1 {The column “{0}” must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}}.}" " =2 {The columns “{0}” and “{1}” must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.}" " =3 {The columns “{0}”, “{1}” and “{2}” must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.}" " other {The columns “{0}”, “{1}” and # others must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.}}", args=icu_args, )
def _wrap_render_errors(render_call): try: return render_call() except ModuleError as err: return RenderResult(errors=[ RenderError( I18nMessage.trans( "py.renderer.execute.wf_module.user_visible_bug_during_render", default= "Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: {message}", args={"message": format_for_user_debugging(err)}, )) ])
def user_visible_bug_fetch_result(output_path: Path, message: str) -> FetchResult: output_path.write_bytes(b"") return FetchResult( path=output_path, # empty errors=[ RenderError( I18nMessage.trans( "py.fetcher.fetch.user_visible_bug_during_fetch", default= "Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: {message}", args={"message": message}, )) ], )
def fetch_or_wrap_error( ctx: contextlib.ExitStack, chroot_context: ChrootContext, basedir: Path, module_id_name: str, module_zipfile: ModuleZipfile, migrated_params_or_error: Union[Dict[str, Any], ModuleError], secrets: Dict[str, Any], last_fetch_result: Optional[FetchResult], maybe_input_crr: Optional[CachedRenderResult], output_path: Path, ): """ Fetch, and do not raise any exceptions worth catching. Exceptions are wrapped -- the result is a FetchResult with `.errors`. This function is slow indeed. Perhaps call it from EventLoop.run_in_executor(). (Why not make it async? Because all the logic inside -- compile module, fetch() -- is sandboxed, meaning it gets its own processes. We may eventually avoid asyncio entirely in `fetcher`. These problems are all handled: * Module was deleted (`module_zipfile is None`) * Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`. * Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`. * migrated_params_or_error is a `ModuleError` * migrated_params_or_error is invalid (`ValueError`) * input_crr points to a nonexistent file (`FileNotFoundError`) """ # module_zipfile=None is allowed if module_zipfile is None: logger.info("fetch() deleted module '%s'", module_id_name) return FetchResult( output_path, [ RenderError( I18nMessage.trans( "py.fetcher.fetch.no_loaded_module", default="Cannot fetch: module was deleted", )) ], ) module_spec = module_zipfile.get_spec() param_schema = module_spec.get_param_schema() if isinstance(migrated_params_or_error, ModuleError): # raise the exception so we can log it try: raise migrated_params_or_error except ModuleError: # We'll always get here logger.exception("%s:migrate_params() raised error", module_zipfile.path.name) return user_visible_bug_fetch_result( output_path, format_for_user_debugging(migrated_params_or_error)) migrated_params = migrated_params_or_error try: param_schema.validate(migrated_params) except ValueError: logger.exception("Invalid return value from %s:migrate_params()", module_zipfile.path.name) return user_visible_bug_fetch_result( output_path, "%s:migrate_params() output invalid params" % module_zipfile.path.name, ) # get input_metadata, input_parquet_path. (This can't error.) input_parquet_path, input_metadata = _download_cached_render_result( ctx, maybe_input_crr, dir=basedir) # Clean params, so they're of the correct type. (This can't error.) params = Params( fetchprep.clean_value(param_schema, migrated_params, input_metadata)) # actually fetch try: return invoke_fetch( module_zipfile, chroot_context=chroot_context, basedir=basedir, params=params, secrets=secrets, last_fetch_result=last_fetch_result, input_parquet_filename=(None if input_parquet_path is None else input_parquet_path.name), output_filename=output_path.name, ) except ModuleError as err: logger.exception("Error calling %s:fetch()", module_zipfile.path.name) return user_visible_bug_fetch_result(output_path, format_for_user_debugging(err))
async def _refresh_oauth2_token(service: oauth.OAuth2, refresh_token: str) -> Dict[str, Any]: """ Exchange a "refresh_token" for an "access_token" and "token_type". This involves an HTTP request to an OAuth2 token server. Raise _RefreshOauth2TokenError on error. ref: https://www.oauth.com/oauth2-servers/access-tokens/refreshing-access-tokens/ """ timeout = aiohttp.ClientTimeout(total=60) try: async with aiohttp.ClientSession(timeout=timeout) as session: async with session.post( service.refresh_url, data={ "grant_type": "refresh_token", "refresh_token": refresh_token, "client_id": service.client_id, "client_secret": service.client_secret, }, ) as response: # Most of these error cases are _very_ unlikely -- to the point # that we should write error messages only after experiencing # most in production.... # raises ClientPayloadError, ClientResponseError (ContentTypeError), # asyncio.TimeoutError body = await response.json(encoding="utf-8") if (response.status // 400) == 1 and "error" in body: # Includes errors that are part of the OAuth2 spec. # TODO we can actually translate some of these error codes. ref: # https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/#error raise _RefreshOauth2TokenError( I18nMessage.trans( "py.fetcher.secrets._refresh_oauth2_token.error.general", default= "Token server responded with {status_code}: {error} ({description})", args={ "status_code": response.status, "error": str(body["error"]), "description": body.get("error_description"), }, )) if response.status != 200: # Probably a server error. Servers don't usually break. raise _RefreshOauth2TokenError( I18nMessage.trans( "py.fetcher.secrets._refresh_oauth2_token.server_error.general", default= "{service_id} responded with HTTP {status_code} {reason}: {description}", args={ "service_id": service.service_id, "status_code": response.status, "reason": response.reason, "description": body, }, )) return { "token_type": body.get("token_type"), "access_token": body.get("access_token"), } except asyncio.TimeoutError: raise _RefreshOauth2TokenError( I18nMessage.trans( "py.fetcher.secrets._refresh_oauth2_token.timeout_error", default="Timeout during OAuth2 token refresh", )) except aiohttp.ClientError as err: raise _RefreshOauth2TokenError( I18nMessage.trans( "py.fetcher.secrets._refresh_oauth2_token.client_error", default="HTTP error during OAuth2 token refresh: {error}", args={"error": str(err)}, ))
def _service_no_longer_configured_error(service: str): return I18nMessage.trans( "py.fetcher.secrets._service_no_longer_configured_error", default='Service "{service}" is no longer configured', args={"service": service}, )
async def _render_wfmodule( chroot_context: ChrootContext, workflow: Workflow, wf_module: WfModule, module_zipfile: Optional[ModuleZipfile], raw_params: Dict[str, Any], tab: Tab, input_result: RenderResult, tab_results: Dict[Tab, Optional[RenderResult]], output_path: Path, ) -> RenderResult: """ Prepare and call `wf_module`'s `render()`; return a RenderResult. The actual render runs in a background thread so the event loop can process other events. """ basedir = output_path.parent if wf_module.order > 0 and input_result.status != "ok": return RenderResult() # 'unreachable' if module_zipfile is None: return RenderResult(errors=[ RenderError( I18nMessage.trans( "py.renderer.execute.wf_module.noModule", default= "Please delete this step: an administrator uninstalled its code.", )) ]) # exit_stack: stuff that gets deleted when the render is done with contextlib.ExitStack() as exit_stack: try: # raise UnneededExecution, TabCycleError, TabOutputUnreachableError, # PromptingError fetch_result, params = await _execute_wfmodule_pre( basedir, exit_stack, workflow, wf_module, module_zipfile, raw_params, input_result.table, tab_results, ) except TabCycleError: return RenderResult(errors=[ RenderError( I18nMessage.trans( "py.renderer.execute.wf_module.TabCycleError", default= "The chosen tab depends on this one. Please choose another tab.", )) ]) except TabOutputUnreachableError: return RenderResult(errors=[ RenderError( I18nMessage.trans( "py.renderer.execute.wf_module.TabOutputUnreachableError", default= "The chosen tab has no output. Please select another one.", )) ]) except PromptingError as err: return RenderResult(errors=err.as_render_errors()) # Render may take a while. run_in_executor to push that slowdown to a # thread and keep our event loop responsive. loop = asyncio.get_event_loop() return await loop.run_in_executor( None, _wrap_render_errors, partial( invoke_render, module_zipfile, chroot_context=chroot_context, basedir=basedir, input_table=input_result.table, params=params, tab=tab, fetch_result=fetch_result, output_filename=output_path.name, ), )