def as_render_error(self) -> RenderError: """Build a RenderError that describes this error.""" return RenderError( trans( "py.renderer.execute.types.PromptingError.CannotCoerceValueToNumber", default="“{value}” is not a number. Please enter a number.", arguments={"value": self.value}, ))
def as_render_error(self) -> RenderError: """Build a RenderError that describes this error.""" return RenderError( trans( "py.renderer.execute.types.PromptingError.CannotCoerceValueToTimestamp", default= "“{value}” is not a timestamp. Please enter a value with the format “YYYY-MM-DD” or “YYYY-MM-DDThh:mmZ”.", arguments={"value": self.value}, ))
def as_render_errors(self) -> List[RenderError]: """Build RenderError(s) that describe this error. Render errors will include a QuickFix that would resolve this error.""" if self.should_be_text: message = trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.shouldBeText", default="Convert to Text", ) else: # i18n: The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "timestamp"; however, including an (possibly empty) "other" case is mandatory. message = trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.general", default= "Convert {found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}", arguments={ "found_type": self.found_type, "best_wanted_type": self.best_wanted_type_id, }, ) params = {"colnames": self.column_names} if "text" in self.wanted_types: module_id = "converttotext" elif "number" in self.wanted_types: module_id = "converttexttonumber" elif "timestamp" in self.wanted_types: module_id = "convert-date" else: raise RuntimeError( f"Unhandled wanted_types: {self.wanted_types}") return [ RenderError( self._as_i18n_message(), [ QuickFix(message, QuickFixAction.PrependStep(module_id, params)) ], ) ]
def _as_i18n_message(self) -> I18nMessage: """Build a message to prompt the user to use a quick fix.""" # TODO make each quick fix get its own paragraph. (For now, quick # fixes are nothing but buttons.) icu_args = { "columns": len(self.column_names), **{ str(i): self.column_names[i] for i in range(0, len(self.column_names)) }, } if self.should_be_text: # Convert to Text # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names as {0}, {1}, {2}, etc. return trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.shouldBeText", default="{ columns, plural, offset:2" " =1 {The column “{0}” must be converted to Text.}" " =2 {The columns “{0}” and “{1}” must be converted to Text.}" " =3 {The columns “{0}”, “{1}” and “{2}” must be converted to Text.}" " other {The columns “{0}”, “{1}” and # others must be converted to Text.}}", arguments=icu_args, ) else: icu_args["found_type"] = self.found_type icu_args["best_wanted_type"] = self.best_wanted_type_id # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "timestamp"; however, including a (possibly empty) "other" case is mandatory. return trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.general", default="{ columns, plural, offset:2" " =1 {The column “{0}” must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}}.}" " =2 {The columns “{0}” and “{1}” must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}.}" " =3 {The columns “{0}”, “{1}” and “{2}” must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}.}" " other {The columns “{0}”, “{1}” and # others must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}.}}", arguments=icu_args, )
def user_visible_bug_fetch_result(output_path: Path, message: str) -> FetchResult: output_path.write_bytes(b"") return FetchResult( path=output_path, # empty errors=[ FetchError( trans( "py.fetcher.fetch.user_visible_bug_during_fetch", default= "Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: {message}", arguments={"message": message}, )) ], )
def _service_no_longer_configured_error(service: str): return trans( "py.fetcher.secrets._service_no_longer_configured_error", default='Service "{service}" is no longer configured', arguments={"service": service}, )
async def _refresh_oauth2_token(service: oauth.OAuth2, refresh_token: str) -> Dict[str, Any]: """Exchange a "refresh_token" for an "access_token" and "token_type". This involves an HTTP request to an OAuth2 token server. Raise _RefreshOauth2TokenError on error. ref: https://www.oauth.com/oauth2-servers/access-tokens/refreshing-access-tokens/ """ timeout = httpx.Timeout(30) try: async with httpx.AsyncClient(timeout=timeout) as client: response = await client.post( service.refresh_url, data={ "grant_type": "refresh_token", "refresh_token": refresh_token, "client_id": service.client_id, "client_secret": service.client_secret, }, ) # Most of these error cases are _very_ unlikely -- to the point # that we should write error messages only after experiencing # most in production.... # raises ClientPayloadError, ClientResponseError (ContentTypeError), # asyncio.TimeoutError try: maybe_json = response.json() except ValueError: maybe_json = None if ((response.status_code // 400) == 1 and maybe_json and "error" in maybe_json): # Includes errors that are part of the OAuth2 spec. # TODO we can actually translate some of these error codes. ref: # https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/#error raise _RefreshOauth2TokenError( trans( "py.fetcher.secrets._refresh_oauth2_token.error.general", default= "Token server responded with {status_code}: {error} ({description})", arguments={ "status_code": response.status_code, "error": str(maybe_json.get("error", "")), "description": str(maybe_json.get("error_description", "")), }, )) if response.status_code != 200 or maybe_json is None: # Probably a server error. Servers don't usually break. raise _RefreshOauth2TokenError( trans( "py.fetcher.secrets._refresh_oauth2_token.server_error.general", default= "{service_id} responded with HTTP {status_code} {reason}: {description}", arguments={ "service_id": service.service_id, "status_code": response.status_code, "reason": response.reason_phrase, "description": response.content.decode("latin-1"), }, )) return { "token_type": maybe_json.get("token_type"), "access_token": maybe_json.get("access_token"), } except httpx.TimeoutException: raise _RefreshOauth2TokenError( trans( "py.fetcher.secrets._refresh_oauth2_token.timeout_error", default="Timeout during OAuth2 token refresh", )) except httpx.RequestError as err: raise _RefreshOauth2TokenError( trans( "py.fetcher.secrets._refresh_oauth2_token.client_error", default="HTTP error during OAuth2 token refresh: {error}", arguments={"error": str(err)}, ))
def fetch_or_wrap_error( exit_stack: contextlib.ExitStack, chroot_context: ChrootContext, basedir: Path, module_id_name: str, module_zipfile: ModuleZipfile, migrated_params_or_error: Union[Dict[str, Any], ModuleError], secrets: Dict[str, Any], last_fetch_result: Optional[FetchResult], maybe_input_crr: Optional[CachedRenderResult], output_path: Path, ): """Fetch, and do not raise any exceptions worth catching. Exceptions are wrapped -- the result is a FetchResult with `.errors`. This function is slow indeed. Perhaps call it from EventLoop.run_in_executor(). (Why not make it async? Because all the logic inside -- compile module, fetch() -- is sandboxed, meaning it gets its own processes. We may eventually avoid asyncio entirely in `fetcher`. These problems are all handled: * Module was deleted (`module_zipfile is None`) * Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`. * Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`. * migrated_params_or_error is a `ModuleError` * migrated_params_or_error is invalid (`ValueError`) * input_crr points to a nonexistent file (`FileNotFoundError`) """ # module_zipfile=None is allowed if module_zipfile is None: logger.info("fetch() deleted module '%s'", module_id_name) return FetchResult( output_path, [ FetchError( trans( "py.fetcher.fetch.no_loaded_module", default="Cannot fetch: module was deleted", )) ], ) module_spec = module_zipfile.get_spec() param_schema = module_spec.param_schema if isinstance(migrated_params_or_error, ModuleError): # raise the exception so we can log it try: raise migrated_params_or_error except ModuleError: # We'll always get here logger.exception("%s:migrate_params() raised error", module_zipfile.path.name) return user_visible_bug_fetch_result( output_path, format_for_user_debugging(migrated_params_or_error)) migrated_params = migrated_params_or_error try: param_schema.validate(migrated_params) except ValueError: logger.exception("Invalid return value from %s:migrate_params()", module_zipfile.path.name) return user_visible_bug_fetch_result( output_path, "%s:migrate_params() output invalid params" % module_zipfile.path.name, ) # get input_metadata, input_parquet_path. (This can't error.) input_parquet_path, input_metadata = _download_cached_render_result( exit_stack, maybe_input_crr, dir=basedir) # Clean params, so they're of the correct type. (This can't error.) params = fetchprep.clean_value(param_schema, migrated_params, input_metadata) # actually fetch try: return invoke_fetch( module_zipfile, chroot_context=chroot_context, basedir=basedir, params=params, secrets=secrets, last_fetch_result=last_fetch_result, input_parquet_filename=(None if input_parquet_path is None else input_parquet_path.name), output_filename=output_path.name, ) except ModuleError as err: logger.exception("Error calling %s:fetch()", module_zipfile.path.name) return user_visible_bug_fetch_result(output_path, format_for_user_debugging(err))
async def _render_step( chroot_context: ChrootContext, workflow: Workflow, step: Step, module_zipfile: Optional[ModuleZipfile], raw_params: Dict[str, Any], tab_name: str, input_path: Path, input_table_columns: List[Column], tab_results: Dict[Tab, Optional[StepResult]], output_path: Path, ) -> LoadedRenderResult: """Prepare and call `step`'s `render()`; return a LoadedRenderResult. The actual render runs in a background thread so the event loop can process other events. """ basedir = output_path.parent if step.order > 0 and not input_table_columns: return LoadedRenderResult.unreachable(output_path) if module_zipfile is None: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.noModule", default="Please delete this step: an administrator uninstalled its code.", ) ) ], ) # exit_stack: stuff that gets deleted when the render is done with contextlib.ExitStack() as exit_stack: try: # raise UnneededExecution, TabCycleError, TabOutputUnreachableError, # NoLoadedDataError, PromptingError fetch_result, params, tab_outputs, uploaded_files = await _execute_step_pre( basedir=basedir, exit_stack=exit_stack, workflow=workflow, step=step, module_zipfile=module_zipfile, raw_params=raw_params, input_path=input_path, input_table_columns=input_table_columns, tab_results=tab_results, ) except NoLoadedDataError: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.NoLoadedDataError", default="Please Add Data before this step.", ) ) ], ) except TabCycleError: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.TabCycleError", default="The chosen tab depends on this one. Please choose another tab.", ) ) ], ) except TabOutputUnreachableError: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.TabOutputUnreachableError", default="The chosen tab has no output. Please select another one.", ) ) ], ) except PromptingError as err: return LoadedRenderResult.from_errors( output_path, errors=err.as_render_errors() ) # Render may take a while. run_in_executor to push that slowdown to a # thread and keep our event loop responsive. loop = asyncio.get_event_loop() try: return await loop.run_in_executor( None, partial( invoke_render, module_zipfile, chroot_context=chroot_context, basedir=basedir, input_filename=input_path.name, params=params, tab_name=tab_name, tab_outputs=tab_outputs, uploaded_files=uploaded_files, fetch_result=fetch_result, output_filename=output_path.name, ), ) except ModuleError as err: output_path.write_bytes(b"") # SECURITY return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.user_visible_bug_during_render", default="Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: {message}", arguments={"message": format_for_user_debugging(err)}, ) ) ], )
def as_render_error(self) -> RenderError: """Build RenderError(s) that describe this error. Render errors should include at least one QuickFix to resolve the error. Errors the user can see: (wanted_types = {date, timestamp}) "A", "B" and 2 others are Text. [Convert to Date] [Convert to Timestamp] (wanted_types = {number}) "A" and "B" are Date. Select Number. (wanted_types = {text} - special case because all types convert to text) "A" is not Text. [Convert to Text] """ if self.should_be_text: icu_args = { "columns": len(self.column_names), **{ str(i): name for i, name in enumerate(self.column_names) }, } # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names as {0}, {1}, {2}, etc. message = trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.shouldBeText", default="{ columns, plural, offset:2" " =1 {“{0}” is not Text.}" " =2 {“{0}” and “{1}” are not Text.}" " =3 {“{0}”, “{1}” and “{2}” are not Text.}" " other {“{0}”, “{1}” and # others are not Text.}}", arguments=icu_args, ) return RenderError( message, quick_fixes=[ QuickFix( trans( "py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.shouldBeText", default="Convert to Text", ), QuickFixAction.PrependStep( "converttotext", dict(colnames=self.column_names)), ) ], ) else: icu_args = { "columns": len(self.column_names), "found_type": self.found_type, **{ str(i): name for i, name in enumerate(self.column_names) }, } quick_fixes = [ QuickFix( # i18n: The parameter {wanted_type} will have values among "text", "number", "date", "timestamp" or "other". ("other" may translate to "".) trans( "py.renderer.execute.types.PromptingError.WrongColumnType.general.quick_fix", default= "Convert to {wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}", arguments=dict(wanted_type=wanted_type), ), QuickFixAction.PrependStep( _QUICK_FIX_CONVERSIONS[(self.found_type, wanted_type)], dict(colnames=self.column_names), ), ) for wanted_type in sorted( self.wanted_types) # sort for determinism if (self.found_type, wanted_type) in _QUICK_FIX_CONVERSIONS ] if quick_fixes: # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameter {found_type} will be "date", "text", "number", "timestamp" or "other". ("other" may translate to "".) message = trans( "py.renderer.execute.types.PromptingError.WrongColumnType.general.message.before_convert_buttons", default="{columns, plural, offset:2" " =1 {“{0}” is {found_type, select, text {Text} number {Number} timestamp {Timestamp} date {Date} other {}}.}" " =2 {“{0}” and “{1}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}" " =3 {“{0}”, “{1}” and “{2}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}" " other {“{0}”, “{1}” and # others are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}}", arguments=icu_args, ) else: # i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameters {found_type} and {best_wanted_type} will be "date", "text", "number", "timestamp" or "other". ("other" may translate to "".) message = trans( "py.renderer.execute.types.PromptingError.WrongColumnType.general.message.without_convert_buttons", default="{columns, plural, offset:2" " =1 {“{0}” is {found_type, select, text {Text} number {Number} timestamp {Timestamp} date {Date} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}" " =2 {“{0}” and “{1}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}" " =3 {“{0}”, “{1}” and “{2}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}" " other {“{0}”, “{1}” and # others are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}}", arguments=dict(best_wanted_type=self.best_wanted_type, **icu_args), ) return RenderError(message, quick_fixes=quick_fixes)