def as_render_errors(self) -> List[RenderError]:
"""Build RenderError(s) that describe this error.
Render errors will include a QuickFix that would resolve this error."""
if self.should_be_text:
message = I18nMessage.trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.shouldBeText",
default="Convert to Text.",
)
else:
# i18n: The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "datetime"; however, including an (possibly empty) "other" case is mandatory.
message = I18nMessage.trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.general",
default=
"Convert { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.",
args={
"found_type": self.found_type,
"best_wanted_type": self.best_wanted_type_id,
},
)
params = {"colnames": self.column_names}
if "text" in self.wanted_types:
module_id = "converttotext"
elif "number" in self.wanted_types:
module_id = "converttexttonumber"
elif "datetime" in self.wanted_types:
module_id = "convert-date"
else:
raise RuntimeError(
f"Unhandled wanted_types: {self.wanted_types}")
return [
RenderError(
self._as_i18n_message(),
[
QuickFix(message,
QuickFixAction.PrependStep(module_id, params))
],
)
]
def _as_i18n_message(self) -> I18nMessage:
"""Build a message to prompt the user to use a quick fix."""
# TODO make each quick fix get its own paragraph. (For now, quick
# fixes are nothing but buttons.)
icu_args = {
"columns": len(self.column_names),
**{
str(i): self.column_names[i]
for i in range(0, len(self.column_names))
},
}
if self.should_be_text:
# Convert to Text
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names as {0}, {1}, {2}, etc.
return I18nMessage.trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.shouldBeText",
default="{ columns, plural, offset:2"
" =1 {The column “{0}” must be converted to Text.}"
" =2 {The columns “{0}” and “{1}” must be converted to Text.}"
" =3 {The columns “{0}”, “{1}” and “{2}” must be converted to Text.}"
" other {The columns “{0}”, “{1}” and # others must be converted to Text.}}",
args=icu_args,
)
else:
icu_args["found_type"] = self.found_type
icu_args["best_wanted_type"] = self.best_wanted_type_id
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "datetime"; however, including a (possibly empty) "other" case is mandatory.
return I18nMessage.trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.general",
default="{ columns, plural, offset:2"
" =1 {The column “{0}” must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}}.}"
" =2 {The columns “{0}” and “{1}” must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.}"
" =3 {The columns “{0}”, “{1}” and “{2}” must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.}"
" other {The columns “{0}”, “{1}” and # others must be converted from { found_type, select, text {Text} number {Numbers} datetime {Dates & Times} other {}} to {best_wanted_type, select, text {Text} number {Numbers} datetime {Dates & Times} other{}}.}}",
args=icu_args,
)
def _wrap_render_errors(render_call):
try:
return render_call()
except ModuleError as err:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.user_visible_bug_during_render",
default=
"Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: {message}",
args={"message": format_for_user_debugging(err)},
))
])
def user_visible_bug_fetch_result(output_path: Path,
message: str) -> FetchResult:
output_path.write_bytes(b"")
return FetchResult(
path=output_path, # empty
errors=[
RenderError(
I18nMessage.trans(
"py.fetcher.fetch.user_visible_bug_during_fetch",
default=
"Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: {message}",
args={"message": message},
))
],
)
def fetch_or_wrap_error(
ctx: contextlib.ExitStack,
chroot_context: ChrootContext,
basedir: Path,
module_id_name: str,
module_zipfile: ModuleZipfile,
migrated_params_or_error: Union[Dict[str, Any], ModuleError],
secrets: Dict[str, Any],
last_fetch_result: Optional[FetchResult],
maybe_input_crr: Optional[CachedRenderResult],
output_path: Path,
):
"""
Fetch, and do not raise any exceptions worth catching.
Exceptions are wrapped -- the result is a FetchResult with `.errors`.
This function is slow indeed. Perhaps call it from
EventLoop.run_in_executor(). (Why not make it async? Because all the logic
inside -- compile module, fetch() -- is sandboxed, meaning it gets its own
processes. We may eventually avoid asyncio entirely in `fetcher`.
These problems are all handled:
* Module was deleted (`module_zipfile is None`)
* Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`.
* Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`.
* migrated_params_or_error is a `ModuleError`
* migrated_params_or_error is invalid (`ValueError`)
* input_crr points to a nonexistent file (`FileNotFoundError`)
"""
# module_zipfile=None is allowed
if module_zipfile is None:
logger.info("fetch() deleted module '%s'", module_id_name)
return FetchResult(
output_path,
[
RenderError(
I18nMessage.trans(
"py.fetcher.fetch.no_loaded_module",
default="Cannot fetch: module was deleted",
))
],
)
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
if isinstance(migrated_params_or_error, ModuleError):
# raise the exception so we can log it
try:
raise migrated_params_or_error
except ModuleError:
# We'll always get here
logger.exception("%s:migrate_params() raised error",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path, format_for_user_debugging(migrated_params_or_error))
migrated_params = migrated_params_or_error
try:
param_schema.validate(migrated_params)
except ValueError:
logger.exception("Invalid return value from %s:migrate_params()",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path,
"%s:migrate_params() output invalid params" %
module_zipfile.path.name,
)
# get input_metadata, input_parquet_path. (This can't error.)
input_parquet_path, input_metadata = _download_cached_render_result(
ctx, maybe_input_crr, dir=basedir)
# Clean params, so they're of the correct type. (This can't error.)
params = Params(
fetchprep.clean_value(param_schema, migrated_params, input_metadata))
# actually fetch
try:
return invoke_fetch(
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
params=params,
secrets=secrets,
last_fetch_result=last_fetch_result,
input_parquet_filename=(None if input_parquet_path is None else
input_parquet_path.name),
output_filename=output_path.name,
)
except ModuleError as err:
logger.exception("Error calling %s:fetch()", module_zipfile.path.name)
return user_visible_bug_fetch_result(output_path,
format_for_user_debugging(err))
async def _refresh_oauth2_token(service: oauth.OAuth2,
refresh_token: str) -> Dict[str, Any]:
"""
Exchange a "refresh_token" for an "access_token" and "token_type".
This involves an HTTP request to an OAuth2 token server.
Raise _RefreshOauth2TokenError on error.
ref: https://www.oauth.com/oauth2-servers/access-tokens/refreshing-access-tokens/
"""
timeout = aiohttp.ClientTimeout(total=60)
try:
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.post(
service.refresh_url,
data={
"grant_type": "refresh_token",
"refresh_token": refresh_token,
"client_id": service.client_id,
"client_secret": service.client_secret,
},
) as response:
# Most of these error cases are _very_ unlikely -- to the point
# that we should write error messages only after experiencing
# most in production....
# raises ClientPayloadError, ClientResponseError (ContentTypeError),
# asyncio.TimeoutError
body = await response.json(encoding="utf-8")
if (response.status // 400) == 1 and "error" in body:
# Includes errors that are part of the OAuth2 spec.
# TODO we can actually translate some of these error codes. ref:
# https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/#error
raise _RefreshOauth2TokenError(
I18nMessage.trans(
"py.fetcher.secrets._refresh_oauth2_token.error.general",
default=
"Token server responded with {status_code}: {error} ({description})",
args={
"status_code": response.status,
"error": str(body["error"]),
"description": body.get("error_description"),
},
))
if response.status != 200:
# Probably a server error. Servers don't usually break.
raise _RefreshOauth2TokenError(
I18nMessage.trans(
"py.fetcher.secrets._refresh_oauth2_token.server_error.general",
default=
"{service_id} responded with HTTP {status_code} {reason}: {description}",
args={
"service_id": service.service_id,
"status_code": response.status,
"reason": response.reason,
"description": body,
},
))
return {
"token_type": body.get("token_type"),
"access_token": body.get("access_token"),
}
except asyncio.TimeoutError:
raise _RefreshOauth2TokenError(
I18nMessage.trans(
"py.fetcher.secrets._refresh_oauth2_token.timeout_error",
default="Timeout during OAuth2 token refresh",
))
except aiohttp.ClientError as err:
raise _RefreshOauth2TokenError(
I18nMessage.trans(
"py.fetcher.secrets._refresh_oauth2_token.client_error",
default="HTTP error during OAuth2 token refresh: {error}",
args={"error": str(err)},
))
def _service_no_longer_configured_error(service: str):
return I18nMessage.trans(
"py.fetcher.secrets._service_no_longer_configured_error",
default='Service "{service}" is no longer configured',
args={"service": service},
)
async def _render_wfmodule(
chroot_context: ChrootContext,
workflow: Workflow,
wf_module: WfModule,
module_zipfile: Optional[ModuleZipfile],
raw_params: Dict[str, Any],
tab: Tab,
input_result: RenderResult,
tab_results: Dict[Tab, Optional[RenderResult]],
output_path: Path,
) -> RenderResult:
"""
Prepare and call `wf_module`'s `render()`; return a RenderResult.
The actual render runs in a background thread so the event loop can process
other events.
"""
basedir = output_path.parent
if wf_module.order > 0 and input_result.status != "ok":
return RenderResult() # 'unreachable'
if module_zipfile is None:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.noModule",
default=
"Please delete this step: an administrator uninstalled its code.",
))
])
# exit_stack: stuff that gets deleted when the render is done
with contextlib.ExitStack() as exit_stack:
try:
# raise UnneededExecution, TabCycleError, TabOutputUnreachableError,
# PromptingError
fetch_result, params = await _execute_wfmodule_pre(
basedir,
exit_stack,
workflow,
wf_module,
module_zipfile,
raw_params,
input_result.table,
tab_results,
)
except TabCycleError:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.TabCycleError",
default=
"The chosen tab depends on this one. Please choose another tab.",
))
])
except TabOutputUnreachableError:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.TabOutputUnreachableError",
default=
"The chosen tab has no output. Please select another one.",
))
])
except PromptingError as err:
return RenderResult(errors=err.as_render_errors())
# Render may take a while. run_in_executor to push that slowdown to a
# thread and keep our event loop responsive.
loop = asyncio.get_event_loop()
return await loop.run_in_executor(
None,
_wrap_render_errors,
partial(
invoke_render,
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
input_table=input_result.table,
params=params,
tab=tab,
fetch_result=fetch_result,
output_filename=output_path.name,
),
)
