def as_render_error(self) -> RenderError:
"""Build a RenderError that describes this error."""
return RenderError(
trans(
"py.renderer.execute.types.PromptingError.CannotCoerceValueToNumber",
default="“{value}” is not a number. Please enter a number.",
arguments={"value": self.value},
))
def as_render_error(self) -> RenderError:
"""Build a RenderError that describes this error."""
return RenderError(
trans(
"py.renderer.execute.types.PromptingError.CannotCoerceValueToTimestamp",
default=
"“{value}” is not a timestamp. Please enter a value with the format “YYYY-MM-DD” or “YYYY-MM-DDThh:mmZ”.",
arguments={"value": self.value},
))
def as_render_errors(self) -> List[RenderError]:
"""Build RenderError(s) that describe this error.
Render errors will include a QuickFix that would resolve this error."""
if self.should_be_text:
message = trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.shouldBeText",
default="Convert to Text",
)
else:
# i18n: The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "timestamp"; however, including an (possibly empty) "other" case is mandatory.
message = trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.general",
default=
"Convert {found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}",
arguments={
"found_type": self.found_type,
"best_wanted_type": self.best_wanted_type_id,
},
)
params = {"colnames": self.column_names}
if "text" in self.wanted_types:
module_id = "converttotext"
elif "number" in self.wanted_types:
module_id = "converttexttonumber"
elif "timestamp" in self.wanted_types:
module_id = "convert-date"
else:
raise RuntimeError(
f"Unhandled wanted_types: {self.wanted_types}")
return [
RenderError(
self._as_i18n_message(),
[
QuickFix(message,
QuickFixAction.PrependStep(module_id, params))
],
)
]
def _as_i18n_message(self) -> I18nMessage:
"""Build a message to prompt the user to use a quick fix."""
# TODO make each quick fix get its own paragraph. (For now, quick
# fixes are nothing but buttons.)
icu_args = {
"columns": len(self.column_names),
**{
str(i): self.column_names[i]
for i in range(0, len(self.column_names))
},
}
if self.should_be_text:
# Convert to Text
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names as {0}, {1}, {2}, etc.
return trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.shouldBeText",
default="{ columns, plural, offset:2"
" =1 {The column “{0}” must be converted to Text.}"
" =2 {The columns “{0}” and “{1}” must be converted to Text.}"
" =3 {The columns “{0}”, “{1}” and “{2}” must be converted to Text.}"
" other {The columns “{0}”, “{1}” and # others must be converted to Text.}}",
arguments=icu_args,
)
else:
icu_args["found_type"] = self.found_type
icu_args["best_wanted_type"] = self.best_wanted_type_id
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameters {found_type} and {best_wanted_type} will have values among "text", "number", "timestamp"; however, including a (possibly empty) "other" case is mandatory.
return trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.general",
default="{ columns, plural, offset:2"
" =1 {The column “{0}” must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}}.}"
" =2 {The columns “{0}” and “{1}” must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}.}"
" =3 {The columns “{0}”, “{1}” and “{2}” must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}.}"
" other {The columns “{0}”, “{1}” and # others must be converted from { found_type, select, text {Text} number {Numbers} timestamp {Timestamps} other {}} to {best_wanted_type, select, text {Text} number {Numbers} timestamp {Timestamps} other{}}.}}",
arguments=icu_args,
)
def user_visible_bug_fetch_result(output_path: Path,
message: str) -> FetchResult:
output_path.write_bytes(b"")
return FetchResult(
path=output_path, # empty
errors=[
FetchError(
trans(
"py.fetcher.fetch.user_visible_bug_during_fetch",
default=
"Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: {message}",
arguments={"message": message},
))
],
)
def _service_no_longer_configured_error(service: str):
return trans(
"py.fetcher.secrets._service_no_longer_configured_error",
default='Service "{service}" is no longer configured',
arguments={"service": service},
)
async def _refresh_oauth2_token(service: oauth.OAuth2,
refresh_token: str) -> Dict[str, Any]:
"""Exchange a "refresh_token" for an "access_token" and "token_type".
This involves an HTTP request to an OAuth2 token server.
Raise _RefreshOauth2TokenError on error.
ref: https://www.oauth.com/oauth2-servers/access-tokens/refreshing-access-tokens/
"""
timeout = httpx.Timeout(30)
try:
async with httpx.AsyncClient(timeout=timeout) as client:
response = await client.post(
service.refresh_url,
data={
"grant_type": "refresh_token",
"refresh_token": refresh_token,
"client_id": service.client_id,
"client_secret": service.client_secret,
},
)
# Most of these error cases are _very_ unlikely -- to the point
# that we should write error messages only after experiencing
# most in production....
# raises ClientPayloadError, ClientResponseError (ContentTypeError),
# asyncio.TimeoutError
try:
maybe_json = response.json()
except ValueError:
maybe_json = None
if ((response.status_code // 400) == 1 and maybe_json
and "error" in maybe_json):
# Includes errors that are part of the OAuth2 spec.
# TODO we can actually translate some of these error codes. ref:
# https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/#error
raise _RefreshOauth2TokenError(
trans(
"py.fetcher.secrets._refresh_oauth2_token.error.general",
default=
"Token server responded with {status_code}: {error} ({description})",
arguments={
"status_code":
response.status_code,
"error":
str(maybe_json.get("error", "")),
"description":
str(maybe_json.get("error_description", "")),
},
))
if response.status_code != 200 or maybe_json is None:
# Probably a server error. Servers don't usually break.
raise _RefreshOauth2TokenError(
trans(
"py.fetcher.secrets._refresh_oauth2_token.server_error.general",
default=
"{service_id} responded with HTTP {status_code} {reason}: {description}",
arguments={
"service_id": service.service_id,
"status_code": response.status_code,
"reason": response.reason_phrase,
"description": response.content.decode("latin-1"),
},
))
return {
"token_type": maybe_json.get("token_type"),
"access_token": maybe_json.get("access_token"),
}
except httpx.TimeoutException:
raise _RefreshOauth2TokenError(
trans(
"py.fetcher.secrets._refresh_oauth2_token.timeout_error",
default="Timeout during OAuth2 token refresh",
))
except httpx.RequestError as err:
raise _RefreshOauth2TokenError(
trans(
"py.fetcher.secrets._refresh_oauth2_token.client_error",
default="HTTP error during OAuth2 token refresh: {error}",
arguments={"error": str(err)},
))
def fetch_or_wrap_error(
exit_stack: contextlib.ExitStack,
chroot_context: ChrootContext,
basedir: Path,
module_id_name: str,
module_zipfile: ModuleZipfile,
migrated_params_or_error: Union[Dict[str, Any], ModuleError],
secrets: Dict[str, Any],
last_fetch_result: Optional[FetchResult],
maybe_input_crr: Optional[CachedRenderResult],
output_path: Path,
):
"""Fetch, and do not raise any exceptions worth catching.
Exceptions are wrapped -- the result is a FetchResult with `.errors`.
This function is slow indeed. Perhaps call it from
EventLoop.run_in_executor(). (Why not make it async? Because all the logic
inside -- compile module, fetch() -- is sandboxed, meaning it gets its own
processes. We may eventually avoid asyncio entirely in `fetcher`.
These problems are all handled:
* Module was deleted (`module_zipfile is None`)
* Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`.
* Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`.
* migrated_params_or_error is a `ModuleError`
* migrated_params_or_error is invalid (`ValueError`)
* input_crr points to a nonexistent file (`FileNotFoundError`)
"""
# module_zipfile=None is allowed
if module_zipfile is None:
logger.info("fetch() deleted module '%s'", module_id_name)
return FetchResult(
output_path,
[
FetchError(
trans(
"py.fetcher.fetch.no_loaded_module",
default="Cannot fetch: module was deleted",
))
],
)
module_spec = module_zipfile.get_spec()
param_schema = module_spec.param_schema
if isinstance(migrated_params_or_error, ModuleError):
# raise the exception so we can log it
try:
raise migrated_params_or_error
except ModuleError:
# We'll always get here
logger.exception("%s:migrate_params() raised error",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path, format_for_user_debugging(migrated_params_or_error))
migrated_params = migrated_params_or_error
try:
param_schema.validate(migrated_params)
except ValueError:
logger.exception("Invalid return value from %s:migrate_params()",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path,
"%s:migrate_params() output invalid params" %
module_zipfile.path.name,
)
# get input_metadata, input_parquet_path. (This can't error.)
input_parquet_path, input_metadata = _download_cached_render_result(
exit_stack, maybe_input_crr, dir=basedir)
# Clean params, so they're of the correct type. (This can't error.)
params = fetchprep.clean_value(param_schema, migrated_params,
input_metadata)
# actually fetch
try:
return invoke_fetch(
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
params=params,
secrets=secrets,
last_fetch_result=last_fetch_result,
input_parquet_filename=(None if input_parquet_path is None else
input_parquet_path.name),
output_filename=output_path.name,
)
except ModuleError as err:
logger.exception("Error calling %s:fetch()", module_zipfile.path.name)
return user_visible_bug_fetch_result(output_path,
format_for_user_debugging(err))
async def _render_step(
chroot_context: ChrootContext,
workflow: Workflow,
step: Step,
module_zipfile: Optional[ModuleZipfile],
raw_params: Dict[str, Any],
tab_name: str,
input_path: Path,
input_table_columns: List[Column],
tab_results: Dict[Tab, Optional[StepResult]],
output_path: Path,
) -> LoadedRenderResult:
"""Prepare and call `step`'s `render()`; return a LoadedRenderResult.
The actual render runs in a background thread so the event loop can process
other events.
"""
basedir = output_path.parent
if step.order > 0 and not input_table_columns:
return LoadedRenderResult.unreachable(output_path)
if module_zipfile is None:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.noModule",
default="Please delete this step: an administrator uninstalled its code.",
)
)
],
)
# exit_stack: stuff that gets deleted when the render is done
with contextlib.ExitStack() as exit_stack:
try:
# raise UnneededExecution, TabCycleError, TabOutputUnreachableError,
# NoLoadedDataError, PromptingError
fetch_result, params, tab_outputs, uploaded_files = await _execute_step_pre(
basedir=basedir,
exit_stack=exit_stack,
workflow=workflow,
step=step,
module_zipfile=module_zipfile,
raw_params=raw_params,
input_path=input_path,
input_table_columns=input_table_columns,
tab_results=tab_results,
)
except NoLoadedDataError:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.NoLoadedDataError",
default="Please Add Data before this step.",
)
)
],
)
except TabCycleError:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.TabCycleError",
default="The chosen tab depends on this one. Please choose another tab.",
)
)
],
)
except TabOutputUnreachableError:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.TabOutputUnreachableError",
default="The chosen tab has no output. Please select another one.",
)
)
],
)
except PromptingError as err:
return LoadedRenderResult.from_errors(
output_path, errors=err.as_render_errors()
)
# Render may take a while. run_in_executor to push that slowdown to a
# thread and keep our event loop responsive.
loop = asyncio.get_event_loop()
try:
return await loop.run_in_executor(
None,
partial(
invoke_render,
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
input_filename=input_path.name,
params=params,
tab_name=tab_name,
tab_outputs=tab_outputs,
uploaded_files=uploaded_files,
fetch_result=fetch_result,
output_filename=output_path.name,
),
)
except ModuleError as err:
output_path.write_bytes(b"") # SECURITY
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.user_visible_bug_during_render",
default="Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: {message}",
arguments={"message": format_for_user_debugging(err)},
)
)
],
)
def as_render_error(self) -> RenderError:
"""Build RenderError(s) that describe this error.
Render errors should include at least one QuickFix to resolve the error.
Errors the user can see:
(wanted_types = {date, timestamp})
"A", "B" and 2 others are Text.
[Convert to Date]
[Convert to Timestamp]
(wanted_types = {number})
"A" and "B" are Date. Select Number.
(wanted_types = {text} - special case because all types convert to text)
"A" is not Text.
[Convert to Text]
"""
if self.should_be_text:
icu_args = {
"columns": len(self.column_names),
**{
str(i): name
for i, name in enumerate(self.column_names)
},
}
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names as {0}, {1}, {2}, etc.
message = trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_error_message.shouldBeText",
default="{ columns, plural, offset:2"
" =1 {“{0}” is not Text.}"
" =2 {“{0}” and “{1}” are not Text.}"
" =3 {“{0}”, “{1}” and “{2}” are not Text.}"
" other {“{0}”, “{1}” and # others are not Text.}}",
arguments=icu_args,
)
return RenderError(
message,
quick_fixes=[
QuickFix(
trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.as_quick_fixes.shouldBeText",
default="Convert to Text",
),
QuickFixAction.PrependStep(
"converttotext",
dict(colnames=self.column_names)),
)
],
)
else:
icu_args = {
"columns": len(self.column_names),
"found_type": self.found_type,
**{
str(i): name
for i, name in enumerate(self.column_names)
},
}
quick_fixes = [
QuickFix(
# i18n: The parameter {wanted_type} will have values among "text", "number", "date", "timestamp" or "other". ("other" may translate to "".)
trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.general.quick_fix",
default=
"Convert to {wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}",
arguments=dict(wanted_type=wanted_type),
),
QuickFixAction.PrependStep(
_QUICK_FIX_CONVERSIONS[(self.found_type,
wanted_type)],
dict(colnames=self.column_names),
),
) for wanted_type in sorted(
self.wanted_types) # sort for determinism
if (self.found_type, wanted_type) in _QUICK_FIX_CONVERSIONS
]
if quick_fixes:
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameter {found_type} will be "date", "text", "number", "timestamp" or "other". ("other" may translate to "".)
message = trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.general.message.before_convert_buttons",
default="{columns, plural, offset:2"
" =1 {“{0}” is {found_type, select, text {Text} number {Number} timestamp {Timestamp} date {Date} other {}}.}"
" =2 {“{0}” and “{1}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}"
" =3 {“{0}”, “{1}” and “{2}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}"
" other {“{0}”, “{1}” and # others are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}}",
arguments=icu_args,
)
else:
# i18n: The parameter {columns} will contain the total number of columns that need to be converted; you will also receive the column names: {0}, {1}, {2}, etc. The parameters {found_type} and {best_wanted_type} will be "date", "text", "number", "timestamp" or "other". ("other" may translate to "".)
message = trans(
"py.renderer.execute.types.PromptingError.WrongColumnType.general.message.without_convert_buttons",
default="{columns, plural, offset:2"
" =1 {“{0}” is {found_type, select, text {Text} number {Number} timestamp {Timestamp} date {Date} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}"
" =2 {“{0}” and “{1}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}"
" =3 {“{0}”, “{1}” and “{2}” are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}"
" other {“{0}”, “{1}” and # others are {found_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}. Select {best_wanted_type, select, text {Text} number {Number} date {Date} timestamp {Timestamp} other {}}.}}",
arguments=dict(best_wanted_type=self.best_wanted_type,
**icu_args),
)
return RenderError(message, quick_fixes=quick_fixes)
