def _wrap_render_errors(render_call):
try:
return render_call()
except ModuleError as err:
return RenderResult.from_deprecated_error(
"Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: "
+ format_for_user_debugging(err)
)
def test_exited_stack_trace(self):
self.assertEqual(
format_for_user_debugging(
ModuleExitedError(
1,
"""\n File "/app/cjwkernel/errors.py", line 1, in <module>\n import signals\nModuleNotFoundError: No module named 'signals'\n""",
)),
"exit code 1: ModuleNotFoundError: No module named 'signals'",
)
def test_compile_error(self):
# Build an error with "from"
try:
try:
compile("abcd(", "foo.py", "exec")
except SyntaxError as err:
raise ModuleCompileError from err
except ModuleCompileError as ex:
err = ex
self.assertEqual(
format_for_user_debugging(err),
"SyntaxError: unexpected EOF while parsing (foo.py, line 1)",
)
def _wrap_render_errors(render_call):
try:
return render_call()
except ModuleError as err:
return RenderResult(errors=[
RenderError(
I18nMessage.trans(
"py.renderer.execute.wf_module.user_visible_bug_during_render",
default=
"Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: {message}",
args={"message": format_for_user_debugging(err)},
))
])
def test_exited_unknown_without_message(self):
self.assertEqual(format_for_user_debugging(ModuleError()),
"ModuleError")
def test_exited_sigsys(self):
self.assertEqual(
# SIGSYS usually means "seccomp killed you"
format_for_user_debugging(ModuleExitedError(-31, "")),
"SIGSYS",
)
def test_exited_sigkill(self):
self.assertEqual(format_for_user_debugging(ModuleExitedError(-9, "")),
"SIGKILL")
def test_timeout_error(self):
self.assertEqual(format_for_user_debugging(ModuleTimeoutError()),
"timed out")
def test_broken_compile_error(self):
# don't crash
self.assertEqual(format_for_user_debugging(ModuleCompileError()),
"ModuleCompileError")
def fetch_or_wrap_error(
ctx: contextlib.ExitStack,
chroot_context: ChrootContext,
basedir: Path,
module_id_name: str,
module_zipfile: ModuleZipfile,
migrated_params_or_error: Union[Dict[str, Any], ModuleError],
secrets: Dict[str, Any],
last_fetch_result: Optional[FetchResult],
maybe_input_crr: Optional[CachedRenderResult],
output_path: Path,
):
"""
Fetch, and do not raise any exceptions worth catching.
Exceptions are wrapped -- the result is a FetchResult with `.errors`.
This function is slow indeed. Perhaps call it from
EventLoop.run_in_executor(). (Why not make it async? Because all the logic
inside -- compile module, fetch() -- is sandboxed, meaning it gets its own
processes. We may eventually avoid asyncio entirely in `fetcher`.
These problems are all handled:
* Module was deleted (`module_zipfile is None`)
* Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`.
* Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`.
* migrated_params_or_error is a `ModuleError`
* migrated_params_or_error is invalid (`ValueError`)
* input_crr points to a nonexistent file (`FileNotFoundError`)
"""
# module_zipfile=None is allowed
if module_zipfile is None:
logger.info("fetch() deleted module '%s'", module_id_name)
return FetchResult(
output_path,
[
RenderError(
I18nMessage.trans(
"py.fetcher.fetch.no_loaded_module",
default="Cannot fetch: module was deleted",
))
],
)
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
if isinstance(migrated_params_or_error, ModuleError):
# raise the exception so we can log it
try:
raise migrated_params_or_error
except ModuleError:
# We'll always get here
logger.exception("%s:migrate_params() raised error",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path, format_for_user_debugging(migrated_params_or_error))
migrated_params = migrated_params_or_error
try:
param_schema.validate(migrated_params)
except ValueError:
logger.exception("Invalid return value from %s:migrate_params()",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path,
"%s:migrate_params() output invalid params" %
module_zipfile.path.name,
)
# get input_metadata, input_parquet_path. (This can't error.)
input_parquet_path, input_metadata = _download_cached_render_result(
ctx, maybe_input_crr, dir=basedir)
# Clean params, so they're of the correct type. (This can't error.)
params = Params(
fetchprep.clean_value(param_schema, migrated_params, input_metadata))
# actually fetch
try:
return invoke_fetch(
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
params=params,
secrets=secrets,
last_fetch_result=last_fetch_result,
input_parquet_filename=(None if input_parquet_path is None else
input_parquet_path.name),
output_filename=output_path.name,
)
except ModuleError as err:
logger.exception("Error calling %s:fetch()", module_zipfile.path.name)
return user_visible_bug_fetch_result(output_path,
format_for_user_debugging(err))
def fetch_or_wrap_error(
ctx: contextlib.ExitStack,
basedir: Path,
wf_module: WfModule,
module_version: ModuleVersion,
secrets: Dict[str, Any],
last_fetch_result: Optional[FetchResult],
maybe_input_crr: Optional[CachedRenderResult],
output_path: Path,
):
"""
Fetch `wf_module`, and do not raise any exceptions worth catching.
Exceptions are wrapped -- the result is a FetchResult with `.errors`.
This function is slow indeed. Perhaps call it from
EventLoop.run_in_executor(). (Why not make it async? Because all the
logic inside -- compile module, migrate_params() and fetch() -- is
sandboxed, meaning it gets its own processes. We may eventually avoid
asyncio entirely in `fetcher`.
These problems are all handled:
* Module was deleted (`module_version is None`)
* Module file has gone missing
* Module does not compile (`cjwkernel.errors.ModuleCompileError`)
* Module times out (`cjwkernel.errors.ModuleTimeoutError`), either in
migrate_params() or in fetch().
* Module crashes (`cjwkernel.errors.ModuleExitedError`), either in
migrate_params() or in fetch().
* Module migrate_params() returns invalid data (`ValueError`)
* input_crr points to a nonexistent file (`FileNotFoundError`)
"""
# module_version=None is allowed
try:
loaded_module = LoadedModule.for_module_version(module_version)
except FileNotFoundError:
logger.exception("Module %s code disappeared", module_version.id_name)
return user_visible_bug_fetch_result(output_path, "FileNotFoundError")
except ModuleError as err:
logger.exception("Error loading module %s", module_version.id_name)
return user_visible_bug_fetch_result(
output_path,
format_for_user_debugging(err) + " (during load)")
if loaded_module is None:
logger.info("fetch() deleted module '%s'", wf_module.module_id_name)
return FetchResult(
output_path,
[
RenderError(
I18nMessage.TODO_i18n("Cannot fetch: module was deleted"))
],
)
# Migrate params, so fetch() gets newest values
try:
# TODO use params.get_migrated_params(). (Remember to use a
# Workflow.cooperative_lock().)
params = loaded_module.migrate_params(wf_module.params)
except ModuleError as err:
logger.exception("Error calling %s.migrate_params()",
module_version.id_name)
return user_visible_bug_fetch_result(output_path,
format_for_user_debugging(err))
try:
module_version.param_schema.validate(params)
except ValueError:
logger.exception("Invalid return value from %s.migrate_params()",
module_version.id_name)
return user_visible_bug_fetch_result(
output_path,
"%s.migrate_params() output invalid params" %
module_version.id_name,
)
# get input_metadata, input_parquet_path. (This can't error.)
input_parquet_path, input_metadata = _with_downloaded_cached_render_result(
ctx, maybe_input_crr, dir=basedir)
# Clean params, so they're of the correct type. (This can't error.)
params = Params(
fetchprep.clean_value(module_version.param_schema, params,
input_metadata))
# actually fetch
try:
return loaded_module.fetch(
basedir=basedir,
params=params,
secrets=secrets,
last_fetch_result=last_fetch_result,
input_parquet_filename=(None if input_parquet_path is None else
input_parquet_path.name),
output_filename=output_path.name,
)
except ModuleError as err:
logger.exception("Error calling %s.fetch()", module_version.id_name)
return user_visible_bug_fetch_result(output_path,
format_for_user_debugging(err))
async def _render_step(
chroot_context: ChrootContext,
workflow: Workflow,
step: Step,
module_zipfile: Optional[ModuleZipfile],
raw_params: Dict[str, Any],
tab_name: str,
input_path: Path,
input_table_columns: List[Column],
tab_results: Dict[Tab, Optional[StepResult]],
output_path: Path,
) -> LoadedRenderResult:
"""Prepare and call `step`'s `render()`; return a LoadedRenderResult.
The actual render runs in a background thread so the event loop can process
other events.
"""
basedir = output_path.parent
if step.order > 0 and not input_table_columns:
return LoadedRenderResult.unreachable(output_path)
if module_zipfile is None:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.noModule",
default="Please delete this step: an administrator uninstalled its code.",
)
)
],
)
# exit_stack: stuff that gets deleted when the render is done
with contextlib.ExitStack() as exit_stack:
try:
# raise UnneededExecution, TabCycleError, TabOutputUnreachableError,
# NoLoadedDataError, PromptingError
fetch_result, params, tab_outputs, uploaded_files = await _execute_step_pre(
basedir=basedir,
exit_stack=exit_stack,
workflow=workflow,
step=step,
module_zipfile=module_zipfile,
raw_params=raw_params,
input_path=input_path,
input_table_columns=input_table_columns,
tab_results=tab_results,
)
except NoLoadedDataError:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.NoLoadedDataError",
default="Please Add Data before this step.",
)
)
],
)
except TabCycleError:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.TabCycleError",
default="The chosen tab depends on this one. Please choose another tab.",
)
)
],
)
except TabOutputUnreachableError:
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.TabOutputUnreachableError",
default="The chosen tab has no output. Please select another one.",
)
)
],
)
except PromptingError as err:
return LoadedRenderResult.from_errors(
output_path, errors=err.as_render_errors()
)
# Render may take a while. run_in_executor to push that slowdown to a
# thread and keep our event loop responsive.
loop = asyncio.get_event_loop()
try:
return await loop.run_in_executor(
None,
partial(
invoke_render,
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
input_filename=input_path.name,
params=params,
tab_name=tab_name,
tab_outputs=tab_outputs,
uploaded_files=uploaded_files,
fetch_result=fetch_result,
output_filename=output_path.name,
),
)
except ModuleError as err:
output_path.write_bytes(b"") # SECURITY
return LoadedRenderResult.from_errors(
output_path,
errors=[
RenderError(
trans(
"py.renderer.execute.step.user_visible_bug_during_render",
default="Something unexpected happened. We have been notified and are "
"working to fix it. If this persists, contact us. Error code: {message}",
arguments={"message": format_for_user_debugging(err)},
)
)
],
)
