def _wrap_render_errors(render_call): try: return render_call() except ModuleError as err: return RenderResult.from_deprecated_error( "Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: " + format_for_user_debugging(err) )
def test_exited_stack_trace(self): self.assertEqual( format_for_user_debugging( ModuleExitedError( 1, """\n File "/app/cjwkernel/errors.py", line 1, in <module>\n import signals\nModuleNotFoundError: No module named 'signals'\n""", )), "exit code 1: ModuleNotFoundError: No module named 'signals'", )
def test_compile_error(self): # Build an error with "from" try: try: compile("abcd(", "foo.py", "exec") except SyntaxError as err: raise ModuleCompileError from err except ModuleCompileError as ex: err = ex self.assertEqual( format_for_user_debugging(err), "SyntaxError: unexpected EOF while parsing (foo.py, line 1)", )
def _wrap_render_errors(render_call): try: return render_call() except ModuleError as err: return RenderResult(errors=[ RenderError( I18nMessage.trans( "py.renderer.execute.wf_module.user_visible_bug_during_render", default= "Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: {message}", args={"message": format_for_user_debugging(err)}, )) ])
def test_exited_unknown_without_message(self): self.assertEqual(format_for_user_debugging(ModuleError()), "ModuleError")
def test_exited_sigsys(self): self.assertEqual( # SIGSYS usually means "seccomp killed you" format_for_user_debugging(ModuleExitedError(-31, "")), "SIGSYS", )
def test_exited_sigkill(self): self.assertEqual(format_for_user_debugging(ModuleExitedError(-9, "")), "SIGKILL")
def test_timeout_error(self): self.assertEqual(format_for_user_debugging(ModuleTimeoutError()), "timed out")
def test_broken_compile_error(self): # don't crash self.assertEqual(format_for_user_debugging(ModuleCompileError()), "ModuleCompileError")
def fetch_or_wrap_error( ctx: contextlib.ExitStack, chroot_context: ChrootContext, basedir: Path, module_id_name: str, module_zipfile: ModuleZipfile, migrated_params_or_error: Union[Dict[str, Any], ModuleError], secrets: Dict[str, Any], last_fetch_result: Optional[FetchResult], maybe_input_crr: Optional[CachedRenderResult], output_path: Path, ): """ Fetch, and do not raise any exceptions worth catching. Exceptions are wrapped -- the result is a FetchResult with `.errors`. This function is slow indeed. Perhaps call it from EventLoop.run_in_executor(). (Why not make it async? Because all the logic inside -- compile module, fetch() -- is sandboxed, meaning it gets its own processes. We may eventually avoid asyncio entirely in `fetcher`. These problems are all handled: * Module was deleted (`module_zipfile is None`) * Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`. * Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`. * migrated_params_or_error is a `ModuleError` * migrated_params_or_error is invalid (`ValueError`) * input_crr points to a nonexistent file (`FileNotFoundError`) """ # module_zipfile=None is allowed if module_zipfile is None: logger.info("fetch() deleted module '%s'", module_id_name) return FetchResult( output_path, [ RenderError( I18nMessage.trans( "py.fetcher.fetch.no_loaded_module", default="Cannot fetch: module was deleted", )) ], ) module_spec = module_zipfile.get_spec() param_schema = module_spec.get_param_schema() if isinstance(migrated_params_or_error, ModuleError): # raise the exception so we can log it try: raise migrated_params_or_error except ModuleError: # We'll always get here logger.exception("%s:migrate_params() raised error", module_zipfile.path.name) return user_visible_bug_fetch_result( output_path, format_for_user_debugging(migrated_params_or_error)) migrated_params = migrated_params_or_error try: param_schema.validate(migrated_params) except ValueError: logger.exception("Invalid return value from %s:migrate_params()", module_zipfile.path.name) return user_visible_bug_fetch_result( output_path, "%s:migrate_params() output invalid params" % module_zipfile.path.name, ) # get input_metadata, input_parquet_path. (This can't error.) input_parquet_path, input_metadata = _download_cached_render_result( ctx, maybe_input_crr, dir=basedir) # Clean params, so they're of the correct type. (This can't error.) params = Params( fetchprep.clean_value(param_schema, migrated_params, input_metadata)) # actually fetch try: return invoke_fetch( module_zipfile, chroot_context=chroot_context, basedir=basedir, params=params, secrets=secrets, last_fetch_result=last_fetch_result, input_parquet_filename=(None if input_parquet_path is None else input_parquet_path.name), output_filename=output_path.name, ) except ModuleError as err: logger.exception("Error calling %s:fetch()", module_zipfile.path.name) return user_visible_bug_fetch_result(output_path, format_for_user_debugging(err))
def fetch_or_wrap_error( ctx: contextlib.ExitStack, basedir: Path, wf_module: WfModule, module_version: ModuleVersion, secrets: Dict[str, Any], last_fetch_result: Optional[FetchResult], maybe_input_crr: Optional[CachedRenderResult], output_path: Path, ): """ Fetch `wf_module`, and do not raise any exceptions worth catching. Exceptions are wrapped -- the result is a FetchResult with `.errors`. This function is slow indeed. Perhaps call it from EventLoop.run_in_executor(). (Why not make it async? Because all the logic inside -- compile module, migrate_params() and fetch() -- is sandboxed, meaning it gets its own processes. We may eventually avoid asyncio entirely in `fetcher`. These problems are all handled: * Module was deleted (`module_version is None`) * Module file has gone missing * Module does not compile (`cjwkernel.errors.ModuleCompileError`) * Module times out (`cjwkernel.errors.ModuleTimeoutError`), either in migrate_params() or in fetch(). * Module crashes (`cjwkernel.errors.ModuleExitedError`), either in migrate_params() or in fetch(). * Module migrate_params() returns invalid data (`ValueError`) * input_crr points to a nonexistent file (`FileNotFoundError`) """ # module_version=None is allowed try: loaded_module = LoadedModule.for_module_version(module_version) except FileNotFoundError: logger.exception("Module %s code disappeared", module_version.id_name) return user_visible_bug_fetch_result(output_path, "FileNotFoundError") except ModuleError as err: logger.exception("Error loading module %s", module_version.id_name) return user_visible_bug_fetch_result( output_path, format_for_user_debugging(err) + " (during load)") if loaded_module is None: logger.info("fetch() deleted module '%s'", wf_module.module_id_name) return FetchResult( output_path, [ RenderError( I18nMessage.TODO_i18n("Cannot fetch: module was deleted")) ], ) # Migrate params, so fetch() gets newest values try: # TODO use params.get_migrated_params(). (Remember to use a # Workflow.cooperative_lock().) params = loaded_module.migrate_params(wf_module.params) except ModuleError as err: logger.exception("Error calling %s.migrate_params()", module_version.id_name) return user_visible_bug_fetch_result(output_path, format_for_user_debugging(err)) try: module_version.param_schema.validate(params) except ValueError: logger.exception("Invalid return value from %s.migrate_params()", module_version.id_name) return user_visible_bug_fetch_result( output_path, "%s.migrate_params() output invalid params" % module_version.id_name, ) # get input_metadata, input_parquet_path. (This can't error.) input_parquet_path, input_metadata = _with_downloaded_cached_render_result( ctx, maybe_input_crr, dir=basedir) # Clean params, so they're of the correct type. (This can't error.) params = Params( fetchprep.clean_value(module_version.param_schema, params, input_metadata)) # actually fetch try: return loaded_module.fetch( basedir=basedir, params=params, secrets=secrets, last_fetch_result=last_fetch_result, input_parquet_filename=(None if input_parquet_path is None else input_parquet_path.name), output_filename=output_path.name, ) except ModuleError as err: logger.exception("Error calling %s.fetch()", module_version.id_name) return user_visible_bug_fetch_result(output_path, format_for_user_debugging(err))
async def _render_step( chroot_context: ChrootContext, workflow: Workflow, step: Step, module_zipfile: Optional[ModuleZipfile], raw_params: Dict[str, Any], tab_name: str, input_path: Path, input_table_columns: List[Column], tab_results: Dict[Tab, Optional[StepResult]], output_path: Path, ) -> LoadedRenderResult: """Prepare and call `step`'s `render()`; return a LoadedRenderResult. The actual render runs in a background thread so the event loop can process other events. """ basedir = output_path.parent if step.order > 0 and not input_table_columns: return LoadedRenderResult.unreachable(output_path) if module_zipfile is None: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.noModule", default="Please delete this step: an administrator uninstalled its code.", ) ) ], ) # exit_stack: stuff that gets deleted when the render is done with contextlib.ExitStack() as exit_stack: try: # raise UnneededExecution, TabCycleError, TabOutputUnreachableError, # NoLoadedDataError, PromptingError fetch_result, params, tab_outputs, uploaded_files = await _execute_step_pre( basedir=basedir, exit_stack=exit_stack, workflow=workflow, step=step, module_zipfile=module_zipfile, raw_params=raw_params, input_path=input_path, input_table_columns=input_table_columns, tab_results=tab_results, ) except NoLoadedDataError: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.NoLoadedDataError", default="Please Add Data before this step.", ) ) ], ) except TabCycleError: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.TabCycleError", default="The chosen tab depends on this one. Please choose another tab.", ) ) ], ) except TabOutputUnreachableError: return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.TabOutputUnreachableError", default="The chosen tab has no output. Please select another one.", ) ) ], ) except PromptingError as err: return LoadedRenderResult.from_errors( output_path, errors=err.as_render_errors() ) # Render may take a while. run_in_executor to push that slowdown to a # thread and keep our event loop responsive. loop = asyncio.get_event_loop() try: return await loop.run_in_executor( None, partial( invoke_render, module_zipfile, chroot_context=chroot_context, basedir=basedir, input_filename=input_path.name, params=params, tab_name=tab_name, tab_outputs=tab_outputs, uploaded_files=uploaded_files, fetch_result=fetch_result, output_filename=output_path.name, ), ) except ModuleError as err: output_path.write_bytes(b"") # SECURITY return LoadedRenderResult.from_errors( output_path, errors=[ RenderError( trans( "py.renderer.execute.step.user_visible_bug_during_render", default="Something unexpected happened. We have been notified and are " "working to fix it. If this persists, contact us. Error code: {message}", arguments={"message": format_for_user_debugging(err)}, ) ) ], )