def validate_zipfile(module_zipfile: ModuleZipfile) -> None:
"""Ensure `path` points to a valid ModuleZipfile.
Raise `WorkbenchModuleImportError` with an English-language description
of the flaw otherwise. (This can help module authors fix their mistakes.)
"""
try:
module_zipfile.get_spec() # raise KeyError, ValueError, BadZipFile
# raise KeyError, UnicodeDecodeError, SyntaxError, BadZipFile
compiled_module = module_zipfile.compile_code_without_executing()
cjwstate.modules.kernel.validate(compiled_module) # raise ModuleError
module_zipfile.get_optional_html() # raise UnicodeError, BadZipFile
module_zipfile.get_optional_js_module() # raise UnicodeError, BadZipFile
except zipfile.BadZipFile as err:
raise WorkbenchModuleImportError("Bad zipfile: %s" % str(err)) from err
except ValueError as err:
raise WorkbenchModuleImportError(
"Module .yaml is invalid: %s" % str(err)
) from err
except KeyError as err:
raise WorkbenchModuleImportError(
"Zipfile is missing a required file: %s" % str(err)
) from err
except SyntaxError as err:
raise WorkbenchModuleImportError(
"Module Python code has a syntax error: %s" % str(err)
) from err
except UnicodeError as err:
raise WorkbenchModuleImportError(
"Module Python, HTML or JS code is invalid UTF-8: %s" % str(err)
) from err
except ModuleError as err:
raise WorkbenchModuleImportError(
"Module Python code failed to run: %s" % str(err)
) from err
def import_zipfile(path: Path) -> clientside.Module:
"""
Save a zipfile to database and minio and build a `clientside.Module`.
Raise `WorkbenchModuleImportError` if `path` points to an invalid module.
Otherwise, do not raise any errors one can sensibly recover from.
"""
temp_zipfile = ModuleZipfile(path)
validate_zipfile(temp_zipfile) # raise WorkbenchModuleImportError
module_id = temp_zipfile.module_id
version = temp_zipfile.version
module_spec = temp_zipfile.get_spec()
js_module = temp_zipfile.get_optional_js_module() or ""
minio.fput_file(minio.ExternalModulesBucket,
"%s/%s" % (module_id, path.name), path)
ModuleVersion.objects.update_or_create(
id_name=module_id,
source_version_hash=version,
spec=asdict(temp_zipfile.get_spec()),
js_module=js_module,
)
return clientside.Module(module_spec, js_module)
def download_module_zipfile(
tempdir: Path,
module_id: ModuleId,
version: ModuleVersion,
*,
deprecated_spec: Dict[str, Any],
deprecated_js_module: str,
) -> ModuleZipfile:
"""
Produce a local-path ModuleZipfile by downloading from minio.
Raise `RuntimeError` (_from_ another kind of error -- `FileNotFoundError`,
`KeyError`, `ValueError`, `SyntaxError`, `BadZipFile`,
`UnicodeDecodeError` or more) if the zipfile is not a valid Workbench
module. We spend the time testing the zipfile for validity because A) it's
good to catch errors quickly; and B) fetcher, renderer and server all need
to execute code on each module, so they're destined to validate the module
anyway.
The zipfile is always written to "{tempdir}/{module_id}.{version}.zip".
This function is not re-entrant when called with the same parameters.
Callers may use locks to avoid trying to download the same data multiple
times.
"""
logger.info("download_module_zipfile(%s.%s.zip)", module_id, version)
zippath = tempdir / ("%s.%s.zip" % (module_id, version))
try:
_download_module_zipfile_modern(zippath, module_id, version)
except FileNotFoundError as original_error:
try:
_download_module_zipfile_deprecated(
zippath,
module_id,
version,
spec=deprecated_spec,
js_module=deprecated_js_module,
)
except FileNotFoundError:
raise RuntimeError from original_error
ret = ModuleZipfile(zippath) # raise ZipfileError
try:
# raise KeyError or SyntaxError
compiled_module = ret.compile_code_without_executing()
ret.get_spec() # raise KeyError or ValueError
cjwstate.modules.kernel.validate(compiled_module) # raise ModuleError
except Exception as err:
raise RuntimeError from err
return ret
def _execute_step_pre(
*,
basedir: Path,
exit_stack: contextlib.ExitStack,
workflow: Workflow,
step: Step,
module_zipfile: ModuleZipfile,
raw_params: Dict[str, Any],
input_path: Path,
input_table_columns: List[Column],
tab_results: Dict[Tab, Optional[StepResult]],
) -> ExecuteStepPreResult:
"""First step of execute_step().
Raise TabCycleError or TabOutputUnreachableError if the module depends on
tabs with errors.
Raise NoLoadedDataError if there is no input table and the module's
loads_data is False (the default).
Raise PromptingError if the module parameters are invalid.
Raise UnneededExecution if `step` has changed.
(We won't call the render() method in any of these cases.)
All this runs synchronously within a database lock. (It's a separate
function so that when we're done awaiting it, we can continue executing in
a context that doesn't use a database thread.)
`tab_results.keys()` must be ordered as the Workflow's tabs are.
"""
# raises UnneededExecution
with locked_step(workflow, step) as safe_step:
fetch_result = _load_fetch_result(safe_step, basedir, exit_stack)
module_spec = module_zipfile.get_spec()
if not module_spec.loads_data and not input_table_columns:
raise NoLoadedDataError
# raise TabCycleError, TabOutputUnreachableError, PromptingError
params, tab_outputs, uploaded_files = renderprep.prep_params(
params=raw_params,
schema=module_spec.param_schema,
step_id=step.id,
input_table_columns=input_table_columns,
tab_results=tab_results,
basedir=basedir,
exit_stack=exit_stack,
)
return ExecuteStepPreResult(fetch_result, params, tab_outputs, uploaded_files)
def _execute_wfmodule_pre(
basedir: Path,
exit_stack: contextlib.ExitStack,
workflow: Workflow,
wf_module: WfModule,
module_zipfile: ModuleZipfile,
raw_params: Dict[str, Any],
input_table: ArrowTable,
tab_results: Dict[Tab, Optional[RenderResult]],
) -> ExecuteStepPreResult:
"""
First step of execute_wfmodule().
Raise TabCycleError or TabOutputUnreachableError if the module depends on
tabs with errors. (We won't call the render() method in that case.)
Raise PromptingError if the module parameters are invalid. (We'll skip
render() and prompt the user with quickfixes in that case.)
Raise UnneededExecution if `wf_module` has changed.
All this runs synchronously within a database lock. (It's a separate
function so that when we're done awaiting it, we can continue executing in
a context that doesn't use a database thread.)
`tab_results.keys()` must be ordered as the Workflow's tabs are.
"""
# raises UnneededExecution
with locked_wf_module(workflow, wf_module) as safe_wf_module:
fetch_result = _load_fetch_result(safe_wf_module, basedir, exit_stack)
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
render_context = renderprep.RenderContext(
wf_module.id,
input_table,
tab_results,
basedir,
exit_stack,
raw_params, # ugh
)
# raise TabCycleError, TabOutputUnreachableError, PromptingError
params = renderprep.get_param_values(param_schema, raw_params,
render_context)
return ExecuteStepPreResult(fetch_result, params)
def _build_source_catalog(module_zipfile: ModuleZipfile) -> Catalog:
source_catalog = Catalog(default_locale)
spec = module_zipfile.get_spec()
for message_id, source_string in find_spec_messages(spec).items():
source_catalog.add(message_id, string=source_string)
with zipfile.ZipFile(module_zipfile.path, mode="r") as zf:
for info in zf.infolist():
if info.filename.endswith(".py"):
with zf.open(info) as code_io:
for message_id, message_properties in find_messages_in_module_code(
code_io, info.filename).items():
source_catalog.add(
message_id,
string=message_properties["string"],
auto_comments=message_properties["comments"],
locations=message_properties["locations"],
)
return source_catalog
def _get_migrated_params(wf_module: WfModule,
module_zipfile: ModuleZipfile) -> Dict[str, Any]:
"""
Build the Params dict which will be passed to render().
Call LoadedModule.migrate_params() to ensure the params are up-to-date.
On ModuleError or ValueError, log the error and return default params. This
will render the "wrong" thing ... but the front-end should show the migrate
error (as it's rendering the form) so users should figure out the problem.
(What's the alternative? Abort the whole workflow render? We can't render
_any_ module until we've migrated _all_ modules; and it's hard to imagine
showing the user a huge, aborted render.)
Assume we are called within a `workflow.cooperative_lock()`.
"""
if module_zipfile is None:
# This is a deleted module. Renderer will pass the input through to
# the output.
return {}
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
try:
result = get_migrated_params(wf_module, module_zipfile=module_zipfile)
except ModuleError:
# LoadedModule logged this error; no need to log it again.
return param_schema.coerce(None)
# Is the module buggy? It might be. Log that error, and return a valid
# set of params anyway -- even if it isn't the params the user wants.
try:
param_schema.validate(result)
return result
except ValueError as err:
logger.exception(
"%s:migrate_params() gave wrong retval: %s",
module_zipfile.path.name,
str(err),
)
return param_schema.coerce(result)
def fetch_or_wrap_error(
ctx: contextlib.ExitStack,
chroot_context: ChrootContext,
basedir: Path,
module_id_name: str,
module_zipfile: ModuleZipfile,
migrated_params_or_error: Union[Dict[str, Any], ModuleError],
secrets: Dict[str, Any],
last_fetch_result: Optional[FetchResult],
maybe_input_crr: Optional[CachedRenderResult],
output_path: Path,
):
"""
Fetch, and do not raise any exceptions worth catching.
Exceptions are wrapped -- the result is a FetchResult with `.errors`.
This function is slow indeed. Perhaps call it from
EventLoop.run_in_executor(). (Why not make it async? Because all the logic
inside -- compile module, fetch() -- is sandboxed, meaning it gets its own
processes. We may eventually avoid asyncio entirely in `fetcher`.
These problems are all handled:
* Module was deleted (`module_zipfile is None`)
* Module times out (`cjwkernel.errors.ModuleTimeoutError`), in `fetch()`.
* Module crashes (`cjwkernel.errors.ModuleExitedError`), in `fetch()`.
* migrated_params_or_error is a `ModuleError`
* migrated_params_or_error is invalid (`ValueError`)
* input_crr points to a nonexistent file (`FileNotFoundError`)
"""
# module_zipfile=None is allowed
if module_zipfile is None:
logger.info("fetch() deleted module '%s'", module_id_name)
return FetchResult(
output_path,
[
RenderError(
I18nMessage.trans(
"py.fetcher.fetch.no_loaded_module",
default="Cannot fetch: module was deleted",
))
],
)
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
if isinstance(migrated_params_or_error, ModuleError):
# raise the exception so we can log it
try:
raise migrated_params_or_error
except ModuleError:
# We'll always get here
logger.exception("%s:migrate_params() raised error",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path, format_for_user_debugging(migrated_params_or_error))
migrated_params = migrated_params_or_error
try:
param_schema.validate(migrated_params)
except ValueError:
logger.exception("Invalid return value from %s:migrate_params()",
module_zipfile.path.name)
return user_visible_bug_fetch_result(
output_path,
"%s:migrate_params() output invalid params" %
module_zipfile.path.name,
)
# get input_metadata, input_parquet_path. (This can't error.)
input_parquet_path, input_metadata = _download_cached_render_result(
ctx, maybe_input_crr, dir=basedir)
# Clean params, so they're of the correct type. (This can't error.)
params = Params(
fetchprep.clean_value(param_schema, migrated_params, input_metadata))
# actually fetch
try:
return invoke_fetch(
module_zipfile,
chroot_context=chroot_context,
basedir=basedir,
params=params,
secrets=secrets,
last_fetch_result=last_fetch_result,
input_parquet_filename=(None if input_parquet_path is None else
input_parquet_path.name),
output_filename=output_path.name,
)
except ModuleError as err:
logger.exception("Error calling %s:fetch()", module_zipfile.path.name)
return user_visible_bug_fetch_result(output_path,
format_for_user_debugging(err))