def _get_render_cache(wf_module: WfModule) -> CachedRenderResult:
revision = wf_module.last_relevant_delta_id or 0
cached_result = wf_module.get_cached_render_result()
if cached_result and cached_result.delta_id == revision:
return cached_result
else:
return None
def execute_wfmodule(wf_module: WfModule,
last_result: ProcessResult) -> CachedRenderResult:
"""
Render a single WfModule; cache and return output.
CONCURRENCY NOTES: This function is reasonably concurrency-friendly:
* It locks the workflow, so two renders won't happen on the same workflow
at the same time.
* It returns a valid cache result immediately.
* It checks with the database that `wf_module` hasn't been deleted from
its workflow.
* It checks with the database that `wf_module` hasn't been deleted from
the database entirely.
* It checks with the database that `wf_module` hasn't been modified. (It
is very common for a user to request a module's output -- kicking off a
sequence of `execute_wfmodule` -- and then change a param in a prior
module, making all those calls obsolete.
* It runs in a transaction (obviously -- FOR UPDATE and all), which will
stall `models.Delta` as it tries to write last_relevant_delta_id,
effectively stalling users' update HTTP requests until after the
`wf_module`'s render is complete.
These guarantees mean:
* It's relatively cheap to render twice.
* Users who modify a WfModule while it's rendering will be stalled -- for
as short a duration as possible.
* When a user changes a workflow significantly, all prior renders will end
relatively cheaply.
Raises `UnneededExecution` when the input WfModule should not be rendered.
"""
with locked_wf_module(wf_module) as safe_wf_module:
cached_render_result = wf_module.get_cached_render_result()
# If the cache is good, just return it -- skipping the render() call
if (
cached_render_result
and (cached_render_result.delta_id
== wf_module.last_relevant_delta_id)
):
return cached_render_result
result = dispatch.module_dispatch_render(safe_wf_module,
last_result.dataframe)
cached_render_result = safe_wf_module.cache_render_result(
safe_wf_module.last_relevant_delta_id,
result
)
# Save safe_wf_module, not wf_module, because we know we've only
# changed the cached_render_result columns. (We know because we
# locked the row before fetching it.) `wf_module.save()` might
# overwrite some newer values.
safe_wf_module.save()
return cached_render_result
def _execute_wfmodule_pre(wf_module: WfModule) -> Tuple:
"""
First step of execute_wfmodule().
Returns a Tuple in this order:
* cached_render_result: if non-None, the quick return value of
execute_wfmodule().
* loaded_module: a ModuleVersion for dispatching render
* params: Params for dispatching render
* fetch_result: optional ProcessResult for dispatching render
* old_result: if wf_module.notifications is set, the previous
result we'll compare against after render.
All this runs synchronously within a database lock. (It's a separate
function so that when we're done awaiting it, we can continue executing in
a context that doesn't use a database thread.)
"""
with locked_wf_module(wf_module) as safe_wf_module:
cached_render_result = wf_module.get_cached_render_result()
old_result = None
if cached_render_result:
# If the cache is good, skip everything. No need for old_result,
# because we know the output won't change (since we won't even run
# render()).
if (cached_render_result.delta_id ==
wf_module.last_relevant_delta_id):
return (cached_render_result, None, None, None, None)
if safe_wf_module.notifications:
old_result = cached_render_result.result
module_version = wf_module.module_version
params = safe_wf_module.get_params()
fetch_result = safe_wf_module.get_fetch_result()
loaded_module = LoadedModule.for_module_version_sync(module_version)
return (None, loaded_module, params, fetch_result, old_result)
