def _get_render_cache(wf_module: WfModule) -> CachedRenderResult: revision = wf_module.last_relevant_delta_id or 0 cached_result = wf_module.get_cached_render_result() if cached_result and cached_result.delta_id == revision: return cached_result else: return None
def execute_wfmodule(wf_module: WfModule, last_result: ProcessResult) -> CachedRenderResult: """ Render a single WfModule; cache and return output. CONCURRENCY NOTES: This function is reasonably concurrency-friendly: * It locks the workflow, so two renders won't happen on the same workflow at the same time. * It returns a valid cache result immediately. * It checks with the database that `wf_module` hasn't been deleted from its workflow. * It checks with the database that `wf_module` hasn't been deleted from the database entirely. * It checks with the database that `wf_module` hasn't been modified. (It is very common for a user to request a module's output -- kicking off a sequence of `execute_wfmodule` -- and then change a param in a prior module, making all those calls obsolete. * It runs in a transaction (obviously -- FOR UPDATE and all), which will stall `models.Delta` as it tries to write last_relevant_delta_id, effectively stalling users' update HTTP requests until after the `wf_module`'s render is complete. These guarantees mean: * It's relatively cheap to render twice. * Users who modify a WfModule while it's rendering will be stalled -- for as short a duration as possible. * When a user changes a workflow significantly, all prior renders will end relatively cheaply. Raises `UnneededExecution` when the input WfModule should not be rendered. """ with locked_wf_module(wf_module) as safe_wf_module: cached_render_result = wf_module.get_cached_render_result() # If the cache is good, just return it -- skipping the render() call if ( cached_render_result and (cached_render_result.delta_id == wf_module.last_relevant_delta_id) ): return cached_render_result result = dispatch.module_dispatch_render(safe_wf_module, last_result.dataframe) cached_render_result = safe_wf_module.cache_render_result( safe_wf_module.last_relevant_delta_id, result ) # Save safe_wf_module, not wf_module, because we know we've only # changed the cached_render_result columns. (We know because we # locked the row before fetching it.) `wf_module.save()` might # overwrite some newer values. safe_wf_module.save() return cached_render_result
def _execute_wfmodule_pre(wf_module: WfModule) -> Tuple: """ First step of execute_wfmodule(). Returns a Tuple in this order: * cached_render_result: if non-None, the quick return value of execute_wfmodule(). * loaded_module: a ModuleVersion for dispatching render * params: Params for dispatching render * fetch_result: optional ProcessResult for dispatching render * old_result: if wf_module.notifications is set, the previous result we'll compare against after render. All this runs synchronously within a database lock. (It's a separate function so that when we're done awaiting it, we can continue executing in a context that doesn't use a database thread.) """ with locked_wf_module(wf_module) as safe_wf_module: cached_render_result = wf_module.get_cached_render_result() old_result = None if cached_render_result: # If the cache is good, skip everything. No need for old_result, # because we know the output won't change (since we won't even run # render()). if (cached_render_result.delta_id == wf_module.last_relevant_delta_id): return (cached_render_result, None, None, None, None) if safe_wf_module.notifications: old_result = cached_render_result.result module_version = wf_module.module_version params = safe_wf_module.get_params() fetch_result = safe_wf_module.get_fetch_result() loaded_module = LoadedModule.for_module_version_sync(module_version) return (None, loaded_module, params, fetch_result, old_result)