def _call_forward_and_load_clientside_update(
workflow_id: int,
) -> Tuple[Optional[Delta], Optional[clientside.Update], Optional[int]]:
now = datetime.datetime.now()
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
delta = workflow.deltas.filter(
id__gt=workflow.last_delta_id).first()
if delta is None:
# Nothing to redo: we're at the end of the delta chain
return None, None, None
command = NAME_TO_COMMAND[delta.command_name]
command.forward(delta)
workflow.last_delta_id = delta.id
workflow.updated_at = now
workflow.save(update_fields=["last_delta_id", "updated_at"])
delta.last_applied_at = now
delta.save(update_fields=["last_applied_at"])
return (
delta,
command.load_clientside_update(delta),
delta.id
if command.get_modifies_render_output(delta) else None,
)
except Workflow.DoesNotExist:
return None, None, None
def _call_backward_and_load_clientside_update(
workflow_id: int,
) -> Tuple[Optional[Delta], Optional[clientside.Update], Optional[int]]:
now = datetime.datetime.now()
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
# raise Delta.DoesNotExist if we're at the beginning of the undo chain
delta = workflow.deltas.exclude(
command_name=InitWorkflow.__name__).get(
id=workflow.last_delta_id)
command = NAME_TO_COMMAND[delta.command_name]
command.backward(delta)
# Point workflow to previous delta
# Only update prev_delta_id: other columns may have been edited in
# backward().
workflow.last_delta_id = delta.prev_delta_id or 0
workflow.updated_at = now
workflow.save(update_fields=["last_delta_id", "updated_at"])
delta.last_applied_at = now
delta.save(update_fields=["last_applied_at"])
return (
delta,
command.load_clientside_update(delta),
(workflow.last_delta_id
if command.get_modifies_render_output(delta) else None),
)
except (Workflow.DoesNotExist, Delta.DoesNotExist):
return None, None, None
def load_database_objects(workflow_id: int, wf_module_id: int) -> DatabaseObjects:
"""
Query WfModule info or raise WfModule.DoesNotExist/Workflow.DoesNotExist.
Return tuple containing:
* wf_module: cjwstate.models.WfModule
* module_version: Optional[cjwstate.models.ModuleVersion]
* stored_object: Optional[cjwstate.models.StoredObject]
* input_cached_render_result: Optional[cjwstate.models.CachedRenderResult]
of previous module in tab
"""
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
# raise WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(workflow_lock.workflow).get(
id=wf_module_id
)
try:
module_version = ModuleVersion.objects.latest(wf_module.module_id_name)
except ModuleVersion.DoesNotExist:
module_version = None
try:
stored_object = wf_module.stored_objects.get(
stored_at=wf_module.stored_data_version
)
except StoredObject.DoesNotExist:
stored_object = None
try:
# raise WfModule.DoesNotExist -- but we'll catch this one
prev_module = wf_module.tab.live_wf_modules.get(order=wf_module.order - 1)
input_crr = prev_module.cached_render_result # may be None
except WfModule.DoesNotExist:
input_crr = None
return DatabaseObjects(wf_module, module_version, stored_object, input_crr)
def _call_forward_and_load_ws_data(
delta: Delta) -> Tuple[Dict[str, Any], bool]:
with Workflow.lookup_and_cooperative_lock(id=delta.workflow_id):
delta.forward()
delta.workflow.last_delta = delta
delta.workflow.save(update_fields=["last_delta_id"])
return (delta.load_ws_data(), delta.get_modifies_render_output())
def _call_forward_and_load_clientside_update(
delta: Delta,
) -> Tuple[clientside.Update, bool]:
with Workflow.lookup_and_cooperative_lock(id=delta.workflow_id):
delta.forward()
delta.workflow.last_delta = delta
delta.workflow.save(update_fields=["last_delta_id"])
return (delta.load_clientside_update(), delta.get_modifies_render_output())
def load_database_objects(workflow_id: int,
wf_module_id: int) -> DatabaseObjects:
"""
Query WfModule info.
Raise `WfModule.DoesNotExist` or `Workflow.DoesNotExist` if the step was
deleted.
Catch a `ModuleError` from migrate_params() and return it as part of the
`DatabaseObjects`.
"""
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
# raise WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(
workflow_lock.workflow).get(id=wf_module_id)
# module_zipfile
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
module_zipfile = None
# migrated_params_or_error
if module_zipfile is None:
migrated_params_or_error = {}
else:
try:
migrated_params_or_error = cjwstate.params.get_migrated_params(
wf_module,
module_zipfile=module_zipfile) # raise ModuleError
except ModuleError as err:
migrated_params_or_error = err
# stored_object
try:
stored_object = wf_module.stored_objects.get(
stored_at=wf_module.stored_data_version)
except StoredObject.DoesNotExist:
stored_object = None
# input_crr
try:
# raise WfModule.DoesNotExist -- but we'll catch this one
prev_module = wf_module.tab.live_wf_modules.get(
order=wf_module.order - 1)
input_crr = prev_module.cached_render_result # may be None
except WfModule.DoesNotExist:
input_crr = None
return DatabaseObjects(
wf_module,
module_zipfile,
migrated_params_or_error,
stored_object,
input_crr,
)
def wrapper(request: HttpRequest, workflow_id: int, wf_module_slug: str,
*args, **kwargs):
auth_header = request.headers.get("Authorization", "")
auth_header_match = AuthTokenHeaderRegex.match(auth_header)
if not auth_header_match:
return ErrorResponse(403,
"authorization-bearer-token-not-provided")
bearer_token = auth_header_match.group(1)
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
try:
wf_module = WfModule.live_in_workflow(workflow).get(
slug=wf_module_slug)
except WfModule.DoesNotExist:
return ErrorResponse(404, "step-not-found")
try:
module_zipfile = MODULE_REGISTRY.latest(
wf_module.module_id_name)
except KeyError:
return ErrorResponse(400, "step-module-deleted")
try:
file_param_id_name = next(
iter(pf.id_name
for pf in module_zipfile.get_spec().param_fields
if pf.type == "file"))
except StopIteration:
return ErrorResponse(400, "step-has-no-file-param")
api_token = wf_module.file_upload_api_token
if not api_token:
return ErrorResponse(403, "step-has-no-api-token")
bearer_token_hash = hashlib.sha256(
bearer_token.encode("utf-8")).digest()
api_token_hash = hashlib.sha256(
api_token.encode("utf-8")).digest()
if bearer_token_hash != api_token_hash or bearer_token != api_token:
return ErrorResponse(403,
"authorization-bearer-token-invalid")
return f(
request,
workflow_lock,
wf_module,
file_param_id_name,
*args,
**kwargs,
)
except Workflow.DoesNotExist:
return ErrorResponse(404, "workflow-not-found")
def _call_backward_and_load_ws_data(
delta: Delta) -> Tuple[Dict[str, Any], bool]:
with Workflow.lookup_and_cooperative_lock(id=delta.workflow_id):
delta.backward()
# Point workflow to previous delta
# Only update prev_delta_id: other columns may have been edited in
# backward().
delta.workflow.last_delta = delta.prev_delta
delta.workflow.save(update_fields=["last_delta_id"])
return (delta.load_ws_data(), delta.get_modifies_render_output())
def _first_forward_and_save_returning_clientside_update(
cls, workflow_id: int, **kwargs
) -> Tuple[Optional[Delta], Optional[clientside.Update], bool]:
"""
Create and execute `cls` command; return `(Delta, WebSocket data, render?)`.
If `amend_create_kwargs()` returns `None`, return `(None, None)` here.
All this, in a cooperative lock.
Return `(None, None, False)` if `cls.amend_create_kwargs()` returns `None`.
This is how `cls.amend_create_kwargs()` suggests the Delta should not be
created at all.
"""
# raises Workflow.DoesNotExist
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
create_kwargs = cls.amend_create_kwargs(workflow=workflow, **kwargs)
if not create_kwargs:
return (None, None, False)
# Lookup unapplied deltas to delete. That's the head of the linked
# list that comes _after_ `workflow.last_delta`.
orphan_delta: Optional[Delta] = Delta.objects.filter(
prev_delta_id=workflow.last_delta_id
).first()
if orphan_delta:
orphan_delta.delete_with_successors()
delta = cls.objects.create(
prev_delta_id=workflow.last_delta_id, **create_kwargs
)
delta.forward()
if orphan_delta:
# We just deleted deltas; now we can garbage-collect Tabs and
# WfModules that are soft-deleted and have no deltas referring
# to them.
workflow.delete_orphan_soft_deleted_models()
# Point workflow to us
workflow.last_delta = delta
workflow.save(update_fields=["last_delta_id"])
return (
delta,
delta.load_clientside_update(),
delta.get_modifies_render_output(),
)
def _locked_step(workflow_id: int, step: Step):
"""Refresh step from database and yield with workflow lock.
Raise Workflow.DoesNotExist or Step.DoesNotExist in the event of a
race. (Even soft-deleted Step or Tab raises Step.DoesNotExist,
to simulate hard deletion -- because sooner or later soft-delete won't be
a thing any more.)
"""
# raise Workflow.DoesNotExist
with Workflow.lookup_and_cooperative_lock(id=workflow_id):
# raise Step.DoesNotExist
step.refresh_from_db()
if step.is_deleted or step.tab.is_deleted:
raise Step.DoesNotExist("soft-deleted")
yield
def _locked_wf_module(workflow_id: int, wf_module: WfModule):
"""
Refresh wf_module from database and yield with workflow lock.
Raise Workflow.DoesNotExist or WfModule.DoesNotExist in the event of a
race. (Even soft-deleted WfModule or Tab raises WfModule.DoesNotExist,
to simulate hard deletion -- because sooner or later soft-delete won't be
a thing any more.)
"""
# raise Workflow.DoesNotExist
with Workflow.lookup_and_cooperative_lock(id=workflow_id):
# raise WfModule.DoesNotExist
wf_module.refresh_from_db()
if wf_module.is_deleted or wf_module.tab.is_deleted:
raise WfModule.DoesNotExist("soft-deleted")
yield
def _lookup_requested_workflow_with_auth_and_cooperative_lock(
self, ) -> ContextManager[DbObjectCooperativeLock]:
"""Either yield the requested workflow, or raise Workflow.DoesNotExist
Workflow.DoesNotExist means "permission denied" or "workflow does not exist".
"""
workflow_id_or_secret_id = self.scope["url_route"]["kwargs"][
"workflow_id_or_secret_id"]
if isinstance(workflow_id_or_secret_id, int):
return Workflow.authorized_lookup_and_cooperative_lock(
"read",
self.scope["user"],
self.scope["session"],
id=workflow_id_or_secret_id,
) # raise Workflow.DoesNotExist
else:
return Workflow.lookup_and_cooperative_lock(
secret_id=workflow_id_or_secret_id
) # raise Workflow.DoesNotExist
def update_next_update_time(workflow_id, wf_module, now):
"""Schedule next update, skipping missed updates if any."""
tick = timedelta(seconds=max(wf_module.update_interval,
settings.MIN_AUTOFETCH_INTERVAL))
try:
with Workflow.lookup_and_cooperative_lock(id=workflow_id):
wf_module.refresh_from_db()
next_update = wf_module.next_update
if next_update:
while next_update <= now:
next_update += tick
WfModule.objects.filter(id=wf_module.id).update(
last_update_check=now, next_update=next_update)
except (WfModule.DoesNotExist, Workflow.DoesNotExist):
# [2019-05-27] `wf_module.workflow` throws `Workflow.DoesNotExist` if
# the WfModule is deleted. This handler is for deleted-Workflow _and_
# deleted-WfModule.
pass
def delete_stale_lesson_workflows() -> None:
from cjwstate.models import Workflow # after django.setup()
now = datetime.datetime.now()
expire_date = now - timedelta(seconds=LessonFreshDuration)
to_delete = list(
Workflow.objects.filter(
lesson_slug__isnull=False, last_viewed_at__lt=expire_date
).values_list("id", flat=True)
)
for workflow_id in to_delete:
try:
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow:
logger.info("Deleting workflow %d", workflow_id)
workflow.delete()
except Workflow.DoesNotExist:
logger.info(
"Tried to delete workflow %d, but it was deleted before we could",
workflow_id,
)
def _load_workflow_and_step_sync(
request: HttpRequest,
workflow_id_or_secret_id: Union[int, str],
step_slug: str,
accessing: Literal["all", "chart", "table"],
) -> Tuple[Workflow, Step]:
"""Load (Workflow, Step) from database, or raise Http404 or PermissionDenied.
`Step.tab` will be loaded. (`Step.tab.workflow_id` is needed to access the render
cache.)
To avoid PermissionDenied:
* The workflow must be public; OR
* The user must be workflow owner, editor or viewer; OR
* The user must be workflow report-viewer and the step must be a chart or
table in the report.
"""
try:
if isinstance(workflow_id_or_secret_id, int):
search = {"id": workflow_id_or_secret_id}
has_secret = False
else:
search = {"secret_id": workflow_id_or_secret_id}
has_secret = True
with Workflow.lookup_and_cooperative_lock(**search) as workflow_lock:
workflow = workflow_lock.workflow
if (has_secret or workflow.public
or workflow.request_authorized_owner(request)):
need_report_auth = False
elif request.user is None or request.user.is_anonymous:
raise PermissionDenied()
else:
try:
acl_entry = workflow.acl.filter(
email=request.user.email).get()
except AclEntry.DoesNotExist:
raise PermissionDenied()
if acl_entry.role in {Role.VIEWER, Role.EDITOR}:
need_report_auth = False
elif acl_entry.role == Role.REPORT_VIEWER:
need_report_auth = True
else:
raise PermissionDenied() # role we don't handle yet
step = (Step.live_in_workflow(
workflow.id).select_related("tab").get(slug=step_slug)
) # or Step.DoesNotExist
if need_report_auth: # user is report-viewer
if workflow.has_custom_report:
if (accessing == "chart" and
workflow.blocks.filter(step_id=step.id).exists()):
pass # the step is a chart
elif (accessing == "table" and
workflow.blocks.filter(tab_id=step.tab_id).exists()
and not step.tab.live_steps.filter(
order__gt=step.order)):
pass # step is a table (last step of a report-included tab)
else:
raise PermissionDenied()
else:
# Auto-report: all Charts are allowed; everything else is not
try:
if accessing == "chart" and (MODULE_REGISTRY.latest(
step.module_id_name).get_spec().html_output):
pass
else:
raise PermissionDenied()
except KeyError: # not a module
raise PermissionDenied()
return workflow, step
except (Workflow.DoesNotExist, Step.DoesNotExist):
raise Http404()
def _first_forward_and_save_returning_clientside_update(
cls, workflow_id: int, **kwargs
) -> Tuple[Optional[Delta], Optional[clientside.Update], Optional[int]]:
"""
Create and execute `cls` command; return `(Delta, WebSocket data, render?)`.
If `amend_create_kwargs()` returns `None`, return `(None, None)` here.
All this, in a cooperative lock.
Return `(None, None, None)` if `cls.amend_create_kwargs()` returns `None`.
This is how `cls.amend_create_kwargs()` suggests the Delta should not be
created at all.
"""
now = datetime.datetime.now()
command = NAME_TO_COMMAND[cls.__name__]
try:
# raise Workflow.DoesNotExist
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
create_kwargs = command.amend_create_kwargs(workflow=workflow,
**kwargs)
if not create_kwargs:
return None, None, None
# Lookup unapplied deltas to delete. That's the linked list that comes
# _after_ `workflow.last_delta_id`.
n_deltas_deleted, _ = workflow.deltas.filter(
id__gt=workflow.last_delta_id).delete()
# prev_delta is none when we're at the start of the undo stack
prev_delta = workflow.deltas.filter(
id=workflow.last_delta_id).first()
# Delta.objects.create() and command.forward() may raise unexpected errors
# Defer delete_orphan_soft_deleted_models(), to reduce the risk of this
# race: 1. Delete DB objects; 2. Delete S3 files; 3. ROLLBACK. (We aren't
# avoiding the race _entirely_ here, but we're at least avoiding causing
# the race through errors in Delta or Command.)
delta = Delta.objects.create(
command_name=cls.__name__,
prev_delta=prev_delta,
last_applied_at=now,
**create_kwargs,
)
command.forward(delta)
# Point workflow to us
workflow.last_delta_id = delta.id
workflow.updated_at = datetime.datetime.now()
workflow.save(update_fields=["last_delta_id", "updated_at"])
if n_deltas_deleted:
# We just deleted deltas; now we can garbage-collect Tabs and
# Steps that are soft-deleted and have no deltas referring
# to them.
workflow.delete_orphan_soft_deleted_models()
return (
delta,
command.load_clientside_update(delta),
delta.id
if command.get_modifies_render_output(delta) else None,
)
except Workflow.DoesNotExist:
return None, None, None
