def _do_set_notifications(scope, step: Step, notifications: bool):
step.notifications = notifications
step.save(update_fields=["notifications"])
if notifications:
server.utils.log_user_event_from_scope(scope,
"Enabled email notifications",
{"stepId": step.id})
def _step_delete_secret_and_build_delta(
workflow: Workflow, step: Step,
param: str) -> Optional[clientside.Update]:
"""Write a new secret (or `None`) to `step`, or raise.
Return a `clientside.Update`, or `None` if the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
step.refresh_from_db()
except Step.DoesNotExist:
return None # no-op
if step.secrets.get(param) is None:
return None # no-op
step.secrets = dict(step.secrets) # shallow copy
del step.secrets[param]
step.save(update_fields=["secrets"])
return clientside.Update(
steps={
step.id: clientside.StepUpdate(secrets=step.secret_metadata)
})
def get_migrated_params(
step: Step, *, module_zipfile: ModuleZipfile = None
) -> Dict[str, Any]:
"""Read `step.params`, calling migrate_params() or using cache fields.
Call this within a `Workflow.cooperative_lock()`.
If migrate_params() was already called for this version of the module,
return the cached value. See `step.cached_migrated_params`,
`step.cached_migrated_params_module_version`.
Raise `ModuleError` if migration fails.
Raise `KeyError` if the module was deleted.
Raise `RuntimeError` (unrecoverable) if there is a problem loading or
executing the module. (Modules are validated before import, so this should
not happen.)
The result may be invalid. Call `validate()` to raise a `ValueError` to
detect that case.
TODO avoid holding the database lock whilst executing stuff on the kernel.
(This will involve auditing and modifying all callers to handle new error
cases.)
"""
if module_zipfile is None:
# raise KeyError
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
stale = (
module_zipfile.version == "develop"
# works if cached version (and thus cached _result_) is None
or module_zipfile.version != step.cached_migrated_params_module_version
)
if not stale:
return step.cached_migrated_params
else:
# raise ModuleError
params = invoke_migrate_params(module_zipfile, step.params)
step.cached_migrated_params = params
step.cached_migrated_params_module_version = module_zipfile.version
try:
step.save(
update_fields=[
"cached_migrated_params",
"cached_migrated_params_module_version",
]
)
except ValueError:
# Step was deleted, so we get:
# "ValueError: Cannot force an update in save() with no primary key."
pass
return params
def _do_mark_result_unchanged(
workflow_id: int, step: Step, now: datetime.datetime
) -> None:
"""Do database manipulations for mark_result_unchanged().
Modify `step` in-place.
Raise Step.DoesNotExist or Workflow.DoesNotExist in case of a race.
"""
with _locked_step(workflow_id, step):
step.is_busy = False
step.last_update_check = now
step.save(update_fields=["is_busy", "last_update_check"])
def _step_set_secret_and_build_delta(
workflow: Workflow, step: Step, param: str,
secret: str) -> Optional[clientside.Update]:
"""Write a new secret to `step`, or raise.
Return a `clientside.Update`, or `None` if the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
step.refresh_from_db()
except Step.DoesNotExist:
return None # no-op
if step.secrets.get(param, {}).get("secret") == secret:
return None # no-op
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise HandlerError(
f"BadRequest: ModuleZipfile {step.module_id_name} does not exist"
)
module_spec = module_zipfile.get_spec()
if not any(p.type == "secret" and p.secret_logic.provider == "string"
for p in module_spec.param_fields):
raise HandlerError(
f"BadRequest: param is not a secret string parameter")
created_at = datetime.datetime.now()
created_at_str = (
created_at.strftime("%Y-%m-%dT%H:%M:%S") + "." +
created_at.strftime("%f")[0:3] # milliseconds
+ "Z")
step.secrets = {
**step.secrets,
param: {
"name": created_at_str,
"secret": secret
},
}
step.save(update_fields=["secrets"])
return clientside.Update(
steps={
step.id: clientside.StepUpdate(secrets=step.secret_metadata)
})
def clear_cached_render_result_for_step(step: Step) -> None:
"""Delete a CachedRenderResult, if it exists.
This deletes the Parquet file from disk, _then_ empties relevant
database fields and saves them (and only them).
"""
delete_parquet_files_for_step(step.workflow_id, step.id)
step.cached_render_result_delta_id = None
step.cached_render_result_errors = []
step.cached_render_result_json = b"null"
step.cached_render_result_status = None
step.cached_render_result_columns = None
step.cached_render_result_nrows = None
step.save(update_fields=STEP_FIELDS)
def _do_try_set_autofetch(scope, step: Step, auto_update_data: bool,
update_interval: int):
# We may ROLLBACK; if we do, we need to remember the old values
old_auto_update_data = step.auto_update_data
old_update_interval = step.update_interval
check_quota = (auto_update_data and step.auto_update_data
and update_interval < step.update_interval) or (
auto_update_data and not step.auto_update_data)
quota_exceeded = None
try:
with transaction.atomic():
step.auto_update_data = auto_update_data
step.update_interval = update_interval
if auto_update_data:
step.next_update = datetime.datetime.now(
) + datetime.timedelta(seconds=update_interval)
else:
step.next_update = None
step.save(update_fields=[
"auto_update_data", "update_interval", "next_update"
])
# Now before we commit, let's see if we've surpassed the user's limit;
# roll back if we have.
#
# Only rollback if we're _increasing_ our fetch count. If we're
# lowering it, allow that -- even if the user is over limit, we still
# want to commit because it's an improvement.
if check_quota:
autofetches = autofetch.list_autofetches_json(scope)
if autofetches["nFetchesPerDay"] > autofetches[
"maxFetchesPerDay"]:
raise AutofetchQuotaExceeded(autofetches)
except AutofetchQuotaExceeded as err:
step.auto_update_data = old_auto_update_data
step.update_interval = old_update_interval
quota_exceeded = err.autofetches
retval = {
"isAutofetch": step.auto_update_data,
"fetchInterval": step.update_interval,
}
if quota_exceeded is not None:
retval["quotaExceeded"] = quota_exceeded # a dict
return retval
def cache_render_result(
workflow: Workflow, step: Step, delta_id: int, result: LoadedRenderResult
) -> None:
"""Save `result` for later viewing.
Raise AssertionError if `delta_id` is not what we expect.
Since this alters data, call it within a lock:
with workflow.cooperative_lock():
step.refresh_from_db() # may change delta_id
cache_render_result(workflow, step, delta_id, result)
"""
assert delta_id == step.last_relevant_delta_id
assert result is not None
json_bytes = json_encode(result.json).encode("utf-8")
if not result.columns:
if result.errors:
status = "error"
else:
status = "unreachable"
else:
status = "ok"
step.cached_render_result_delta_id = delta_id
step.cached_render_result_errors = result.errors
step.cached_render_result_status = status
step.cached_render_result_json = json_bytes
step.cached_render_result_columns = result.columns
step.cached_render_result_nrows = result.table.num_rows
# Now we get to the part where things can end up inconsistent. Try to
# err on the side of not-caching when that happens.
delete_parquet_files_for_step(workflow.id, step.id) # makes old cache inconsistent
step.save(update_fields=STEP_FIELDS) # makes new cache inconsistent
if result.table.num_columns: # only write non-zero-column tables
with tempfile_context() as parquet_path:
cjwparquet.write(parquet_path, result.table)
s3.fput_file(
BUCKET, parquet_key(workflow.id, step.id, delta_id), parquet_path
) # makes new cache consistent
def _do_create_result(
workflow_id: int, step: Step, result: FetchResult, now: datetime.datetime
) -> None:
"""Do database manipulations for create_result().
Modify `step` in-place.
Do *not* do the logic in SetStepDataVersion. We're creating a new
version, not doing something undoable.
Raise Step.DoesNotExist or Workflow.DoesNotExist in case of a race.
"""
with _locked_step(workflow_id, step):
storedobjects.create_stored_object(
workflow_id, step.id, result.path, stored_at=now
)
storedobjects.delete_old_files_to_enforce_storage_limits(step=step)
# Assume caller sends new list to clients via SetStepDataVersion
step.fetch_errors = result.errors
step.is_busy = False
step.last_update_check = now
step.save(update_fields=["fetch_errors", "is_busy", "last_update_check"])
def _do_set_file_upload_api_token(step: Step, api_token: Optional[str]):
step.file_upload_api_token = api_token
step.save(update_fields=["file_upload_api_token"])
def _do_clear_unseen_notification(step: Step):
step.has_unseen_notification = False
step.save(update_fields=["has_unseen_notification"])
def _do_set_collapsed(step: Step, is_collapsed: bool):
step.is_collapsed = is_collapsed
step.save(update_fields=["is_collapsed"])