def _wf_module_delete_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule,
param: str) -> Optional[clientside.Update]:
"""
Write a new secret (or `None`) to `wf_module`, or raise.
Return a `clientside.Update`, or `None` if the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param) is None:
return None # no-op
wf_module.secrets = dict(wf_module.secrets) # shallow copy
del wf_module.secrets[param]
wf_module.save(update_fields=["secrets"])
return clientside.Update(steps={
wf_module.id:
clientside.StepUpdate(secrets=wf_module.secret_metadata)
})
def _do_finish_upload(
workflow: Workflow, wf_module: WfModule, uuid: uuidgen.UUID, filename: str
) -> clientside.Update:
with workflow.cooperative_lock():
wf_module.refresh_from_db()
try:
in_progress_upload = wf_module.in_progress_uploads.get(
id=uuid, is_completed=False
)
except InProgressUpload.DoesNotExist:
raise HandlerError(
"BadRequest: key is not being uploaded for this WfModule right now. "
"(Even a valid key becomes invalid after you create, finish or abort "
"an upload on its WfModule.)"
)
try:
in_progress_upload.convert_to_uploaded_file(filename)
except FileNotFoundError:
raise HandlerError(
"BadRequest: file not found. "
"You must upload the file before calling finish_upload."
)
return clientside.Update(
steps={
wf_module.id: clientside.StepUpdate(
files=wf_module.to_clientside().files
)
}
)
def _wf_module_delete_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str
) -> Optional[Dict[str, Any]]:
"""
Write a new secret (or `None`) to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database has not been modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param) is None:
return None # no-op
wf_module.secrets = dict(wf_module.secrets) # shallow copy
del wf_module.secrets[param]
wf_module.save(update_fields=["secrets"])
return {
"updateWfModules": {
str(wf_module.id): {"secrets": wf_module.secret_metadata}
}
}
def _do_set_notifications(scope, wf_module: WfModule, notifications: bool):
wf_module.notifications = notifications
wf_module.save(update_fields=["notifications"])
if notifications:
server.utils.log_user_event_from_scope(
scope, "Enabled email notifications", {"wfModuleId": wf_module.id}
)
def _do_create_result(
workflow_id: int, wf_module: WfModule, result: FetchResult, now: timezone.datetime
) -> None:
"""
Do database manipulations for create_result().
Modify `wf_module` in-place.
Do *not* do the logic in ChangeDataVersionCommand. We're creating a new
version, not doing something undoable.
Raise WfModule.DoesNotExist or Workflow.DoesNotExist in case of a race.
"""
error = ""
if result.errors:
if result.errors[0].message.id != "TODO_i18n":
raise RuntimeError("TODO handle i18n-ready fetch-result errors")
elif result.errors[0].quick_fixes:
raise RuntimeError("TODO handle quick fixes from fetches")
else:
error = result.errors[0].message.args["text"]
with _locked_wf_module(workflow_id, wf_module):
storedobjects.create_stored_object(
workflow_id, wf_module.id, result.path, stored_at=now
)
storedobjects.enforce_storage_limits(wf_module)
wf_module.fetch_error = error
wf_module.is_busy = False
wf_module.last_update_check = now
wf_module.save(update_fields=["fetch_error", "is_busy", "last_update_check"])
def wfmodule_render(request: HttpRequest, wf_module: WfModule, format=None):
# Get first and last row from query parameters, or default to all if not
# specified
try:
startrow = int_or_none(request.GET.get("startrow"))
endrow = int_or_none(request.GET.get("endrow"))
except ValueError:
return Response(
{"message": "bad row number", "status_code": 400},
status=status.HTTP_400_BAD_REQUEST,
)
with wf_module.workflow.cooperative_lock():
wf_module.refresh_from_db()
cached_result = wf_module.cached_render_result
if cached_result is None:
# assume we'll get another request after execute finishes
return JsonResponse({"start_row": 0, "end_row": 0, "rows": []})
try:
startrow, endrow, records = _make_render_tuple(
cached_result, startrow, endrow
)
except CorruptCacheError:
# assume we'll get another request after execute finishes
return JsonResponse({"start_row": 0, "end_row": 0, "rows": []})
return JsonResponse({"start_row": startrow, "end_row": endrow, "rows": records})
def get_migrated_params(
wf_module: WfModule,
*,
module_zipfile: ModuleZipfile = None) -> Dict[str, Any]:
"""
Read `wf_module.params`, calling migrate_params() or using cache fields.
Call this within a `Workflow.cooperative_lock()`.
If migrate_params() was already called for this version of the module,
return the cached value. See `wf_module.cached_migrated_params`,
`wf_module.cached_migrated_params_module_version`.
Raise `ModuleError` if migration fails.
Raise `KeyError` if the module was deleted.
Raise `RuntimeError` (unrecoverable) if there is a problem loading or
executing the module. (Modules are validated before import, so this should
not happen.)
The result may be invalid. Call `validate()` to raise a `ValueError` to
detect that case.
TODO avoid holding the database lock whilst executing stuff on the kernel.
(This will involve auditing and modifying all callers to handle new error
cases.)
"""
if module_zipfile is None:
# raise KeyError
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
stale = (
module_zipfile.version == "develop"
# works if cached version (and thus cached _result_) is None
or (module_zipfile.get_param_schema_version() !=
wf_module.cached_migrated_params_module_version))
if not stale:
return wf_module.cached_migrated_params
else:
# raise ModuleError
params = invoke_migrate_params(module_zipfile, wf_module.params)
wf_module.cached_migrated_params = params
wf_module.cached_migrated_params_module_version = (
module_zipfile.get_param_schema_version())
try:
wf_module.save(update_fields=[
"cached_migrated_params",
"cached_migrated_params_module_version",
])
except ValueError:
# WfModule was deleted, so we get:
# "ValueError: Cannot force an update in save() with no primary key."
pass
return params
def _do_mark_result_unchanged(workflow_id: int, wf_module: WfModule,
now: timezone.datetime) -> None:
"""
Do database manipulations for mark_result_unchanged().
Modify `wf_module` in-place.
Raise WfModule.DoesNotExist or Workflow.DoesNotExist in case of a race.
"""
with _locked_wf_module(workflow_id, wf_module):
wf_module.is_busy = False
wf_module.last_update_check = now
wf_module.save(update_fields=["is_busy", "last_update_check"])
def get_migrated_params(wf_module: WfModule) -> Dict[str, Any]:
"""
Read `wf_module.params`, calling migrate_params() or using cache fields.
Call this within a `Workflow.cooperative_lock()`.
If migrate_params() was already called for this version of the module,
return the cached value. See `wf_module.cached_migrated_params`,
`wf_module.cached_migrated_params_module_version`.
Raise `ModuleError` if migration fails.
Return `{}` if the module was deleted.
The result may be invalid. Call `validate()` to raise a `ValueError` to
detect that case.
"""
module_version = wf_module.module_version
if module_version is None:
return {}
if module_version.source_version_hash == "develop":
stale = True
elif (
# works if cached version (and thus cached _result_) is None
module_version.param_schema_version !=
wf_module.cached_migrated_params_module_version):
stale = True
else:
stale = False
if not stale:
return wf_module.cached_migrated_params
else:
loaded_module = LoadedModule.for_module_version(module_version)
if loaded_module:
params = wf_module.params # the user-supplied params
params = loaded_module.migrate_params(params) # raises ModuleError
wf_module.cached_migrated_params = params
wf_module.cached_migrated_params_module_version = (
module_version.param_schema_version)
# Write to DB, like wf_module.save(fields=[...]), even if the
# WfModule was deleted in a race
WfModule.objects.filter(id=wf_module.id).update(
cached_migrated_params=wf_module.cached_migrated_params,
cached_migrated_params_module_version=(
wf_module.cached_migrated_params_module_version),
)
return params
else:
return {}
def _wf_module_set_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str,
secret: str) -> Optional[clientside.Update]:
"""
Write a new secret to `wf_module`, or raise.
Return a `clientside.Update`, or `None` if the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param, {}).get("secret") == secret:
return None # no-op
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
raise HandlerError(
f"BadRequest: ModuleZipfile {wf_module.module_id_name} does not exist"
)
module_spec = module_zipfile.get_spec()
if not any(p.type == "secret" and p.secret_logic.provider == "string"
for p in module_spec.param_fields):
raise HandlerError(
f"BadRequest: param is not a secret string parameter")
created_at = timezone.now()
created_at_str = (
created_at.strftime("%Y-%m-%dT%H:%M:%S") + "." +
created_at.strftime("%f")[0:3] # milliseconds
+ "Z")
wf_module.secrets = {
**wf_module.secrets,
param: {
"name": created_at_str,
"secret": secret
},
}
wf_module.save(update_fields=["secrets"])
return clientside.Update(steps={
wf_module.id:
clientside.StepUpdate(secrets=wf_module.secret_metadata)
})
def _wf_module_set_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str, secret: str
) -> Optional[Dict[str, Any]]:
"""
Write a new secret to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param, {}).get("secret") == secret:
return None # no-op
module_version = wf_module.module_version
if module_version is None:
raise HandlerError(f"BadRequest: ModuleVersion does not exist")
if not any(
p.type == "secret" and p.secret_logic.provider == "string"
for p in module_version.param_fields
):
raise HandlerError(f"BadRequest: param is not a secret string parameter")
created_at = timezone.now()
created_at_str = (
created_at.strftime("%Y-%m-%dT%H:%M:%S")
+ "."
+ created_at.strftime("%f")[0:3] # milliseconds
+ "Z"
)
wf_module.secrets = {
**wf_module.secrets,
param: {"name": created_at_str, "secret": secret},
}
wf_module.save(update_fields=["secrets"])
return {
"updateWfModules": {
str(wf_module.id): {"secrets": wf_module.secret_metadata}
}
}
def test_input_crr_corrupt_cache_error_is_none(
self, downloaded_parquet_file, load_module
):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
downloaded_parquet_file.side_effect = rendercache.CorruptCacheError(
"file not found"
)
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
input_crr,
self.output_path,
)
# fetch is still called, with `None` as argument.
self.assertIsNone(
load_module.return_value.fetch.call_args[1]["input_parquet_filename"]
)
def test_input_crr(self, downloaded_parquet_file, clean_value, load_module):
load_module.return_value.migrate_params.return_value = {}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
clean_value.return_value = {}
downloaded_parquet_file.return_value = Path("/path/to/x.parquet")
input_metadata = TableMetadata(3, [Column("A", ColumnType.Text())])
input_crr = CachedRenderResult(1, 2, 3, "ok", [], {}, input_metadata)
fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(),
MockModuleVersion(),
{},
None,
input_crr,
self.output_path,
)
# Passed file is downloaded from rendercache
downloaded_parquet_file.assert_called_with(input_crr, dir=self.basedir)
self.assertEqual(
load_module.return_value.fetch.call_args[1]["input_parquet_filename"],
"x.parquet",
)
# clean_value() is called with input metadata from CachedRenderResult
clean_value.assert_called()
self.assertEqual(clean_value.call_args[0][2], input_metadata)
def test_simple(self, load_module):
load_module.return_value.migrate_params.return_value = {"A": "B"}
load_module.return_value.fetch.return_value = FetchResult(self.output_path, [])
result = fetch.fetch_or_wrap_error(
self.ctx,
self.chroot_context,
self.basedir,
WfModule(params={"A": "input"}, secrets={"C": "wrong"}),
MockModuleVersion(
id_name="A", param_schema=ParamDType.Dict({"A": ParamDType.String()})
),
{"C": "D"},
None,
None,
self.output_path,
)
self.assertEqual(result, FetchResult(self.output_path, []))
load_module.return_value.migrate_params.assert_called_with({"A": "input"})
load_module.return_value.fetch.assert_called_with(
chroot_context=self.chroot_context,
basedir=self.basedir,
params=Params({"A": "B"}),
secrets={"C": "D"},
last_fetch_result=None,
input_parquet_filename=None,
output_filename=self.output_path.name,
)
def load_database_objects(workflow_id: int, wf_module_id: int) -> DatabaseObjects:
"""
Query WfModule info or raise WfModule.DoesNotExist/Workflow.DoesNotExist.
Return tuple containing:
* wf_module: cjwstate.models.WfModule
* module_version: Optional[cjwstate.models.ModuleVersion]
* stored_object: Optional[cjwstate.models.StoredObject]
* input_cached_render_result: Optional[cjwstate.models.CachedRenderResult]
of previous module in tab
"""
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
# raise WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(workflow_lock.workflow).get(
id=wf_module_id
)
try:
module_version = ModuleVersion.objects.latest(wf_module.module_id_name)
except ModuleVersion.DoesNotExist:
module_version = None
try:
stored_object = wf_module.stored_objects.get(
stored_at=wf_module.stored_data_version
)
except StoredObject.DoesNotExist:
stored_object = None
try:
# raise WfModule.DoesNotExist -- but we'll catch this one
prev_module = wf_module.tab.live_wf_modules.get(order=wf_module.order - 1)
input_crr = prev_module.cached_render_result # may be None
except WfModule.DoesNotExist:
input_crr = None
return DatabaseObjects(wf_module, module_version, stored_object, input_crr)
def _load_wf_module_and_service(
workflow: Workflow, wf_module_id: int,
param: str) -> Tuple[WfModule, oauth.OAuthService]:
"""
Load WfModule and OAuthService from the database, or raise.
Raise WfModule.DoesNotExist if the WfModule is deleted or missing.
Raise SecretDoesNotExist if the WfModule does not have the given param.
Invoke this within a Workflow.cooperative_lock().
"""
# raises WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(workflow).get(pk=wf_module_id)
# raises KeyError, RuntimeError
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
raise SecretDoesNotExist(
f"Module {wf_module.module_id_name} does not exist")
module_spec = module_zipfile.get_spec()
for field in module_spec.param_fields:
if (isinstance(field, ParamSpec.Secret) and field.id_name == param and
(isinstance(field.secret_logic, ParamSpec.Secret.Logic.Oauth1a) or
isinstance(field.secret_logic, ParamSpec.Secret.Logic.Oauth2))):
service_name = field.secret_logic.service
service = oauth.OAuthService.lookup_or_none(service_name)
if service is None:
raise OauthServiceNotConfigured(
f'OAuth not configured for "{service_name}" service')
return wf_module, service
else:
raise SecretDoesNotExist(
f"Param {param} does not point to an OAuth secret")
def _load_wf_module_and_service(
workflow: Workflow, wf_module_id: int,
param: str) -> Tuple[WfModule, oauth.OAuthService]:
"""
Load WfModule and OAuthService from the database, or raise.
Raise WfModule.DoesNotExist if the WfModule is deleted or missing.
Raise ModuleVersion.DoesNotExist if the WfModule does not have the
given param.
Invoke this within a Workflow.cooperative_lock().
"""
# raises WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(workflow).get(pk=wf_module_id)
# raise ModuleVersion.DoesNotExist if ModuleVersion was deleted
module_version = wf_module.module_version
if module_version is None:
raise ModuleVersion.DoesNotExist
for field in module_version.param_fields:
if (isinstance(field, ParamSpec.Secret) and field.id_name == param and
(isinstance(field.secret_logic, ParamSpec.Secret.Logic.Oauth1a) or
isinstance(field.secret_logic, ParamSpec.Secret.Logic.Oauth2))):
service_name = field.secret_logic.service
service = oauth.OAuthService.lookup_or_none(service_name)
if service is None:
raise OauthServiceNotConfigured(
f'OAuth not configured for "{service_name}" service')
return wf_module, service
else:
raise SecretDoesNotExist(
f"Param {param} does not point to an OAuth secret")
def _get_workflow_as_clientside_update(user, session,
workflow_id: int) -> WorkflowUpdateData:
"""
Return (clientside.Update, delta_id).
Raise Workflow.DoesNotExist if a race deletes the Workflow.
The purpose of this method is to hide races from users who disconnect
and reconnect while changes are being made. It's okay for things to be
slightly off, as long as users don't notice. (Long-term, we can build
better a more-correct synchronization strategy.)
"""
with Workflow.authorized_lookup_and_cooperative_lock(
"read", user, session, pk=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
update = clientside.Update(
workflow=workflow.to_clientside(),
tabs={tab.slug: tab.to_clientside()
for tab in workflow.live_tabs},
steps={
step.id: step.to_clientside()
for step in WfModule.live_in_workflow(workflow)
},
)
return WorkflowUpdateData(update, workflow.last_delta_id)
def get_workflow_as_delta_and_needs_render(self):
"""
Return (apply-delta dict, needs_render), or raise Workflow.DoesNotExist
needs_render is a (workflow_id, delta_id) pair.
"""
with Workflow.authorized_lookup_and_cooperative_lock(
"read",
self.scope["user"],
self.scope["session"],
pk=self.workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
request = RequestWrapper(self.scope["user"], self.scope["session"])
ret = {
"updateWorkflow": (WorkflowSerializer(workflow,
context={
"request": request
}).data)
}
tabs = list(workflow.live_tabs)
ret["updateTabs"] = dict(
(tab.slug, TabSerializer(tab).data) for tab in tabs)
wf_modules = list(WfModule.live_in_workflow(workflow.id))
ret["updateWfModules"] = dict(
(str(wfm.id), WfModuleSerializer(wfm).data)
for wfm in wf_modules)
if workflow.are_all_render_results_fresh():
needs_render = None
else:
needs_render = (workflow.id, workflow.last_delta_id)
return (ret, needs_render)
def _locked_wf_module(workflow_id: int, wf_module: WfModule):
"""
Refresh wf_module from database and yield with workflow lock.
Raise Workflow.DoesNotExist or WfModule.DoesNotExist in the event of a
race. (Even soft-deleted WfModule or Tab raises WfModule.DoesNotExist,
to simulate hard deletion -- because sooner or later soft-delete won't be
a thing any more.)
"""
# raise Workflow.DoesNotExist
with Workflow.lookup_and_cooperative_lock(id=workflow_id):
# raise WfModule.DoesNotExist
wf_module.refresh_from_db()
if wf_module.is_deleted or wf_module.tab.is_deleted:
raise WfModule.DoesNotExist("soft-deleted")
yield
def are_all_render_results_fresh(self):
"""Query whether all live WfModules are rendered."""
from .WfModule import WfModule
for wf_module in WfModule.live_in_workflow(self):
if wf_module.cached_render_result is None:
return False
return True
def cache_render_result(workflow: Workflow, wf_module: WfModule, delta_id: int,
result: RenderResult) -> None:
"""
Save `result` for later viewing.
Raise AssertionError if `delta_id` is not what we expect.
Since this alters data, be sure to call it within a lock:
with workflow.cooperative_lock():
wf_module.refresh_from_db() # may change delta_id
cache_render_result(workflow, wf_module, delta_id, result)
"""
assert delta_id == wf_module.last_relevant_delta_id
assert result is not None
json_bytes = json_encode(result.json).encode("utf-8")
if not result.table.metadata.columns:
if result.errors:
status = "error"
else:
status = "unreachable"
else:
status = "ok"
wf_module.cached_render_result_delta_id = delta_id
wf_module.cached_render_result_errors = result.errors
wf_module.cached_render_result_error = "" # DELETEME
wf_module.cached_render_result_quick_fixes = [] # DELETEME
wf_module.cached_render_result_status = status
wf_module.cached_render_result_json = json_bytes
wf_module.cached_render_result_columns = result.table.metadata.columns
wf_module.cached_render_result_nrows = result.table.metadata.n_rows
# Now we get to the part where things can end up inconsistent. Try to
# err on the side of not-caching when that happens.
delete_parquet_files_for_wf_module(
workflow.id, wf_module.id) # makes old cache inconsistent
wf_module.save(
update_fields=WF_MODULE_FIELDS) # makes new cache inconsistent
if result.table.metadata.columns: # only write non-zero-column tables
with tempfile_context() as parquet_path:
parquet.write(parquet_path, result.table.table)
minio.fput_file(BUCKET,
parquet_key(workflow.id, wf_module.id, delta_id),
parquet_path) # makes new cache consistent
def load_database_objects(workflow_id: int,
wf_module_id: int) -> DatabaseObjects:
"""
Query WfModule info.
Raise `WfModule.DoesNotExist` or `Workflow.DoesNotExist` if the step was
deleted.
Catch a `ModuleError` from migrate_params() and return it as part of the
`DatabaseObjects`.
"""
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
# raise WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(
workflow_lock.workflow).get(id=wf_module_id)
# module_zipfile
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
module_zipfile = None
# migrated_params_or_error
if module_zipfile is None:
migrated_params_or_error = {}
else:
try:
migrated_params_or_error = cjwstate.params.get_migrated_params(
wf_module,
module_zipfile=module_zipfile) # raise ModuleError
except ModuleError as err:
migrated_params_or_error = err
# stored_object
try:
stored_object = wf_module.stored_objects.get(
stored_at=wf_module.stored_data_version)
except StoredObject.DoesNotExist:
stored_object = None
# input_crr
try:
# raise WfModule.DoesNotExist -- but we'll catch this one
prev_module = wf_module.tab.live_wf_modules.get(
order=wf_module.order - 1)
input_crr = prev_module.cached_render_result # may be None
except WfModule.DoesNotExist:
input_crr = None
return DatabaseObjects(
wf_module,
module_zipfile,
migrated_params_or_error,
stored_object,
input_crr,
)
def clear_cached_render_result_for_wf_module(wf_module: WfModule) -> None:
"""
Delete a CachedRenderResult, if it exists.
This deletes the Parquet file from disk, _then_ empties relevant
database fields and saves them (and only them).
"""
delete_parquet_files_for_wf_module(wf_module.workflow_id, wf_module.id)
wf_module.cached_render_result_delta_id = None
wf_module.cached_render_result_errors = []
wf_module.cached_render_result_error = ""
wf_module.cached_render_result_json = b"null"
wf_module.cached_render_result_quick_fixes = []
wf_module.cached_render_result_status = None
wf_module.cached_render_result_columns = None
wf_module.cached_render_result_nrows = None
wf_module.save(update_fields=WF_MODULE_FIELDS)
def wrapper(request: HttpRequest, workflow_id: int, wf_module_slug: str,
*args, **kwargs):
auth_header = request.headers.get("Authorization", "")
auth_header_match = AuthTokenHeaderRegex.match(auth_header)
if not auth_header_match:
return ErrorResponse(403,
"authorization-bearer-token-not-provided")
bearer_token = auth_header_match.group(1)
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
try:
wf_module = WfModule.live_in_workflow(workflow).get(
slug=wf_module_slug)
except WfModule.DoesNotExist:
return ErrorResponse(404, "step-not-found")
try:
module_zipfile = MODULE_REGISTRY.latest(
wf_module.module_id_name)
except KeyError:
return ErrorResponse(400, "step-module-deleted")
try:
file_param_id_name = next(
iter(pf.id_name
for pf in module_zipfile.get_spec().param_fields
if pf.type == "file"))
except StopIteration:
return ErrorResponse(400, "step-has-no-file-param")
api_token = wf_module.file_upload_api_token
if not api_token:
return ErrorResponse(403, "step-has-no-api-token")
bearer_token_hash = hashlib.sha256(
bearer_token.encode("utf-8")).digest()
api_token_hash = hashlib.sha256(
api_token.encode("utf-8")).digest()
if bearer_token_hash != api_token_hash or bearer_token != api_token:
return ErrorResponse(403,
"authorization-bearer-token-invalid")
return f(
request,
workflow_lock,
wf_module,
file_param_id_name,
*args,
**kwargs,
)
except Workflow.DoesNotExist:
return ErrorResponse(404, "workflow-not-found")
def _write_wf_module_position(workflow: Workflow, wf_module_id: int) -> None:
"""Write position in DB, or raise (Workflow|Tab|WfModule).DoesNotExist."""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
# Raises WfModule.DoesNotExist, e.g. if tab.is_deleted
wf_module = WfModule.live_in_workflow(workflow).get(pk=wf_module_id)
tab = wf_module.tab
tab.selected_wf_module_position = wf_module.order
tab.save(update_fields=["selected_wf_module_position"])
workflow.selected_tab_position = tab.position
workflow.save(update_fields=["selected_tab_position"])
def _do_try_set_autofetch(
scope, wf_module: WfModule, auto_update_data: bool, update_interval: int
):
# We may ROLLBACK; if we do, we need to remember the old values
old_auto_update_data = wf_module.auto_update_data
old_update_interval = wf_module.update_interval
check_quota = (
auto_update_data
and wf_module.auto_update_data
and update_interval < wf_module.update_interval
) or (auto_update_data and not wf_module.auto_update_data)
quota_exceeded = None
try:
with transaction.atomic():
wf_module.auto_update_data = auto_update_data
wf_module.update_interval = update_interval
if auto_update_data:
wf_module.next_update = timezone.now() + datetime.timedelta(
seconds=update_interval
)
else:
wf_module.next_update = None
wf_module.save(
update_fields=["auto_update_data", "update_interval", "next_update"]
)
# Now before we commit, let's see if we've surpassed the user's limit;
# roll back if we have.
#
# Only rollback if we're _increasing_ our fetch count. If we're
# lowering it, allow that -- even if the user is over limit, we still
# want to commit because it's an improvement.
if check_quota:
autofetches = autofetch.list_autofetches_json(scope)
if autofetches["nFetchesPerDay"] > autofetches["maxFetchesPerDay"]:
raise AutofetchQuotaExceeded(autofetches)
except AutofetchQuotaExceeded as err:
wf_module.auto_update_data = old_auto_update_data
wf_module.update_interval = old_update_interval
quota_exceeded = err.autofetches
retval = {
"isAutofetch": wf_module.auto_update_data,
"fetchInterval": wf_module.update_interval,
}
if quota_exceeded is not None:
retval["quotaExceeded"] = quota_exceeded # a dict
return retval
def test_deleted_wf_module(self):
with self.assertLogs(level=logging.INFO):
result = fetch.fetch_or_wrap_error(
self.ctx,
self.basedir,
WfModule(),
None,
{},
None,
None,
self.output_path,
)
self.assertEqual(self.output_path.stat().st_size, 0)
self.assertEqual(result, self._err("Cannot fetch: module was deleted"))
def wfmodule_render(request: HttpRequest, wf_module: WfModule, format=None):
# Get first and last row from query parameters, or default to all if not
# specified
try:
startrow = int_or_none(request.GET.get("startrow"))
endrow = int_or_none(request.GET.get("endrow"))
except ValueError:
return Response(
{
"message": "bad row number",
"status_code": 400
},
status=status.HTTP_400_BAD_REQUEST,
)
with wf_module.workflow.cooperative_lock():
wf_module.refresh_from_db()
cached_result = wf_module.cached_render_result
if cached_result is None:
# assume we'll get another request after execute finishes
return JsonResponse({"start_row": 0, "end_row": 0, "rows": []})
try:
startrow, endrow, record_json = _make_render_tuple(
cached_result, startrow, endrow)
except CorruptCacheError:
# assume we'll get another request after execute finishes
return JsonResponse({"start_row": 0, "end_row": 0, "rows": []})
data = '{"start_row":%d,"end_row":%d,"rows":%s}' % (startrow, endrow,
record_json)
response = HttpResponse(data.encode("utf-8"),
content_type="application/json",
charset="utf-8")
add_never_cache_headers(response)
return response
def _do_create_result(workflow_id: int, wf_module: WfModule,
result: FetchResult, now: timezone.datetime) -> None:
"""
Do database manipulations for create_result().
Modify `wf_module` in-place.
Do *not* do the logic in ChangeDataVersionCommand. We're creating a new
version, not doing something undoable.
Raise WfModule.DoesNotExist or Workflow.DoesNotExist in case of a race.
"""
with _locked_wf_module(workflow_id, wf_module):
storedobjects.create_stored_object(workflow_id,
wf_module.id,
result.path,
stored_at=now)
storedobjects.enforce_storage_limits(wf_module)
wf_module.fetch_errors = result.errors
wf_module.is_busy = False
wf_module.last_update_check = now
wf_module.save(
update_fields=["fetch_errors", "is_busy", "last_update_check"])
