def _do_set_notifications(scope, wf_module: WfModule, notifications: bool):
wf_module.notifications = notifications
wf_module.save(update_fields=["notifications"])
if notifications:
server.utils.log_user_event_from_scope(
scope, "Enabled email notifications", {"wfModuleId": wf_module.id}
)
def _wf_module_delete_secret_and_build_delta(
workflow: Workflow,
wf_module: WfModule,
param: str
) -> Optional[Dict[str, Any]]:
"""
Write a new secret (or `None`) to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database has not been modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
wf_module.refresh_from_db() # may return None
if (
wf_module is None
or wf_module.secrets.get(param) is None
):
return None
wf_module.secrets = dict(wf_module.secrets)
del wf_module.secrets[param]
wf_module.save(update_fields=['secrets'])
return {
'updateWfModules': {
str(wf_module.id): {
'params': wf_module.get_params().as_dict()
}
}
}
def execute_wfmodule(wf_module: WfModule) -> ProcessResult:
"""
Process all WfModules until the given one; return its result.
By default, this will both read and write each WfModule's cached render
result. Pass nocache=True to avoid modifying the cache.
You must call this within a workflow.cooperative_lock().
"""
# Do we already have what we need? If so, return quickly.
cached_result = _get_render_cache(wf_module)
if cached_result:
return cached_result.result
# Recurse -- ensuring the smallest possible number of renders
input_wf_module = wf_module.previous_in_stack()
if input_wf_module:
input_result = execute_wfmodule(input_wf_module)
else:
input_result = ProcessResult()
result = dispatch.module_dispatch_render(wf_module, input_result.dataframe)
wf_module.cache_render_result(wf_module.last_relevant_delta_id, result)
wf_module.save()
return result
async def commit_result(
wf_module: WfModule,
result: ProcessResult,
stored_object_json: Optional[Dict[str, Any]] = None) -> None:
"""
Store fetched result, if it is a change from wfm's existing data.
Save the WfModule's `status` and `fetch_error`.
Set wfm.last_update_check, regardless.
If there is no error and there is new data, create (and run) a
ChangeDataVersionCommand.
Notify the user.
"""
if result.dataframe.empty and result.error:
workflow = wf_module.workflow
with workflow.cooperative_lock():
wf_module.last_update_check = timezone.now()
wf_module.fetch_error = result.error
wf_module.is_busy = False
wf_module.save()
await websockets.ws_client_rerender_workflow_async(workflow)
else:
await save_result_if_changed(wf_module,
result,
stored_object_json=stored_object_json)
def _wf_module_delete_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule,
param: str) -> Optional[Dict[str, Any]]:
"""
Write a new secret (or `None`) to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database has not been modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param) is None:
return None # no-op
wf_module.secrets = dict(wf_module.secrets) # shallow copy
del wf_module.secrets[param]
wf_module.save(update_fields=['secrets'])
return {
'updateWfModules': {
str(wf_module.id): {
'secrets': wf_module.secret_metadata,
}
}
}
async def save_result_if_changed(
wfm: WfModule,
new_result: ProcessResult,
stored_object_json: Optional[Dict[str, Any]] = None) -> None:
"""
Store fetched table, if it is a change from wfm's existing data.
"Change" here means either a changed table or changed error message.
Set `fetch_error` to `new_result.error`.
Set wfm.is_busy to False.
Set wfm.last_update_check.
Create (and run) a ChangeDataVersionCommand if something changed. This
will kick off an execute cycle, which will render each module and email the
owner if data has changed and notifications are enabled.
Otherwise, notify the user of the wfm.last_update_check.
Return the timestamp (if changed) or None (if not).
"""
with wfm.workflow.cooperative_lock():
wfm.last_update_check = timezone.now()
# Store this data only if it's different from most recent data
new_table = new_result.dataframe
version_added = wfm.store_fetched_table_if_different(
new_table, metadata=json.dumps(stored_object_json))
if version_added:
enforce_storage_limits(wfm)
wfm.is_busy = False
# TODO store fetch_error along with the data
wfm.fetch_error = new_result.error
wfm.save()
# un-indent: COMMIT so we notify the client _after_ COMMIT
if version_added:
# notifies client of status+error_msg+last_update_check
await ChangeDataVersionCommand.create(wfm, version_added)
else:
await websockets.ws_client_send_delta_async(
wfm.workflow_id, {
'updateWfModules': {
str(wfm.id): {
'status': wfm.status,
'error_msg': wfm.error_msg,
'last_update_check': wfm.last_update_check.isoformat(),
}
}
})
def _maybe_add_version(
workflow: Workflow,
wf_module: WfModule,
maybe_result: Optional[ProcessResult],
stored_object_json: Optional[Dict[str, Any]] = None
) -> Optional[timezone.datetime]:
"""
Apply `result` to `wf_module`.
Set `is_busy`, `fetch_error` and `last_update_check`.
Write a new `StoredObject` and returns its `datetime` if the input
`maybe_result` is non-``None`` and the result isn't the same as the
previous one. Che caller may create a ``ChangeDataVersionCommand`` to set
`wf_module`'s next data version.
If the input Workflow or WfModule is deleted, return ``None``.
"""
# Use Django `update_fields` to only write the fields we're
# editing. That's because every value in `wf_module` might be
# stale, so we must ignore those stale values.
fields = {
'is_busy': False,
'last_update_check': timezone.now(),
}
if maybe_result is not None:
fields['fetch_error'] = maybe_result.error
for k, v in fields.items():
setattr(wf_module, k, v)
try:
with wf_module.workflow.cooperative_lock():
if not WfModule.objects.filter(pk=wf_module.id,
is_deleted=False,
tab__is_deleted=False).exists():
return None
if maybe_result is not None:
version_added = wf_module.store_fetched_table_if_different(
maybe_result.dataframe, # TODO store entire result
metadata=json.dumps(stored_object_json))
else:
version_added = None
if version_added:
enforce_storage_limits(wf_module)
wf_module.save(update_fields=fields.keys())
return version_added
except Workflow.DoesNotExist:
return None
def _do_try_set_autofetch(
scope, wf_module: WfModule, auto_update_data: bool, update_interval: int
):
# We may ROLLBACK; if we do, we need to remember the old values
old_auto_update_data = wf_module.auto_update_data
old_update_interval = wf_module.update_interval
check_quota = (
auto_update_data
and wf_module.auto_update_data
and update_interval < wf_module.update_interval
) or (auto_update_data and not wf_module.auto_update_data)
quota_exceeded = None
try:
with transaction.atomic() as trans:
wf_module.auto_update_data = auto_update_data
wf_module.update_interval = update_interval
if auto_update_data:
wf_module.next_update = timezone.now() + datetime.timedelta(
seconds=update_interval
)
else:
wf_module.next_update = None
wf_module.save(
update_fields=["auto_update_data", "update_interval", "next_update"]
)
# Now before we commit, let's see if we've surpassed the user's limit;
# roll back if we have.
#
# Only rollback if we're _increasing_ our fetch count. If we're
# lowering it, allow that -- even if the user is over limit, we still
# want to commit because it's an improvement.
if check_quota:
autofetches = autofetch.list_autofetches_json(scope)
if autofetches["nFetchesPerDay"] > autofetches["maxFetchesPerDay"]:
raise AutofetchQuotaExceeded(autofetches)
except AutofetchQuotaExceeded as err:
wf_module.auto_update_data = old_auto_update_data
wf_module.update_interval = old_update_interval
quota_exceeded = err.autofetches
retval = {
"isAutofetch": wf_module.auto_update_data,
"fetchInterval": wf_module.update_interval,
}
if quota_exceeded is not None:
retval["quotaExceeded"] = quota_exceeded # a dict
return retval
def _wf_module_set_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str,
secret: str) -> Optional[Dict[str, Any]]:
"""
Write a new secret to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param, {}).get('secret') == secret:
return None # no-op
module_version = wf_module.module_version
if module_version is None:
raise HandlerError(f'BadRequest: ModuleVersion does not exist')
if not any(p.type == 'secret' and p.secret_logic.provider == 'string'
for p in module_version.param_fields):
raise HandlerError(
f'BadRequest: param is not a secret string parameter')
created_at = timezone.now()
created_at_str = (
created_at.strftime('%Y-%m-%dT%H:%M:%S') + '.' +
created_at.strftime('%f')[0:3] # milliseconds
+ 'Z')
wf_module.secrets = {
**wf_module.secrets, param: {
'name': created_at_str,
'secret': secret,
}
}
wf_module.save(update_fields=['secrets'])
return {
'updateWfModules': {
str(wf_module.id): {
'secrets': wf_module.secret_metadata,
}
}
}
def save_fetched_table_if_changed(wfm: WfModule, new_table: DataFrame,
error_message: str) -> datetime.datetime:
"""Store retrieved data table, if it is a change from wfm's existing data.
"Change" here means either a changed table or changed error message.
The WfModule's `status` and `error_msg` will be set, according to
`error_message`.
Set wfm.last_update_check, regardless.
Create (and run) a ChangeDataVersionCommand.
Notify the user.
Return the timestamp (if changed) or None (if not).
"""
with wfm.workflow.cooperative_lock():
wfm.last_update_check = timezone.now()
# Store this data only if it's different from most recent data
version_added = wfm.store_fetched_table_if_different(new_table)
if version_added:
enforce_storage_limits(wfm)
wfm.error_msg = error_message or ''
wfm.status = (WfModule.ERROR if error_message else WfModule.READY)
wfm.save()
# un-indent: COMMIT, so we can notify the client and the client sees changes
if version_added:
ChangeDataVersionCommand.create(wfm,
version_added) # also notifies client
if wfm.notifications == True:
Notification.create(wfm, "New data version available")
else:
# no new data version, but we still want client to update WfModule status and last update check time
notify_client_workflow_version_changed(wfm.workflow)
return version_added
def _write_uploaded_file_and_clear_inprogress_file_upload(
wf_module: WfModule
) -> UploadedFile:
"""
Read metadata from S3; write it to a new UploadedFile; save `wf_module`.
Raise FileNotFoundError if `wf_module.inprogress_file_upload_key is None`
or the file does not exist on minio.
Assumptions:
* You hold a cooperative lock on `wf_module.workflow`.
* The client PUT a sensible Content-Disposition header. (Failure means icky
filename, not crash.)
"""
key = wf_module.inprogress_file_upload_key
uuid: str = key.split('/')[-1].split('.')[0]
# TODO raise FileNotFoundError
head = minio.client.head_object(Bucket=minio.UserFilesBucket, Key=key)
size = int(head['ContentLength'])
name = urllib.parse.unquote(head['ContentDisposition'].split("UTF-8''")[-1])
uploaded_file = wf_module.uploaded_files.create(
name=name,
size=size,
uuid=uuid,
bucket=minio.UserFilesBucket,
key=key,
)
wf_module.inprogress_file_upload_id = None
wf_module.inprogress_file_upload_key = None
wf_module.inprogress_file_upload_last_accessed_at = None
wf_module.save(
update_fields=['inprogress_file_upload_id',
'inprogress_file_upload_key',
'inprogress_file_upload_last_accessed_at']
)
return uploaded_file
def _do_create_multipart_upload(
workflow: Workflow,
wf_module: WfModule,
filename: str
) -> Dict[str, str]:
key = _generate_key(wf_module, filename)
with workflow.cooperative_lock():
wf_module.refresh_from_db()
wf_module.abort_inprogress_upload() # in case there is one already
upload_id = minio.create_multipart_upload(minio.UserFilesBucket, key,
filename)
wf_module.inprogress_file_upload_id = upload_id
wf_module.inprogress_file_upload_key = key
wf_module.inprogress_file_upload_last_accessed_at = timezone.now()
wf_module.save(
update_fields=['inprogress_file_upload_id',
'inprogress_file_upload_key',
'inprogress_file_upload_last_accessed_at']
)
return {'key': key, 'uploadId': upload_id}
def _do_prepare_upload(
workflow: Workflow,
wf_module: WfModule,
filename: str,
n_bytes: int,
base64Md5sum: str,
) -> Dict[str, str]:
key = _generate_key(wf_module, filename)
with workflow.cooperative_lock():
wf_module.refresh_from_db()
wf_module.abort_inprogress_upload()
url, headers = minio.presign_upload(minio.UserFilesBucket, key,
filename, n_bytes, base64Md5sum)
wf_module.inprogress_file_upload_id = None
wf_module.inprogress_file_upload_key = key
wf_module.inprogress_file_upload_last_accessed_at = timezone.now()
wf_module.save(
update_fields=['inprogress_file_upload_id',
'inprogress_file_upload_key',
'inprogress_file_upload_last_accessed_at']
)
return {'key': key, 'url': url, 'headers': headers}
def _set_collapsed_in_db(wf_module: WfModule, is_collapsed: bool) -> None:
wf_module.is_collapsed = is_collapsed
wf_module.save(update_fields=['is_collapsed'])
async def save_result_if_changed(
wfm: WfModule,
new_result: ProcessResult,
stored_object_json: Optional[Dict[str,
Any]] = None) -> datetime.datetime:
"""
Store fetched table, if it is a change from wfm's existing data.
"Change" here means either a changed table or changed error message.
Set `fetch_error` to `new_result.error`.
Set sfm.is_busy to False.
Set wfm.last_update_check.
Create (and run) a ChangeDataVersionCommand.
Notify the user.
Return the timestamp (if changed) or None (if not).
"""
with wfm.workflow.cooperative_lock():
wfm.last_update_check = timezone.now()
# Store this data only if it's different from most recent data
old_result = ProcessResult(dataframe=wfm.retrieve_fetched_table(),
error=wfm.error_msg)
new_table = new_result.dataframe
version_added = wfm.store_fetched_table_if_different(
new_table, metadata=json.dumps(stored_object_json))
if version_added:
enforce_storage_limits(wfm)
output_deltas = \
find_output_deltas_to_notify_from_fetched_tables(wfm,
old_result,
new_result)
else:
output_deltas = []
wfm.is_busy = False
wfm.fetch_error = new_result.error
# clears error for good fetch after bad #160367251
# TODO why not simply call render()?
wfm.cached_render_result_error = new_result.error
wfm.save()
# Mark has_unseen_notifications via direct SQL
WfModule.objects \
.filter(id__in=[od.wf_module_id for od in output_deltas]) \
.update(has_unseen_notification=True)
# un-indent: COMMIT so we notify the client _after_ COMMIT
if version_added:
# notifies client
await ChangeDataVersionCommand.create(wfm, version_added)
for output_delta in output_deltas:
email_output_delta(output_delta, version_added)
else:
# no new data version, but we still want client to update WfModule
# status and last update check time
# TODO why not just send WfModule?
await websockets.ws_client_rerender_workflow_async(wfm.workflow)
return version_added
def _do_set_collapsed(wf_module: WfModule, is_collapsed: bool):
wf_module.is_collapsed = is_collapsed
wf_module.save(update_fields=["is_collapsed"])
def _do_set_file_upload_api_token(wf_module: WfModule, api_token: Optional[str]):
wf_module.file_upload_api_token = api_token
wf_module.save(update_fields=["file_upload_api_token"])
def _do_clear_unseen_notification(wf_module: WfModule):
wf_module.has_unseen_notification = False
wf_module.save(update_fields=["has_unseen_notification"])
