async def commit_result(
wf_module: WfModule,
result: ProcessResult,
stored_object_json: Optional[Dict[str, Any]] = None) -> None:
"""
Store fetched result, if it is a change from wfm's existing data.
Save the WfModule's `status` and `fetch_error`.
Set wfm.last_update_check, regardless.
If there is no error and there is new data, create (and run) a
ChangeDataVersionCommand.
Notify the user.
"""
if result.dataframe.empty and result.error:
workflow = wf_module.workflow
with workflow.cooperative_lock():
wf_module.last_update_check = timezone.now()
wf_module.fetch_error = result.error
wf_module.is_busy = False
wf_module.save()
await websockets.ws_client_rerender_workflow_async(workflow)
else:
await save_result_if_changed(wf_module,
result,
stored_object_json=stored_object_json)
async def save_result_if_changed(
wfm: WfModule,
new_result: ProcessResult,
stored_object_json: Optional[Dict[str, Any]] = None) -> None:
"""
Store fetched table, if it is a change from wfm's existing data.
"Change" here means either a changed table or changed error message.
Set `fetch_error` to `new_result.error`.
Set wfm.is_busy to False.
Set wfm.last_update_check.
Create (and run) a ChangeDataVersionCommand if something changed. This
will kick off an execute cycle, which will render each module and email the
owner if data has changed and notifications are enabled.
Otherwise, notify the user of the wfm.last_update_check.
Return the timestamp (if changed) or None (if not).
"""
with wfm.workflow.cooperative_lock():
wfm.last_update_check = timezone.now()
# Store this data only if it's different from most recent data
new_table = new_result.dataframe
version_added = wfm.store_fetched_table_if_different(
new_table, metadata=json.dumps(stored_object_json))
if version_added:
enforce_storage_limits(wfm)
wfm.is_busy = False
# TODO store fetch_error along with the data
wfm.fetch_error = new_result.error
wfm.save()
# un-indent: COMMIT so we notify the client _after_ COMMIT
if version_added:
# notifies client of status+error_msg+last_update_check
await ChangeDataVersionCommand.create(wfm, version_added)
else:
await websockets.ws_client_send_delta_async(
wfm.workflow_id, {
'updateWfModules': {
str(wfm.id): {
'status': wfm.status,
'error_msg': wfm.error_msg,
'last_update_check': wfm.last_update_check.isoformat(),
}
}
})
def save_fetched_table_if_changed(wfm: WfModule, new_table: DataFrame,
error_message: str) -> datetime.datetime:
"""Store retrieved data table, if it is a change from wfm's existing data.
"Change" here means either a changed table or changed error message.
The WfModule's `status` and `error_msg` will be set, according to
`error_message`.
Set wfm.last_update_check, regardless.
Create (and run) a ChangeDataVersionCommand.
Notify the user.
Return the timestamp (if changed) or None (if not).
"""
with wfm.workflow.cooperative_lock():
wfm.last_update_check = timezone.now()
# Store this data only if it's different from most recent data
version_added = wfm.store_fetched_table_if_different(new_table)
if version_added:
enforce_storage_limits(wfm)
wfm.error_msg = error_message or ''
wfm.status = (WfModule.ERROR if error_message else WfModule.READY)
wfm.save()
# un-indent: COMMIT, so we can notify the client and the client sees changes
if version_added:
ChangeDataVersionCommand.create(wfm,
version_added) # also notifies client
if wfm.notifications == True:
Notification.create(wfm, "New data version available")
else:
# no new data version, but we still want client to update WfModule status and last update check time
notify_client_workflow_version_changed(wfm.workflow)
return version_added
async def save_result_if_changed(
wfm: WfModule,
new_result: ProcessResult,
stored_object_json: Optional[Dict[str,
Any]] = None) -> datetime.datetime:
"""
Store fetched table, if it is a change from wfm's existing data.
"Change" here means either a changed table or changed error message.
Set `fetch_error` to `new_result.error`.
Set sfm.is_busy to False.
Set wfm.last_update_check.
Create (and run) a ChangeDataVersionCommand.
Notify the user.
Return the timestamp (if changed) or None (if not).
"""
with wfm.workflow.cooperative_lock():
wfm.last_update_check = timezone.now()
# Store this data only if it's different from most recent data
old_result = ProcessResult(dataframe=wfm.retrieve_fetched_table(),
error=wfm.error_msg)
new_table = new_result.dataframe
version_added = wfm.store_fetched_table_if_different(
new_table, metadata=json.dumps(stored_object_json))
if version_added:
enforce_storage_limits(wfm)
output_deltas = \
find_output_deltas_to_notify_from_fetched_tables(wfm,
old_result,
new_result)
else:
output_deltas = []
wfm.is_busy = False
wfm.fetch_error = new_result.error
# clears error for good fetch after bad #160367251
# TODO why not simply call render()?
wfm.cached_render_result_error = new_result.error
wfm.save()
# Mark has_unseen_notifications via direct SQL
WfModule.objects \
.filter(id__in=[od.wf_module_id for od in output_deltas]) \
.update(has_unseen_notification=True)
# un-indent: COMMIT so we notify the client _after_ COMMIT
if version_added:
# notifies client
await ChangeDataVersionCommand.create(wfm, version_added)
for output_delta in output_deltas:
email_output_delta(output_delta, version_added)
else:
# no new data version, but we still want client to update WfModule
# status and last update check time
# TODO why not just send WfModule?
await websockets.ws_client_rerender_workflow_async(wfm.workflow)
return version_added
