def _wf_module_delete_secret_and_build_delta(
workflow: Workflow,
wf_module: WfModule,
param: str
) -> Optional[Dict[str, Any]]:
"""
Write a new secret (or `None`) to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database has not been modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
wf_module.refresh_from_db() # may return None
if (
wf_module is None
or wf_module.secrets.get(param) is None
):
return None
wf_module.secrets = dict(wf_module.secrets)
del wf_module.secrets[param]
wf_module.save(update_fields=['secrets'])
return {
'updateWfModules': {
str(wf_module.id): {
'params': wf_module.get_params().as_dict()
}
}
}
def _wf_module_delete_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule,
param: str) -> Optional[Dict[str, Any]]:
"""
Write a new secret (or `None`) to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database has not been modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param) is None:
return None # no-op
wf_module.secrets = dict(wf_module.secrets) # shallow copy
del wf_module.secrets[param]
wf_module.save(update_fields=['secrets'])
return {
'updateWfModules': {
str(wf_module.id): {
'secrets': wf_module.secret_metadata,
}
}
}
def wfmodule_render(request: HttpRequest, wf_module: WfModule, format=None):
# Get first and last row from query parameters, or default to all if not
# specified
try:
startrow = int_or_none(request.GET.get("startrow"))
endrow = int_or_none(request.GET.get("endrow"))
except ValueError:
return Response(
{"message": "bad row number", "status_code": 400},
status=status.HTTP_400_BAD_REQUEST,
)
with wf_module.workflow.cooperative_lock():
wf_module.refresh_from_db()
cached_result = wf_module.cached_render_result
if cached_result is None:
# assume we'll get another request after execute finishes
return JsonResponse({"start_row": 0, "end_row": 0, "rows": []})
try:
startrow, endrow, records = _make_render_tuple(
cached_result, startrow, endrow
)
except FileNotFoundError:
# assume we'll get another request after execute finishes
return JsonResponse({"start_row": 0, "end_row": 0, "rows": []})
return JsonResponse({"start_row": startrow, "end_row": endrow, "rows": records})
def _do_complete_multipart_upload(
workflow: Workflow,
wf_module: WfModule,
) -> Tuple[UploadedFile, Dict[str, Any]]:
with workflow.cooperative_lock():
wf_module.refresh_from_db()
uploaded_file = (
_write_uploaded_file_and_clear_inprogress_file_upload(wf_module)
)
return (
uploaded_file.uuid,
serializers.WfModuleSerializer(wf_module).data,
)
def _wf_module_set_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str,
secret: str) -> Optional[Dict[str, Any]]:
"""
Write a new secret to `wf_module`, or raise.
Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if
the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param, {}).get('secret') == secret:
return None # no-op
module_version = wf_module.module_version
if module_version is None:
raise HandlerError(f'BadRequest: ModuleVersion does not exist')
if not any(p.type == 'secret' and p.secret_logic.provider == 'string'
for p in module_version.param_fields):
raise HandlerError(
f'BadRequest: param is not a secret string parameter')
created_at = timezone.now()
created_at_str = (
created_at.strftime('%Y-%m-%dT%H:%M:%S') + '.' +
created_at.strftime('%f')[0:3] # milliseconds
+ 'Z')
wf_module.secrets = {
**wf_module.secrets, param: {
'name': created_at_str,
'secret': secret,
}
}
wf_module.save(update_fields=['secrets'])
return {
'updateWfModules': {
str(wf_module.id): {
'secrets': wf_module.secret_metadata,
}
}
}
def _do_finish_upload(workflow: Workflow, wf_module: WfModule,
uuid: uuidgen.UUID, filename: str) -> Dict[str, Any]:
with workflow.cooperative_lock():
wf_module.refresh_from_db()
try:
in_progress_upload = wf_module.in_progress_uploads.get(
id=uuid, is_completed=False)
except InProgressUpload.DoesNotExist:
raise HandlerError(
"BadRequest: key is not being uploaded for this WfModule right now. "
"(Even a valid key becomes invalid after you create, finish or abort "
"an upload on its WfModule.)")
try:
in_progress_upload.convert_to_uploaded_file(filename)
except FileNotFoundError:
raise HandlerError(
"BadRequest: file not found. "
"You must upload the file before calling finish_upload.")
return serializers.WfModuleSerializer(wf_module).data
def _do_complete_upload(
workflow: Workflow,
wf_module: WfModule,
key: str
) -> Tuple[UploadedFile, Dict[str, Any]]:
with workflow.cooperative_lock():
wf_module.refresh_from_db()
if (
wf_module.inprogress_file_upload_id is not None
or wf_module.inprogress_file_upload_key != key
):
raise HandlerError(
'DoesNotExist: key must point to an incomplete upload'
)
uploaded_file = (
_write_uploaded_file_and_clear_inprogress_file_upload(wf_module)
)
return (
uploaded_file.uuid,
serializers.WfModuleSerializer(wf_module).data,
)
def _do_create_multipart_upload(
workflow: Workflow,
wf_module: WfModule,
filename: str
) -> Dict[str, str]:
key = _generate_key(wf_module, filename)
with workflow.cooperative_lock():
wf_module.refresh_from_db()
wf_module.abort_inprogress_upload() # in case there is one already
upload_id = minio.create_multipart_upload(minio.UserFilesBucket, key,
filename)
wf_module.inprogress_file_upload_id = upload_id
wf_module.inprogress_file_upload_key = key
wf_module.inprogress_file_upload_last_accessed_at = timezone.now()
wf_module.save(
update_fields=['inprogress_file_upload_id',
'inprogress_file_upload_key',
'inprogress_file_upload_last_accessed_at']
)
return {'key': key, 'uploadId': upload_id}
def _do_prepare_upload(
workflow: Workflow,
wf_module: WfModule,
filename: str,
n_bytes: int,
base64Md5sum: str,
) -> Dict[str, str]:
key = _generate_key(wf_module, filename)
with workflow.cooperative_lock():
wf_module.refresh_from_db()
wf_module.abort_inprogress_upload()
url, headers = minio.presign_upload(minio.UserFilesBucket, key,
filename, n_bytes, base64Md5sum)
wf_module.inprogress_file_upload_id = None
wf_module.inprogress_file_upload_key = key
wf_module.inprogress_file_upload_last_accessed_at = timezone.now()
wf_module.save(
update_fields=['inprogress_file_upload_id',
'inprogress_file_upload_key',
'inprogress_file_upload_last_accessed_at']
)
return {'key': key, 'url': url, 'headers': headers}
def _do_create_upload(workflow: Workflow, wf_module: WfModule) -> Dict[str, Any]:
with workflow.cooperative_lock():
wf_module.refresh_from_db()
in_progress_upload = wf_module.in_progress_uploads.create()
return in_progress_upload.generate_upload_parameters()