def test_db_minio_validate_code_with_kernel(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest7",
"name": "regtest7 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest7.yaml", json.dumps(mv.spec).encode("utf-8"))
zf.writestr(
"regtest7.py",
b"def render(table, params):\n return table\nfoo()")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest7/regtest7.b1c2d3.zip",
bytes(bio.getbuffer()),
)
with self.assertRaises(RuntimeError) as cm:
MODULE_REGISTRY.latest("regtest7")
self.assertIsInstance(cm.exception.__cause__, ModuleExitedError)
def test_db_minio_use_cache_for_same_version(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest4",
"name": "regtest4 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest4.yaml", json.dumps(mv.spec).encode("utf-8"))
zf.writestr("regtest4.py",
b"def render(table, params):\n return table")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest4/regtest4.b1c2d2.zip",
bytes(bio.getbuffer()),
)
zf1 = MODULE_REGISTRY.latest("regtest4")
zf2 = MODULE_REGISTRY.latest("regtest4")
self.assertIs(zf2, zf1)
def test_db_s3_validate_spec(self):
mv = create_or_replace_from_spec(
{
"id_name": "regtest8",
"name": "regtest8 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr(
"regtest8.yaml",
json.dumps({
**mv.spec, "parameters": "not an Array"
}).encode("utf-8"),
)
zf.writestr("regtest8.py",
b"def render(table, params):\n return table")
s3.put_bytes(
s3.ExternalModulesBucket,
"regtest8/regtest8.b1c2d3.zip",
bytes(bio.getbuffer()),
)
with self.assertRaises(RuntimeError) as cm:
MODULE_REGISTRY.latest("regtest8")
self.assertIsInstance(cm.exception.__cause__, ValueError)
def test_db_minio_syntax_error_is_runtime_error(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest9",
"name": "regtest9 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr(
"regtest9.yaml",
json.dumps({
**mv.spec, "parameters": "not an Array"
}).encode("utf-8"),
)
zf.writestr("regtest9.py", b"def render(")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest9/regtest9.b1c2d3.zip",
bytes(bio.getbuffer()),
)
with self.assertRaises(RuntimeError) as cm:
MODULE_REGISTRY.latest("regtest9")
self.assertIsInstance(cm.exception.__cause__, SyntaxError)
def test_db_minio_refresh_cache_for_new_version(self):
v1 = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest5",
"name": "regtest5 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest5.yaml", json.dumps(v1.spec).encode("utf-8"))
zf.writestr("regtest5.py",
b"def render(table, params):\n return table")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest5/regtest5.b1c2d2.zip",
bytes(bio.getbuffer()),
)
zipfile1 = MODULE_REGISTRY.latest("regtest5")
ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest5",
"name": "regtest5 v2",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest5/regtest5.b1c2d3.zip",
bytes(bio.getbuffer()), # reuse zipfile to save lines of code
)
zipfile2 = MODULE_REGISTRY.latest("regtest5")
self.assertIsNot(zipfile2, zipfile1)
self.assertEqual(zipfile2.version, "b1c2d3")
def _lookup_service(step: Step, param: str) -> oauth.OAuthService:
"""Find the OAuthService that manages `param` on `step`.
Raise `HandlerError` if we cannot.
"""
module_id = step.module_id_name
try:
module_zipfile = MODULE_REGISTRY.latest(module_id)
except KeyError:
raise HandlerError(f"BadRequest: module {module_id} not found")
module_spec = module_zipfile.get_spec()
for field in module_spec.param_fields:
if (
field.id_name == param
and isinstance(field, ParamField.Secret)
and (
isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a)
or isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2)
)
):
service_name = field.secret_logic.service
service = oauth.OAuthService.lookup_or_none(service_name)
if not service:
allowed_services = ", ".join(settings.OAUTH_SERVICES.keys())
raise HandlerError(f"AuthError: we only support {allowed_services}")
return service
else:
raise HandlerError(f"Module {module_id} has no oauth {param} parameter")
def _load_step_and_service(workflow: Workflow, step_id: int,
param: str) -> Tuple[Step, oauth.OAuthService]:
"""Load Step and OAuthService from the database, or raise.
Raise Step.DoesNotExist if the Step is deleted or missing.
Raise SecretDoesNotExist if the Step does not have the given param.
Invoke this within a Workflow.cooperative_lock().
"""
# raises Step.DoesNotExist
step = Step.live_in_workflow(workflow).get(pk=step_id)
# raises KeyError, RuntimeError
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise SecretDoesNotExist(
f"Module {step.module_id_name} does not exist")
module_spec = module_zipfile.get_spec()
for field in module_spec.param_fields:
if (isinstance(field, ParamField.Secret) and field.id_name == param and
(isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a) or
isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2))):
service_name = field.secret_logic.service
service = oauth.OAuthService.lookup_or_none(service_name)
if service is None:
raise OauthServiceNotConfigured(
f'OAuth not configured for "{service_name}" service')
return step, service
else:
raise SecretDoesNotExist(
f"Param {param} does not point to an OAuth secret")
def test_db_minio_latest_load_deprecated_simple(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest2",
"name": "regtest2 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest2/b1c2d2/regtest2.py",
"def render(table, params):\n return table",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest2/b1c2d2/regtest2.yaml",
json.dumps(mv.spec).encode("utf-8"),
)
zf = MODULE_REGISTRY.latest("regtest2")
self.assertEqual(zf.get_spec(), ModuleSpec(**mv.spec))
self.assertIsNone(zf.get_optional_html())
def build_init_state():
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise Http404("Module is not embeddable")
try:
has_html = module_zipfile.get_optional_html() is not None
except (UnicodeDecodeError, FileNotFoundError, zipfile.BadZipFile):
has_html = False
if not has_html:
raise Http404("Module has no HTML")
ctx = JsonizeContext(
user=AnonymousUser(),
user_profile=None,
session=None,
locale_id=request.locale_id,
module_zipfiles={module_zipfile.module_id: module_zipfile},
)
return {
"workflow":
jsonize_clientside_workflow(
workflow.to_clientside(
include_tab_slugs=False,
include_block_slugs=False,
include_acl=False,
),
ctx,
is_init=True,
),
"step":
jsonize_clientside_step(step.to_clientside(), ctx),
}
def wfmodule_output(request: HttpRequest, wf_module: WfModule, format=None):
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
html = module_zipfile.get_optional_html()
except KeyError:
html = None
return HttpResponse(content=html)
def test_db_minio_latest_load_deprecated_html(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest3",
"name": "regtest3 v2",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest3/b1c2d2/regtest3.py",
"def render(table, params):\n return table",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest3/b1c2d2/regtest3.yaml",
json.dumps(mv.spec).encode("utf-8"),
)
html = "<!DOCTYPE html><html><head><title>Hi</title></head><body>Hello, world!</body></html>"
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest3/b1c2d2/regtest3.html",
html.encode("utf-8"),
)
zf = MODULE_REGISTRY.latest("regtest3")
self.assertEqual(zf.get_optional_html(), html)
def visible_modules(request) -> Dict[str, ModuleZipfile]:
"""Load all ModuleZipfiles the user may use."""
ret = dict(MODULE_REGISTRY.all_latest()) # shallow copy
if not request.user.is_authenticated and "pythoncode" in ret:
del ret["pythoncode"]
return ret
def load_database_objects(workflow_id: int,
wf_module_id: int) -> DatabaseObjects:
"""
Query WfModule info.
Raise `WfModule.DoesNotExist` or `Workflow.DoesNotExist` if the step was
deleted.
Catch a `ModuleError` from migrate_params() and return it as part of the
`DatabaseObjects`.
"""
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
# raise WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(
workflow_lock.workflow).get(id=wf_module_id)
# module_zipfile
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
module_zipfile = None
# migrated_params_or_error
if module_zipfile is None:
migrated_params_or_error = {}
else:
try:
migrated_params_or_error = cjwstate.params.get_migrated_params(
wf_module,
module_zipfile=module_zipfile) # raise ModuleError
except ModuleError as err:
migrated_params_or_error = err
# stored_object
try:
stored_object = wf_module.stored_objects.get(
stored_at=wf_module.stored_data_version)
except StoredObject.DoesNotExist:
stored_object = None
# input_crr
try:
# raise WfModule.DoesNotExist -- but we'll catch this one
prev_module = wf_module.tab.live_wf_modules.get(
order=wf_module.order - 1)
input_crr = prev_module.cached_render_result # may be None
except WfModule.DoesNotExist:
input_crr = None
return DatabaseObjects(
wf_module,
module_zipfile,
migrated_params_or_error,
stored_object,
input_crr,
)
def get_migrated_params(
wf_module: WfModule,
*,
module_zipfile: ModuleZipfile = None) -> Dict[str, Any]:
"""
Read `wf_module.params`, calling migrate_params() or using cache fields.
Call this within a `Workflow.cooperative_lock()`.
If migrate_params() was already called for this version of the module,
return the cached value. See `wf_module.cached_migrated_params`,
`wf_module.cached_migrated_params_module_version`.
Raise `ModuleError` if migration fails.
Raise `KeyError` if the module was deleted.
Raise `RuntimeError` (unrecoverable) if there is a problem loading or
executing the module. (Modules are validated before import, so this should
not happen.)
The result may be invalid. Call `validate()` to raise a `ValueError` to
detect that case.
TODO avoid holding the database lock whilst executing stuff on the kernel.
(This will involve auditing and modifying all callers to handle new error
cases.)
"""
if module_zipfile is None:
# raise KeyError
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
stale = (
module_zipfile.version == "develop"
# works if cached version (and thus cached _result_) is None
or (module_zipfile.get_param_schema_version() !=
wf_module.cached_migrated_params_module_version))
if not stale:
return wf_module.cached_migrated_params
else:
# raise ModuleError
params = invoke_migrate_params(module_zipfile, wf_module.params)
wf_module.cached_migrated_params = params
wf_module.cached_migrated_params_module_version = (
module_zipfile.get_param_schema_version())
try:
wf_module.save(update_fields=[
"cached_migrated_params",
"cached_migrated_params_module_version",
])
except ValueError:
# WfModule was deleted, so we get:
# "ValueError: Cannot force an update in save() with no primary key."
pass
return params
def to_clientside(
self, *, force_module_zipfile: Optional[ModuleZipfile] = None
) -> clientside.StepUpdate:
# module_zipfile, for params
if force_module_zipfile:
module_zipfile = force_module_zipfile
else:
from cjwstate.models.module_registry import MODULE_REGISTRY
try:
module_zipfile = MODULE_REGISTRY.latest(self.module_id_name)
except KeyError:
module_zipfile = None
if module_zipfile is None:
params = {}
else:
from cjwstate.params import get_migrated_params
module_spec = module_zipfile.get_spec()
param_schema = module_spec.param_schema
# raise ModuleError
params = get_migrated_params(self, module_zipfile=module_zipfile)
try:
param_schema.validate(params)
except ValueError:
logger.exception(
"%s.migrate_params() gave invalid output: %r",
self.module_id_name,
params,
)
params = param_schema.default
crr = self._build_cached_render_result_fresh_or_not()
if crr is None:
crr = clientside.Null
return clientside.StepUpdate(
id=self.id,
slug=self.slug,
module_slug=self.module_id_name,
tab_slug=self.tab_slug,
is_busy=self.is_busy,
render_result=crr,
files=self._get_clientside_files(module_zipfile),
params=params,
secrets=self.secret_metadata,
is_collapsed=self.is_collapsed,
notes=self.notes,
is_auto_fetch=self.auto_update_data,
fetch_interval=self.update_interval,
last_fetched_at=self.last_update_check,
is_notify_on_change=self.notifications,
last_relevant_delta_id=self.last_relevant_delta_id,
versions=self._get_clientside_fetched_version_list(module_zipfile),
)
def wrapper(request: HttpRequest, workflow_id: int, wf_module_slug: str,
*args, **kwargs):
auth_header = request.headers.get("Authorization", "")
auth_header_match = AuthTokenHeaderRegex.match(auth_header)
if not auth_header_match:
return ErrorResponse(403,
"authorization-bearer-token-not-provided")
bearer_token = auth_header_match.group(1)
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
try:
wf_module = WfModule.live_in_workflow(workflow).get(
slug=wf_module_slug)
except WfModule.DoesNotExist:
return ErrorResponse(404, "step-not-found")
try:
module_zipfile = MODULE_REGISTRY.latest(
wf_module.module_id_name)
except KeyError:
return ErrorResponse(400, "step-module-deleted")
try:
file_param_id_name = next(
iter(pf.id_name
for pf in module_zipfile.get_spec().param_fields
if pf.type == "file"))
except StopIteration:
return ErrorResponse(400, "step-has-no-file-param")
api_token = wf_module.file_upload_api_token
if not api_token:
return ErrorResponse(403, "step-has-no-api-token")
bearer_token_hash = hashlib.sha256(
bearer_token.encode("utf-8")).digest()
api_token_hash = hashlib.sha256(
api_token.encode("utf-8")).digest()
if bearer_token_hash != api_token_hash or bearer_token != api_token:
return ErrorResponse(403,
"authorization-bearer-token-invalid")
return f(
request,
workflow_lock,
wf_module,
file_param_id_name,
*args,
**kwargs,
)
except Workflow.DoesNotExist:
return ErrorResponse(404, "workflow-not-found")
def load_from_workflow(cls, workflow: Workflow) -> DependencyGraph:
"""Create a DependencyGraph using the database.
Must be called within a `workflow.cooperative_lock()`.
Missing or deleted modules are deemed to have no dependencies.
"""
from cjwstate.models.module_registry import MODULE_REGISTRY
module_zipfiles = MODULE_REGISTRY.all_latest()
tabs = []
steps = {}
for tab in workflow.live_tabs:
tab_step_ids = []
for step in tab.live_steps:
tab_step_ids.append(step.id)
try:
module_zipfile = module_zipfiles[step.module_id_name]
except KeyError:
steps[step.id] = cls.Step(set())
continue
module_spec = module_zipfile.get_spec()
schema = module_spec.get_param_schema()
# Optimization: don't migrate_params() if we know there are no
# tab params. (get_migrated_params() invokes module code, and
# we'd prefer for module code to execute only in the renderer.)
if all(((not isinstance(dtype, ParamDType.Tab)
and not isinstance(dtype, ParamDType.Multitab))
for dtype, v in schema.iter_dfs_dtype_values(
schema.coerce(None)))):
# There are no tab params.
steps[step.id] = cls.Step(set())
continue
from cjwstate.params import get_migrated_params
params = get_migrated_params(step)
# raises ValueError (and we don't handle that right now)
schema.validate(params)
tab_slugs = frozenset(
v for dtype, v in schema.iter_dfs_dtype_values(params)
if isinstance(dtype, ParamDType.Tab))
steps[step.id] = cls.Step(tab_slugs)
tabs.append(cls.Tab(tab.slug, tab_step_ids))
return cls(tabs, steps)
def create_module_zipfile(
module_id: str = "testmodule",
*,
version: Optional[str] = None,
spec_kwargs: Dict[str, Any] = {},
python_code: str = "",
html: Optional[str] = None,
js_module: str = "",
extra_file_contents: Dict[str, bytes] = {},
) -> ModuleZipfile:
"""
Create a ModuleZipfile, stored in the database and s3.
If `version` is not supplied, generate one using the sha1 of the zipfile.
This is usually what you want: s3 reads on overwrites are _eventually_
consistent, so if you 1. write a file; 2. overwrite it; and 3. read it, the
read might result in the file from step 1 or the file from step 2. A sha1
version means overwrites will never modify data, solving the problem.
"""
spec = {
"id_name": module_id,
"name": "Test Module",
"category": "Clean",
"parameters": [],
**spec_kwargs,
}
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr(module_id + ".yaml", json.dumps(spec))
zf.writestr(module_id + ".py", python_code.encode("utf-8"))
if html is not None:
zf.writestr(module_id + ".html", html.encode("utf-8"))
if js_module:
zf.writestr(module_id + ".js", js_module.encode("utf-8"))
for path, content in extra_file_contents.items():
zf.writestr(path, content)
data = bytes(bio.getbuffer())
if version is None:
sha1 = hashlib.sha1()
sha1.update(data)
version = sha1.hexdigest()
s3.put_bytes(
s3.ExternalModulesBucket,
"%s/%s.%s.zip" % (module_id, module_id, version),
data,
)
ModuleVersion.objects.create(id_name=module_id,
source_version_hash=version,
spec=spec,
js_module=js_module)
return MODULE_REGISTRY.latest(module_id)
def amend_create_kwargs(self, *, step, new_values, **kwargs):
"""Prepare values_for_backward|forward["params"].
Raise ValueError if `values_for_forward["params"]` won't be valid
according to the module spec.
"""
if _step_is_deleted(step): # refreshes from DB
return None
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise ValueError("Module %s does not exist" % step.module_id_name)
# Old values: store exactly what we had
old_values = step.params
module_spec = module_zipfile.get_spec()
# [2021-07-05, adamhooper] nested here to prevent circular import error
# in fetcher. Didn't bother to find the circle.
from cjwstate.params import invoke_migrate_params
# New values: store _migrated_ old_values, with new_values applied on
# top
migrated_old_values = invoke_migrate_params(module_zipfile, old_values)
# Ensure migrate_params() didn't generate buggy _old_ values before we
# add _new_ values. This sanity check may protect users' params by
# raising an error early. It's also a way to catch bugs in unit tests.
# (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params
# returns `{}` -- which is often invalid -- and then the `**new_values`
# below overwrites the invalid data. So without this validate(), a unit
# test with an invalid migrate_params() may pass, which is wrong.)
#
# If you're seeing this because your unit test failed, try this:
# self.kernel.migrate_params.side_effect = lambda m, p: p
module_spec.param_schema.validate(
migrated_old_values) # raises ValueError
new_values = {**migrated_old_values, **new_values}
module_spec.param_schema.validate(new_values) # raises ValueError
return {
**kwargs,
"step": step,
"values_for_backward": {
"params": old_values
},
"values_for_forward": {
"params": new_values
},
"step_delta_ids": self.affected_step_delta_ids(step),
}
def create_application() -> ProtocolTypeRouter:
"""Create an ASGI application."""
# Load static modules on startup.
#
# This means starting a kernel and validating all static modules. There are
# two good reasons to load during startup:
#
# 1. In dev mode, this reports errors in modules ASAP -- during startup
# 2. In production, this import line costs time -- better to incur that
# cost during startup than to incur it when responding to some random
# request.
cjwstate.modules.init_module_system()
if not settings.I_AM_TESTING:
# Only the test environment, Django runs migrations itself. We can't
# use MODULE_REGISTRY until it migrates.
MODULE_REGISTRY.all_latest()
return ProtocolTypeRouter({
"websocket":
AuthMiddlewareStack(SetCurrentLocaleAsgiMiddleware(
create_url_router()))
})
def _wf_module_set_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str,
secret: str) -> Optional[clientside.Update]:
"""
Write a new secret to `wf_module`, or raise.
Return a `clientside.Update`, or `None` if the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param, {}).get("secret") == secret:
return None # no-op
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
raise HandlerError(
f"BadRequest: ModuleZipfile {wf_module.module_id_name} does not exist"
)
module_spec = module_zipfile.get_spec()
if not any(p.type == "secret" and p.secret_logic.provider == "string"
for p in module_spec.param_fields):
raise HandlerError(
f"BadRequest: param is not a secret string parameter")
created_at = timezone.now()
created_at_str = (
created_at.strftime("%Y-%m-%dT%H:%M:%S") + "." +
created_at.strftime("%f")[0:3] # milliseconds
+ "Z")
wf_module.secrets = {
**wf_module.secrets,
param: {
"name": created_at_str,
"secret": secret
},
}
wf_module.save(update_fields=["secrets"])
return clientside.Update(steps={
wf_module.id:
clientside.StepUpdate(secrets=wf_module.secret_metadata)
})
def post(self, request: HttpRequest, workflow_id_or_secret_id: Union[int,
str]):
workflow = lookup_workflow_and_auth(authorized_read,
workflow_id_or_secret_id, request)
workflow2 = workflow.duplicate(request.user)
ctx = JsonizeContext(request.locale_id, MODULE_REGISTRY.all_latest())
json_dict = jsonize_clientside_workflow(workflow2.to_clientside(),
ctx,
is_init=True)
async_to_sync(rabbitmq.queue_render)(workflow2.id,
workflow2.last_delta_id)
return JsonResponse(json_dict, status=status.CREATED)
def load_from_workflow(cls, workflow: Workflow) -> DependencyGraph:
"""Create a DependencyGraph using the database.
Must be called within a `workflow.cooperative_lock()`.
Missing or deleted modules are deemed to have no dependencies.
"""
from cjwstate.models.module_registry import MODULE_REGISTRY
module_zipfiles = MODULE_REGISTRY.all_latest()
tabs = []
steps = {}
for tab in workflow.live_tabs:
tab_step_ids = []
for step in tab.live_steps:
tab_step_ids.append(step.id)
try:
module_zipfile = module_zipfiles[step.module_id_name]
except KeyError:
steps[step.id] = cls.Step(frozenset())
continue
module_spec = module_zipfile.get_spec()
schema = module_spec.param_schema
# Optimization: don't migrate_params() if we know there are no
# tab params. (get_migrated_params() invokes module code, so we
# prefer to wait and let it run in the renderer.
if not _schema_contains_tabs(schema):
steps[step.id] = cls.Step(frozenset())
continue
from cjwstate.params import get_migrated_params
params = get_migrated_params(step)
# raises ValueError (and we don't handle that right now)
schema.validate(params)
steps[step.id] = cls.Step(
gather_param_tab_slugs(schema, params))
tabs.append(cls.Tab(tab.slug, tab_step_ids))
return cls(tabs, steps)
def _add_wf_module_to_tab(step_dict, order, tab, delta_id, lesson):
"""
Deserialize a WfModule from lesson initial_workflow.
Raise `KeyError` if a module ID is invalid.
"""
id_name = step_dict["module"]
slug = step_dict["slug"]
# 500 error if bad module id name
module_zipfile = MODULE_REGISTRY.latest(
id_name) # raise KeyError, RuntimeError
module_spec = module_zipfile.get_spec()
# All params not set in json get default values
# Also, we must have a dict with all param values set or we can't migrate
# params later
params = {**module_spec.default_params, **step_dict["params"]}
# Rewrite 'url' params: if the spec has them as relative, make them the
# absolute path -- relative to the lesson URL.
if "url" in params:
if params["url"].startswith("./"):
params["url"] = "".join([
settings.STATIC_URL,
("lessons/" if lesson.course is None else "courses/"),
f"{lesson.locale_id}/",
(lesson.slug if lesson.course is None else
f"{lesson.course.slug}/{lesson.slug}"),
params["url"][1:], # include the '/'
])
# 500 error if params are invalid
# TODO testme
module_spec.get_param_schema().validate(params) # raises ValueError
return tab.wf_modules.create(
order=order,
slug=slug,
module_id_name=id_name,
is_busy=module_spec.loads_data, # assume we'll send a fetch
last_relevant_delta_id=delta_id,
params=params,
is_collapsed=step_dict.get("collapsed", False),
notes=step_dict.get("note", None),
)
def post(self, request: HttpRequest, workflow_id: int):
workflow = lookup_workflow_and_auth(
Workflow.request_authorized_read, workflow_id, request
)
workflow2 = workflow.duplicate(request.user)
ctx = _get_request_jsonize_context(request, MODULE_REGISTRY.all_latest())
json_dict = jsonize_clientside_workflow(
workflow2.to_clientside(), ctx, is_init=True
)
server.utils.log_user_event_from_request(
request, "Duplicate Workflow", {"name": workflow.name}
)
async_to_sync(rabbitmq.queue_render)(workflow2.id, workflow2.last_delta_id)
return JsonResponse(json_dict, status=status.CREATED)
def amend_create_kwargs(
self, *, workflow, tab, slug, module_id_name, position, param_values, **kwargs
):
"""Add a step to the tab.
Raise KeyError if `module_id_name` is invalid.
Raise RuntimeError (unrecoverable) if s3 holds invalid module data.
Raise ValueError if `param_values` do not match the module's spec.
"""
from ..step import Step
# ensure slug is unique, or raise ValueError
if Step.objects.filter(tab__workflow_id=workflow.id, slug=slug).count() > 0:
raise ValueError("slug is not unique. Please pass a unique slug.")
# raise KeyError, RuntimeError
module_zipfile = MODULE_REGISTRY.latest(module_id_name)
module_spec = module_zipfile.get_spec()
# Set _all_ params (not just the user-specified ones). Our
# dropdown-menu actions only specify the relevant params and expect us
# to set the others to defaults.
params = {**module_spec.default_params, **param_values}
module_spec.get_param_schema().validate(params) # raises ValueError
# step starts off "deleted" and gets un-deleted in forward().
step = tab.steps.create(
module_id_name=module_id_name,
order=position,
slug=slug,
is_deleted=True,
params=params,
cached_migrated_params=params,
cached_migrated_params_module_version=module_zipfile.version,
secrets={},
)
return {
**kwargs,
"workflow": workflow,
"step": step,
"step_delta_ids": self.affected_step_delta_ids(step),
}
def amend_create_kwargs(cls, *, wf_module, new_values, **kwargs):
"""
Prepare `old_values` and `new_values`.
Raise ValueError if `new_values` won't be valid according to the module
spec.
"""
if cls.wf_module_is_deleted(wf_module): # refreshes from DB
return None
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
raise ValueError("Module %s does not exist" % wf_module.module_id_name)
# Old values: store exactly what we had
old_values = wf_module.params
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
# New values: store _migrated_ old_values, with new_values applied on
# top
migrated_old_values = invoke_migrate_params(module_zipfile, old_values)
# Ensure migrate_params() didn't generate buggy _old_ values before we
# add _new_ values. This sanity check may protect users' params by
# raising an error early. It's also a way to catch bugs in unit tests.
# (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params
# returns `{}` -- which is often invalid -- and then the `**new_values`
# below overwrites the invalid data. So without this validate(), a unit
# test with an invalid migrate_params() may pass, which is wrong.)
#
# If you're seeing this because your unit test failed, try this:
# self.kernel.migrate_params.side_effect = lambda m, p: p
param_schema.validate(migrated_old_values) # raises ValueError
new_values = {**migrated_old_values, **new_values}
param_schema.validate(new_values) # raises ValueError
return {
**kwargs,
"wf_module": wf_module,
"new_values": new_values,
"old_values": old_values,
"wf_module_delta_ids": cls.affected_wf_module_delta_ids(wf_module),
}
def from_workflow(cls, workflow: Workflow) -> Report.ReportWorkflow:
module_zipfiles = MODULE_REGISTRY.all_latest()
# prefetch would be nice, but it's tricky because A) we need to
# filter out is_deleted; and B) we need to filter for modules that
# have .html files.
all_tabs = [
Report.TabWithIframes.from_tab(tab, module_zipfiles)
for tab in workflow.live_tabs
]
tabs = [tab for tab in all_tabs if tab.wf_modules]
return cls(
id=workflow.id,
name=workflow.name,
owner_name=workbench_user_display(workflow.owner),
updated_at=workflow.last_delta.datetime,
tabs=tabs,
)
def post(self, request: HttpRequest, workflow: Workflow):
workflow2 = workflow.duplicate(request.user)
ctx = JsonizeContext(
request.user,
request.session,
request.locale_id,
dict(MODULE_REGISTRY.all_latest()),
)
json_dict = jsonize_clientside_workflow(workflow2.to_clientside(),
ctx,
is_init=True)
server.utils.log_user_event_from_request(request, "Duplicate Workflow",
{"name": workflow.name})
async_to_sync(rabbitmq.queue_render)(workflow2.id,
workflow2.last_delta_id)
return JsonResponse(json_dict, status=status.HTTP_201_CREATED)
def _load_tab_flows(workflow: Workflow, delta_id: int) -> List[TabFlow]:
"""Query `workflow` for each tab's `TabFlow` (ordered by tab position).
Raise `ModuleError` or `ValueError` if migrate_params() fails. Failed
migration means the whole execute can't happen.
"""
ret = []
with workflow.cooperative_lock(): # reloads workflow
if workflow.last_delta_id != delta_id:
raise UnneededExecution
module_zipfiles = MODULE_REGISTRY.all_latest()
for tab_model in workflow.live_tabs.all():
steps = [
_build_execute_step(step, module_zipfiles=module_zipfiles)
for step in tab_model.live_steps.all()
]
ret.append(TabFlow(Tab(tab_model.slug, tab_model.name), steps))
return ret
