def test_db_minio_validate_code_with_kernel(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest7",
"name": "regtest7 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest7.yaml", json.dumps(mv.spec).encode("utf-8"))
zf.writestr(
"regtest7.py",
b"def render(table, params):\n return table\nfoo()")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest7/regtest7.b1c2d3.zip",
bytes(bio.getbuffer()),
)
with self.assertRaises(RuntimeError) as cm:
MODULE_REGISTRY.latest("regtest7")
self.assertIsInstance(cm.exception.__cause__, ModuleExitedError)
def test_db_s3_validate_spec(self):
mv = create_or_replace_from_spec(
{
"id_name": "regtest8",
"name": "regtest8 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr(
"regtest8.yaml",
json.dumps({
**mv.spec, "parameters": "not an Array"
}).encode("utf-8"),
)
zf.writestr("regtest8.py",
b"def render(table, params):\n return table")
s3.put_bytes(
s3.ExternalModulesBucket,
"regtest8/regtest8.b1c2d3.zip",
bytes(bio.getbuffer()),
)
with self.assertRaises(RuntimeError) as cm:
MODULE_REGISTRY.latest("regtest8")
self.assertIsInstance(cm.exception.__cause__, ValueError)
def test_db_minio_syntax_error_is_runtime_error(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest9",
"name": "regtest9 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr(
"regtest9.yaml",
json.dumps({
**mv.spec, "parameters": "not an Array"
}).encode("utf-8"),
)
zf.writestr("regtest9.py", b"def render(")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest9/regtest9.b1c2d3.zip",
bytes(bio.getbuffer()),
)
with self.assertRaises(RuntimeError) as cm:
MODULE_REGISTRY.latest("regtest9")
self.assertIsInstance(cm.exception.__cause__, SyntaxError)
def test_db_minio_use_cache_for_same_version(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest4",
"name": "regtest4 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest4.yaml", json.dumps(mv.spec).encode("utf-8"))
zf.writestr("regtest4.py",
b"def render(table, params):\n return table")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest4/regtest4.b1c2d2.zip",
bytes(bio.getbuffer()),
)
zf1 = MODULE_REGISTRY.latest("regtest4")
zf2 = MODULE_REGISTRY.latest("regtest4")
self.assertIs(zf2, zf1)
def test_db_minio_refresh_cache_for_new_version(self):
v1 = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest5",
"name": "regtest5 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest5.yaml", json.dumps(v1.spec).encode("utf-8"))
zf.writestr("regtest5.py",
b"def render(table, params):\n return table")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest5/regtest5.b1c2d2.zip",
bytes(bio.getbuffer()),
)
zipfile1 = MODULE_REGISTRY.latest("regtest5")
ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest5",
"name": "regtest5 v2",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest5/regtest5.b1c2d3.zip",
bytes(bio.getbuffer()), # reuse zipfile to save lines of code
)
zipfile2 = MODULE_REGISTRY.latest("regtest5")
self.assertIsNot(zipfile2, zipfile1)
self.assertEqual(zipfile2.version, "b1c2d3")
def build_init_state():
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise Http404("Module is not embeddable")
try:
has_html = module_zipfile.get_optional_html() is not None
except (UnicodeDecodeError, FileNotFoundError, zipfile.BadZipFile):
has_html = False
if not has_html:
raise Http404("Module has no HTML")
ctx = JsonizeContext(
user=AnonymousUser(),
user_profile=None,
session=None,
locale_id=request.locale_id,
module_zipfiles={module_zipfile.module_id: module_zipfile},
)
return {
"workflow":
jsonize_clientside_workflow(
workflow.to_clientside(
include_tab_slugs=False,
include_block_slugs=False,
include_acl=False,
),
ctx,
is_init=True,
),
"step":
jsonize_clientside_step(step.to_clientside(), ctx),
}
def _load_step_and_service(workflow: Workflow, step_id: int,
param: str) -> Tuple[Step, oauth.OAuthService]:
"""Load Step and OAuthService from the database, or raise.
Raise Step.DoesNotExist if the Step is deleted or missing.
Raise SecretDoesNotExist if the Step does not have the given param.
Invoke this within a Workflow.cooperative_lock().
"""
# raises Step.DoesNotExist
step = Step.live_in_workflow(workflow).get(pk=step_id)
# raises KeyError, RuntimeError
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise SecretDoesNotExist(
f"Module {step.module_id_name} does not exist")
module_spec = module_zipfile.get_spec()
for field in module_spec.param_fields:
if (isinstance(field, ParamField.Secret) and field.id_name == param and
(isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a) or
isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2))):
service_name = field.secret_logic.service
service = oauth.OAuthService.lookup_or_none(service_name)
if service is None:
raise OauthServiceNotConfigured(
f'OAuth not configured for "{service_name}" service')
return step, service
else:
raise SecretDoesNotExist(
f"Param {param} does not point to an OAuth secret")
def test_db_minio_latest_load_deprecated_simple(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest2",
"name": "regtest2 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest2/b1c2d2/regtest2.py",
"def render(table, params):\n return table",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest2/b1c2d2/regtest2.yaml",
json.dumps(mv.spec).encode("utf-8"),
)
zf = MODULE_REGISTRY.latest("regtest2")
self.assertEqual(zf.get_spec(), ModuleSpec(**mv.spec))
self.assertIsNone(zf.get_optional_html())
def test_db_minio_latest_load_deprecated_html(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest3",
"name": "regtest3 v2",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest3/b1c2d2/regtest3.py",
"def render(table, params):\n return table",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest3/b1c2d2/regtest3.yaml",
json.dumps(mv.spec).encode("utf-8"),
)
html = "<!DOCTYPE html><html><head><title>Hi</title></head><body>Hello, world!</body></html>"
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest3/b1c2d2/regtest3.html",
html.encode("utf-8"),
)
zf = MODULE_REGISTRY.latest("regtest3")
self.assertEqual(zf.get_optional_html(), html)
def wfmodule_output(request: HttpRequest, wf_module: WfModule, format=None):
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
html = module_zipfile.get_optional_html()
except KeyError:
html = None
return HttpResponse(content=html)
def _lookup_service(step: Step, param: str) -> oauth.OAuthService:
"""Find the OAuthService that manages `param` on `step`.
Raise `HandlerError` if we cannot.
"""
module_id = step.module_id_name
try:
module_zipfile = MODULE_REGISTRY.latest(module_id)
except KeyError:
raise HandlerError(f"BadRequest: module {module_id} not found")
module_spec = module_zipfile.get_spec()
for field in module_spec.param_fields:
if (
field.id_name == param
and isinstance(field, ParamField.Secret)
and (
isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a)
or isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2)
)
):
service_name = field.secret_logic.service
service = oauth.OAuthService.lookup_or_none(service_name)
if not service:
allowed_services = ", ".join(settings.OAUTH_SERVICES.keys())
raise HandlerError(f"AuthError: we only support {allowed_services}")
return service
else:
raise HandlerError(f"Module {module_id} has no oauth {param} parameter")
def load_database_objects(workflow_id: int,
wf_module_id: int) -> DatabaseObjects:
"""
Query WfModule info.
Raise `WfModule.DoesNotExist` or `Workflow.DoesNotExist` if the step was
deleted.
Catch a `ModuleError` from migrate_params() and return it as part of the
`DatabaseObjects`.
"""
with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock:
# raise WfModule.DoesNotExist
wf_module = WfModule.live_in_workflow(
workflow_lock.workflow).get(id=wf_module_id)
# module_zipfile
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
module_zipfile = None
# migrated_params_or_error
if module_zipfile is None:
migrated_params_or_error = {}
else:
try:
migrated_params_or_error = cjwstate.params.get_migrated_params(
wf_module,
module_zipfile=module_zipfile) # raise ModuleError
except ModuleError as err:
migrated_params_or_error = err
# stored_object
try:
stored_object = wf_module.stored_objects.get(
stored_at=wf_module.stored_data_version)
except StoredObject.DoesNotExist:
stored_object = None
# input_crr
try:
# raise WfModule.DoesNotExist -- but we'll catch this one
prev_module = wf_module.tab.live_wf_modules.get(
order=wf_module.order - 1)
input_crr = prev_module.cached_render_result # may be None
except WfModule.DoesNotExist:
input_crr = None
return DatabaseObjects(
wf_module,
module_zipfile,
migrated_params_or_error,
stored_object,
input_crr,
)
def get_migrated_params(
wf_module: WfModule,
*,
module_zipfile: ModuleZipfile = None) -> Dict[str, Any]:
"""
Read `wf_module.params`, calling migrate_params() or using cache fields.
Call this within a `Workflow.cooperative_lock()`.
If migrate_params() was already called for this version of the module,
return the cached value. See `wf_module.cached_migrated_params`,
`wf_module.cached_migrated_params_module_version`.
Raise `ModuleError` if migration fails.
Raise `KeyError` if the module was deleted.
Raise `RuntimeError` (unrecoverable) if there is a problem loading or
executing the module. (Modules are validated before import, so this should
not happen.)
The result may be invalid. Call `validate()` to raise a `ValueError` to
detect that case.
TODO avoid holding the database lock whilst executing stuff on the kernel.
(This will involve auditing and modifying all callers to handle new error
cases.)
"""
if module_zipfile is None:
# raise KeyError
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
stale = (
module_zipfile.version == "develop"
# works if cached version (and thus cached _result_) is None
or (module_zipfile.get_param_schema_version() !=
wf_module.cached_migrated_params_module_version))
if not stale:
return wf_module.cached_migrated_params
else:
# raise ModuleError
params = invoke_migrate_params(module_zipfile, wf_module.params)
wf_module.cached_migrated_params = params
wf_module.cached_migrated_params_module_version = (
module_zipfile.get_param_schema_version())
try:
wf_module.save(update_fields=[
"cached_migrated_params",
"cached_migrated_params_module_version",
])
except ValueError:
# WfModule was deleted, so we get:
# "ValueError: Cannot force an update in save() with no primary key."
pass
return params
def to_clientside(
self, *, force_module_zipfile: Optional[ModuleZipfile] = None
) -> clientside.StepUpdate:
# module_zipfile, for params
if force_module_zipfile:
module_zipfile = force_module_zipfile
else:
from cjwstate.models.module_registry import MODULE_REGISTRY
try:
module_zipfile = MODULE_REGISTRY.latest(self.module_id_name)
except KeyError:
module_zipfile = None
if module_zipfile is None:
params = {}
else:
from cjwstate.params import get_migrated_params
module_spec = module_zipfile.get_spec()
param_schema = module_spec.param_schema
# raise ModuleError
params = get_migrated_params(self, module_zipfile=module_zipfile)
try:
param_schema.validate(params)
except ValueError:
logger.exception(
"%s.migrate_params() gave invalid output: %r",
self.module_id_name,
params,
)
params = param_schema.default
crr = self._build_cached_render_result_fresh_or_not()
if crr is None:
crr = clientside.Null
return clientside.StepUpdate(
id=self.id,
slug=self.slug,
module_slug=self.module_id_name,
tab_slug=self.tab_slug,
is_busy=self.is_busy,
render_result=crr,
files=self._get_clientside_files(module_zipfile),
params=params,
secrets=self.secret_metadata,
is_collapsed=self.is_collapsed,
notes=self.notes,
is_auto_fetch=self.auto_update_data,
fetch_interval=self.update_interval,
last_fetched_at=self.last_update_check,
is_notify_on_change=self.notifications,
last_relevant_delta_id=self.last_relevant_delta_id,
versions=self._get_clientside_fetched_version_list(module_zipfile),
)
def wrapper(request: HttpRequest, workflow_id: int, wf_module_slug: str,
*args, **kwargs):
auth_header = request.headers.get("Authorization", "")
auth_header_match = AuthTokenHeaderRegex.match(auth_header)
if not auth_header_match:
return ErrorResponse(403,
"authorization-bearer-token-not-provided")
bearer_token = auth_header_match.group(1)
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
try:
wf_module = WfModule.live_in_workflow(workflow).get(
slug=wf_module_slug)
except WfModule.DoesNotExist:
return ErrorResponse(404, "step-not-found")
try:
module_zipfile = MODULE_REGISTRY.latest(
wf_module.module_id_name)
except KeyError:
return ErrorResponse(400, "step-module-deleted")
try:
file_param_id_name = next(
iter(pf.id_name
for pf in module_zipfile.get_spec().param_fields
if pf.type == "file"))
except StopIteration:
return ErrorResponse(400, "step-has-no-file-param")
api_token = wf_module.file_upload_api_token
if not api_token:
return ErrorResponse(403, "step-has-no-api-token")
bearer_token_hash = hashlib.sha256(
bearer_token.encode("utf-8")).digest()
api_token_hash = hashlib.sha256(
api_token.encode("utf-8")).digest()
if bearer_token_hash != api_token_hash or bearer_token != api_token:
return ErrorResponse(403,
"authorization-bearer-token-invalid")
return f(
request,
workflow_lock,
wf_module,
file_param_id_name,
*args,
**kwargs,
)
except Workflow.DoesNotExist:
return ErrorResponse(404, "workflow-not-found")
def create_module_zipfile(
module_id: str = "testmodule",
*,
version: Optional[str] = None,
spec_kwargs: Dict[str, Any] = {},
python_code: str = "",
html: Optional[str] = None,
js_module: str = "",
extra_file_contents: Dict[str, bytes] = {},
) -> ModuleZipfile:
"""
Create a ModuleZipfile, stored in the database and s3.
If `version` is not supplied, generate one using the sha1 of the zipfile.
This is usually what you want: s3 reads on overwrites are _eventually_
consistent, so if you 1. write a file; 2. overwrite it; and 3. read it, the
read might result in the file from step 1 or the file from step 2. A sha1
version means overwrites will never modify data, solving the problem.
"""
spec = {
"id_name": module_id,
"name": "Test Module",
"category": "Clean",
"parameters": [],
**spec_kwargs,
}
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr(module_id + ".yaml", json.dumps(spec))
zf.writestr(module_id + ".py", python_code.encode("utf-8"))
if html is not None:
zf.writestr(module_id + ".html", html.encode("utf-8"))
if js_module:
zf.writestr(module_id + ".js", js_module.encode("utf-8"))
for path, content in extra_file_contents.items():
zf.writestr(path, content)
data = bytes(bio.getbuffer())
if version is None:
sha1 = hashlib.sha1()
sha1.update(data)
version = sha1.hexdigest()
s3.put_bytes(
s3.ExternalModulesBucket,
"%s/%s.%s.zip" % (module_id, module_id, version),
data,
)
ModuleVersion.objects.create(id_name=module_id,
source_version_hash=version,
spec=spec,
js_module=js_module)
return MODULE_REGISTRY.latest(module_id)
def amend_create_kwargs(self, *, step, new_values, **kwargs):
"""Prepare values_for_backward|forward["params"].
Raise ValueError if `values_for_forward["params"]` won't be valid
according to the module spec.
"""
if _step_is_deleted(step): # refreshes from DB
return None
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise ValueError("Module %s does not exist" % step.module_id_name)
# Old values: store exactly what we had
old_values = step.params
module_spec = module_zipfile.get_spec()
# [2021-07-05, adamhooper] nested here to prevent circular import error
# in fetcher. Didn't bother to find the circle.
from cjwstate.params import invoke_migrate_params
# New values: store _migrated_ old_values, with new_values applied on
# top
migrated_old_values = invoke_migrate_params(module_zipfile, old_values)
# Ensure migrate_params() didn't generate buggy _old_ values before we
# add _new_ values. This sanity check may protect users' params by
# raising an error early. It's also a way to catch bugs in unit tests.
# (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params
# returns `{}` -- which is often invalid -- and then the `**new_values`
# below overwrites the invalid data. So without this validate(), a unit
# test with an invalid migrate_params() may pass, which is wrong.)
#
# If you're seeing this because your unit test failed, try this:
# self.kernel.migrate_params.side_effect = lambda m, p: p
module_spec.param_schema.validate(
migrated_old_values) # raises ValueError
new_values = {**migrated_old_values, **new_values}
module_spec.param_schema.validate(new_values) # raises ValueError
return {
**kwargs,
"step": step,
"values_for_backward": {
"params": old_values
},
"values_for_forward": {
"params": new_values
},
"step_delta_ids": self.affected_step_delta_ids(step),
}
def _wf_module_set_secret_and_build_delta(
workflow: Workflow, wf_module: WfModule, param: str,
secret: str) -> Optional[clientside.Update]:
"""
Write a new secret to `wf_module`, or raise.
Return a `clientside.Update`, or `None` if the database is not modified.
Raise Workflow.DoesNotExist if the Workflow was deleted.
"""
with workflow.cooperative_lock(): # raises Workflow.DoesNotExist
try:
wf_module.refresh_from_db()
except WfModule.DoesNotExist:
return None # no-op
if wf_module.secrets.get(param, {}).get("secret") == secret:
return None # no-op
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
raise HandlerError(
f"BadRequest: ModuleZipfile {wf_module.module_id_name} does not exist"
)
module_spec = module_zipfile.get_spec()
if not any(p.type == "secret" and p.secret_logic.provider == "string"
for p in module_spec.param_fields):
raise HandlerError(
f"BadRequest: param is not a secret string parameter")
created_at = timezone.now()
created_at_str = (
created_at.strftime("%Y-%m-%dT%H:%M:%S") + "." +
created_at.strftime("%f")[0:3] # milliseconds
+ "Z")
wf_module.secrets = {
**wf_module.secrets,
param: {
"name": created_at_str,
"secret": secret
},
}
wf_module.save(update_fields=["secrets"])
return clientside.Update(steps={
wf_module.id:
clientside.StepUpdate(secrets=wf_module.secret_metadata)
})
def amend_create_kwargs(
self, *, workflow, tab, slug, module_id_name, position, param_values, **kwargs
):
"""Add a step to the tab.
Raise KeyError if `module_id_name` is invalid.
Raise RuntimeError (unrecoverable) if s3 holds invalid module data.
Raise ValueError if `param_values` do not match the module's spec.
"""
from ..step import Step
# ensure slug is unique, or raise ValueError
if Step.objects.filter(tab__workflow_id=workflow.id, slug=slug).count() > 0:
raise ValueError("slug is not unique. Please pass a unique slug.")
# raise KeyError, RuntimeError
module_zipfile = MODULE_REGISTRY.latest(module_id_name)
module_spec = module_zipfile.get_spec()
# Set _all_ params (not just the user-specified ones). Our
# dropdown-menu actions only specify the relevant params and expect us
# to set the others to defaults.
params = {**module_spec.default_params, **param_values}
module_spec.get_param_schema().validate(params) # raises ValueError
# step starts off "deleted" and gets un-deleted in forward().
step = tab.steps.create(
module_id_name=module_id_name,
order=position,
slug=slug,
is_deleted=True,
params=params,
cached_migrated_params=params,
cached_migrated_params_module_version=module_zipfile.version,
secrets={},
)
return {
**kwargs,
"workflow": workflow,
"step": step,
"step_delta_ids": self.affected_step_delta_ids(step),
}
def _add_wf_module_to_tab(step_dict, order, tab, delta_id, lesson):
"""
Deserialize a WfModule from lesson initial_workflow.
Raise `KeyError` if a module ID is invalid.
"""
id_name = step_dict["module"]
slug = step_dict["slug"]
# 500 error if bad module id name
module_zipfile = MODULE_REGISTRY.latest(
id_name) # raise KeyError, RuntimeError
module_spec = module_zipfile.get_spec()
# All params not set in json get default values
# Also, we must have a dict with all param values set or we can't migrate
# params later
params = {**module_spec.default_params, **step_dict["params"]}
# Rewrite 'url' params: if the spec has them as relative, make them the
# absolute path -- relative to the lesson URL.
if "url" in params:
if params["url"].startswith("./"):
params["url"] = "".join([
settings.STATIC_URL,
("lessons/" if lesson.course is None else "courses/"),
f"{lesson.locale_id}/",
(lesson.slug if lesson.course is None else
f"{lesson.course.slug}/{lesson.slug}"),
params["url"][1:], # include the '/'
])
# 500 error if params are invalid
# TODO testme
module_spec.get_param_schema().validate(params) # raises ValueError
return tab.wf_modules.create(
order=order,
slug=slug,
module_id_name=id_name,
is_busy=module_spec.loads_data, # assume we'll send a fetch
last_relevant_delta_id=delta_id,
params=params,
is_collapsed=step_dict.get("collapsed", False),
notes=step_dict.get("note", None),
)
def amend_create_kwargs(cls, *, wf_module, new_values, **kwargs):
"""
Prepare `old_values` and `new_values`.
Raise ValueError if `new_values` won't be valid according to the module
spec.
"""
if cls.wf_module_is_deleted(wf_module): # refreshes from DB
return None
try:
module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name)
except KeyError:
raise ValueError("Module %s does not exist" % wf_module.module_id_name)
# Old values: store exactly what we had
old_values = wf_module.params
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
# New values: store _migrated_ old_values, with new_values applied on
# top
migrated_old_values = invoke_migrate_params(module_zipfile, old_values)
# Ensure migrate_params() didn't generate buggy _old_ values before we
# add _new_ values. This sanity check may protect users' params by
# raising an error early. It's also a way to catch bugs in unit tests.
# (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params
# returns `{}` -- which is often invalid -- and then the `**new_values`
# below overwrites the invalid data. So without this validate(), a unit
# test with an invalid migrate_params() may pass, which is wrong.)
#
# If you're seeing this because your unit test failed, try this:
# self.kernel.migrate_params.side_effect = lambda m, p: p
param_schema.validate(migrated_old_values) # raises ValueError
new_values = {**migrated_old_values, **new_values}
param_schema.validate(new_values) # raises ValueError
return {
**kwargs,
"wf_module": wf_module,
"new_values": new_values,
"old_values": old_values,
"wf_module_delta_ids": cls.affected_wf_module_delta_ids(wf_module),
}
def test_db_minio_latest_order_by_last_update_time(self):
# old version
ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest1",
"name": "regtest1 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
time.sleep(0.000002) # guarantee new timestamp
# new version
v2 = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest1",
"name": "regtest1 v2",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest1.yaml", json.dumps(v2.spec).encode("utf-8"))
zf.writestr("regtest1.py",
b"def render(table, params):\n return table")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest1/regtest1.b1c2d2.zip",
bytes(bio.getbuffer()),
)
zf = MODULE_REGISTRY.latest("regtest1")
self.assertEqual(zf.get_spec(), ModuleSpec(**v2.spec))
def embed(request, wfmodule_id):
try:
wf_module = WfModule.objects.get(pk=wfmodule_id, is_deleted=False)
except WfModule.DoesNotExist:
wf_module = None
if wf_module:
if not wf_module.workflow:
wf_module = None
elif not wf_module.workflow.request_authorized_read(request):
wf_module = None
else:
try:
module_zipfile = MODULE_REGISTRY.latest(
wf_module.module_id_name)
if not module_zipfile.get_spec().html_output:
wf_module = None
except KeyError:
wf_module = None
if wf_module:
ctx = JsonizeContext(
request.user,
request.session,
request.locale_id,
{module_zipfile.module_id: module_zipfile},
)
init_state = {
"workflow":
jsonize_clientside_workflow(
wf_module.workflow.to_clientside(include_tab_slugs=False),
ctx,
is_init=True,
),
"wf_module":
jsonize_clientside_step(wf_module.to_clientside(), ctx),
}
else:
init_state = {"workflow": None, "wf_module": None}
return TemplateResponse(request, "embed.html", {"initState": init_state})
def locked_and_loaded_step(
workflow_id: int, step_slug: str
) -> ContextManager[Tuple[DbObjectCooperativeLock, Step, str]]:
"""Yield `WorkflowLock`, `step` and `file_param_id_name`.
SECURITY: the caller may want to test the Step's `file_upload_api_token`.
Raise UploadError(404, "workflow-not-found") on missing/deleted Workflow.
Raise UploadError(404, "step-not-found") on missing/deleted Step.
Raise UploadError(400, "step-module-deleted") on code-less Step.
Raise UploadError(400, "step-has-no-file-param") on a Step with no File param.
"""
try:
with Workflow.lookup_and_cooperative_lock(
id=workflow_id) as workflow_lock:
workflow = workflow_lock.workflow
try:
step = Step.live_in_workflow(workflow).get(slug=step_slug)
except Step.DoesNotExist:
raise UploadError(404, "step-not-found")
try:
module_zipfile = MODULE_REGISTRY.latest(step.module_id_name)
except KeyError:
raise UploadError(400, "step-module-deleted")
try:
file_param_id_name = next(
iter(pf.id_name
for pf in module_zipfile.get_spec().param_fields
if pf.type == "file"))
except StopIteration:
raise UploadError(400, "step-has-no-file-param")
yield workflow_lock, step, file_param_id_name
except Workflow.DoesNotExist:
raise UploadError(404, "workflow-not-found")
def import_module_from_url(url: str) -> clientside.Module:
"""Import zipfile from a URL.
Return a `ModuleZipFile` on success.
Raise `WorkbenchModuleImportError` if import fails, meaning:
* The URL is not a URL we handle
* There's an HTTP error
* The ModuleZipfile is invalid
"""
match = GITHUB_URL_PATTERN.match(url)
if match:
clientside_module = import_module_from_github(
match.group("owner"), match.group("repo")
)
elif TEST_ZIP_URL_PATTERN.match(url):
clientside_module = import_module_from_test_zip_url(url)
else:
raise WorkbenchModuleImportError(
"Please supply a GitHub URL with owner=CJWorkbench"
)
return clientside_module, MODULE_REGISTRY.latest(clientside_module.spec.id_name)
def to_clientside(self) -> clientside.StepUpdate:
# params
from cjwstate.models.module_registry import MODULE_REGISTRY
try:
module_zipfile = MODULE_REGISTRY.latest(self.module_id_name)
except KeyError:
module_zipfile = None
if module_zipfile is None:
params = {}
else:
from cjwstate.params import get_migrated_params
module_spec = module_zipfile.get_spec()
param_schema = module_spec.get_param_schema()
# raise ModuleError
params = get_migrated_params(self, module_zipfile=module_zipfile)
try:
param_schema.validate(params)
except ValueError:
logger.exception(
"%s.migrate_params() gave invalid output: %r",
self.module_id_name,
params,
)
params = param_schema.coerce(params)
crr = self._build_cached_render_result_fresh_or_not()
if crr is None:
crr = clientside.Null
return clientside.StepUpdate(
id=self.id,
slug=self.slug,
module_slug=self.module_id_name,
tab_slug=self.tab_slug,
is_busy=self.is_busy,
render_result=crr,
files=[
clientside.UploadedFile(
name=name, uuid=uuid, size=size, created_at=created_at
)
for name, uuid, size, created_at in self.uploaded_files.order_by(
"-created_at"
).values_list("name", "uuid", "size", "created_at")
],
params=params,
secrets=self.secret_metadata,
is_collapsed=self.is_collapsed,
notes=self.notes,
is_auto_fetch=self.auto_update_data,
fetch_interval=self.update_interval,
last_fetched_at=self.last_update_check,
is_notify_on_change=self.notifications,
has_unseen_notification=self.has_unseen_notification,
last_relevant_delta_id=self.last_relevant_delta_id,
versions=clientside.FetchedVersionList(
versions=[
clientside.FetchedVersion(created_at=created_at, is_seen=is_seen)
for created_at, is_seen in self.stored_objects.order_by(
"-stored_at"
).values_list("stored_at", "read")
],
selected=self.stored_data_version,
),
)
def _load_workflow_and_step_sync(
request: HttpRequest,
workflow_id_or_secret_id: Union[int, str],
step_slug: str,
accessing: Literal["all", "chart", "table"],
) -> Tuple[Workflow, Step]:
"""Load (Workflow, Step) from database, or raise Http404 or PermissionDenied.
`Step.tab` will be loaded. (`Step.tab.workflow_id` is needed to access the render
cache.)
To avoid PermissionDenied:
* The workflow must be public; OR
* The user must be workflow owner, editor or viewer; OR
* The user must be workflow report-viewer and the step must be a chart or
table in the report.
"""
try:
if isinstance(workflow_id_or_secret_id, int):
search = {"id": workflow_id_or_secret_id}
has_secret = False
else:
search = {"secret_id": workflow_id_or_secret_id}
has_secret = True
with Workflow.lookup_and_cooperative_lock(**search) as workflow_lock:
workflow = workflow_lock.workflow
if (has_secret or workflow.public
or workflow.request_authorized_owner(request)):
need_report_auth = False
elif request.user is None or request.user.is_anonymous:
raise PermissionDenied()
else:
try:
acl_entry = workflow.acl.filter(
email=request.user.email).get()
except AclEntry.DoesNotExist:
raise PermissionDenied()
if acl_entry.role in {Role.VIEWER, Role.EDITOR}:
need_report_auth = False
elif acl_entry.role == Role.REPORT_VIEWER:
need_report_auth = True
else:
raise PermissionDenied() # role we don't handle yet
step = (Step.live_in_workflow(
workflow.id).select_related("tab").get(slug=step_slug)
) # or Step.DoesNotExist
if need_report_auth: # user is report-viewer
if workflow.has_custom_report:
if (accessing == "chart" and
workflow.blocks.filter(step_id=step.id).exists()):
pass # the step is a chart
elif (accessing == "table" and
workflow.blocks.filter(tab_id=step.tab_id).exists()
and not step.tab.live_steps.filter(
order__gt=step.order)):
pass # step is a table (last step of a report-included tab)
else:
raise PermissionDenied()
else:
# Auto-report: all Charts are allowed; everything else is not
try:
if accessing == "chart" and (MODULE_REGISTRY.latest(
step.module_id_name).get_spec().html_output):
pass
else:
raise PermissionDenied()
except KeyError: # not a module
raise PermissionDenied()
return workflow, step
except (Workflow.DoesNotExist, Step.DoesNotExist):
raise Http404()
def _load_module_zipfile(module_id_name: str) -> ModuleZipfile:
"""Return a ModuleZipfile or raise HandlerError."""
try:
return MODULE_REGISTRY.latest(module_id_name)
except KeyError:
raise HandlerError("KeyError: ModuleVersion not found")
def test_latest_internal(self):
zf = MODULE_REGISTRY.latest("pythoncode")
self.assertEqual(zf.get_spec().id_name, "pythoncode")