def test_db_minio_validate_code_with_kernel(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest7", "name": "regtest7 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest7.yaml", json.dumps(mv.spec).encode("utf-8")) zf.writestr( "regtest7.py", b"def render(table, params):\n return table\nfoo()") minio.put_bytes( minio.ExternalModulesBucket, "regtest7/regtest7.b1c2d3.zip", bytes(bio.getbuffer()), ) with self.assertRaises(RuntimeError) as cm: MODULE_REGISTRY.latest("regtest7") self.assertIsInstance(cm.exception.__cause__, ModuleExitedError)
def test_db_minio_use_cache_for_same_version(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest4", "name": "regtest4 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest4.yaml", json.dumps(mv.spec).encode("utf-8")) zf.writestr("regtest4.py", b"def render(table, params):\n return table") minio.put_bytes( minio.ExternalModulesBucket, "regtest4/regtest4.b1c2d2.zip", bytes(bio.getbuffer()), ) zf1 = MODULE_REGISTRY.latest("regtest4") zf2 = MODULE_REGISTRY.latest("regtest4") self.assertIs(zf2, zf1)
def test_db_s3_validate_spec(self): mv = create_or_replace_from_spec( { "id_name": "regtest8", "name": "regtest8 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr( "regtest8.yaml", json.dumps({ **mv.spec, "parameters": "not an Array" }).encode("utf-8"), ) zf.writestr("regtest8.py", b"def render(table, params):\n return table") s3.put_bytes( s3.ExternalModulesBucket, "regtest8/regtest8.b1c2d3.zip", bytes(bio.getbuffer()), ) with self.assertRaises(RuntimeError) as cm: MODULE_REGISTRY.latest("regtest8") self.assertIsInstance(cm.exception.__cause__, ValueError)
def test_db_minio_syntax_error_is_runtime_error(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest9", "name": "regtest9 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr( "regtest9.yaml", json.dumps({ **mv.spec, "parameters": "not an Array" }).encode("utf-8"), ) zf.writestr("regtest9.py", b"def render(") minio.put_bytes( minio.ExternalModulesBucket, "regtest9/regtest9.b1c2d3.zip", bytes(bio.getbuffer()), ) with self.assertRaises(RuntimeError) as cm: MODULE_REGISTRY.latest("regtest9") self.assertIsInstance(cm.exception.__cause__, SyntaxError)
def test_db_minio_refresh_cache_for_new_version(self): v1 = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest5", "name": "regtest5 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest5.yaml", json.dumps(v1.spec).encode("utf-8")) zf.writestr("regtest5.py", b"def render(table, params):\n return table") minio.put_bytes( minio.ExternalModulesBucket, "regtest5/regtest5.b1c2d2.zip", bytes(bio.getbuffer()), ) zipfile1 = MODULE_REGISTRY.latest("regtest5") ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest5", "name": "regtest5 v2", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest5/regtest5.b1c2d3.zip", bytes(bio.getbuffer()), # reuse zipfile to save lines of code ) zipfile2 = MODULE_REGISTRY.latest("regtest5") self.assertIsNot(zipfile2, zipfile1) self.assertEqual(zipfile2.version, "b1c2d3")
def _lookup_service(step: Step, param: str) -> oauth.OAuthService: """Find the OAuthService that manages `param` on `step`. Raise `HandlerError` if we cannot. """ module_id = step.module_id_name try: module_zipfile = MODULE_REGISTRY.latest(module_id) except KeyError: raise HandlerError(f"BadRequest: module {module_id} not found") module_spec = module_zipfile.get_spec() for field in module_spec.param_fields: if ( field.id_name == param and isinstance(field, ParamField.Secret) and ( isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a) or isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2) ) ): service_name = field.secret_logic.service service = oauth.OAuthService.lookup_or_none(service_name) if not service: allowed_services = ", ".join(settings.OAUTH_SERVICES.keys()) raise HandlerError(f"AuthError: we only support {allowed_services}") return service else: raise HandlerError(f"Module {module_id} has no oauth {param} parameter")
def _load_step_and_service(workflow: Workflow, step_id: int, param: str) -> Tuple[Step, oauth.OAuthService]: """Load Step and OAuthService from the database, or raise. Raise Step.DoesNotExist if the Step is deleted or missing. Raise SecretDoesNotExist if the Step does not have the given param. Invoke this within a Workflow.cooperative_lock(). """ # raises Step.DoesNotExist step = Step.live_in_workflow(workflow).get(pk=step_id) # raises KeyError, RuntimeError try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise SecretDoesNotExist( f"Module {step.module_id_name} does not exist") module_spec = module_zipfile.get_spec() for field in module_spec.param_fields: if (isinstance(field, ParamField.Secret) and field.id_name == param and (isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a) or isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2))): service_name = field.secret_logic.service service = oauth.OAuthService.lookup_or_none(service_name) if service is None: raise OauthServiceNotConfigured( f'OAuth not configured for "{service_name}" service') return step, service else: raise SecretDoesNotExist( f"Param {param} does not point to an OAuth secret")
def test_db_minio_latest_load_deprecated_simple(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest2", "name": "regtest2 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest2/b1c2d2/regtest2.py", "def render(table, params):\n return table", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest2/b1c2d2/regtest2.yaml", json.dumps(mv.spec).encode("utf-8"), ) zf = MODULE_REGISTRY.latest("regtest2") self.assertEqual(zf.get_spec(), ModuleSpec(**mv.spec)) self.assertIsNone(zf.get_optional_html())
def build_init_state(): try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise Http404("Module is not embeddable") try: has_html = module_zipfile.get_optional_html() is not None except (UnicodeDecodeError, FileNotFoundError, zipfile.BadZipFile): has_html = False if not has_html: raise Http404("Module has no HTML") ctx = JsonizeContext( user=AnonymousUser(), user_profile=None, session=None, locale_id=request.locale_id, module_zipfiles={module_zipfile.module_id: module_zipfile}, ) return { "workflow": jsonize_clientside_workflow( workflow.to_clientside( include_tab_slugs=False, include_block_slugs=False, include_acl=False, ), ctx, is_init=True, ), "step": jsonize_clientside_step(step.to_clientside(), ctx), }
def wfmodule_output(request: HttpRequest, wf_module: WfModule, format=None): try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) html = module_zipfile.get_optional_html() except KeyError: html = None return HttpResponse(content=html)
def test_db_minio_latest_load_deprecated_html(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest3", "name": "regtest3 v2", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest3/b1c2d2/regtest3.py", "def render(table, params):\n return table", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest3/b1c2d2/regtest3.yaml", json.dumps(mv.spec).encode("utf-8"), ) html = "<!DOCTYPE html><html><head><title>Hi</title></head><body>Hello, world!</body></html>" minio.put_bytes( minio.ExternalModulesBucket, "regtest3/b1c2d2/regtest3.html", html.encode("utf-8"), ) zf = MODULE_REGISTRY.latest("regtest3") self.assertEqual(zf.get_optional_html(), html)
def visible_modules(request) -> Dict[str, ModuleZipfile]: """Load all ModuleZipfiles the user may use.""" ret = dict(MODULE_REGISTRY.all_latest()) # shallow copy if not request.user.is_authenticated and "pythoncode" in ret: del ret["pythoncode"] return ret
def load_database_objects(workflow_id: int, wf_module_id: int) -> DatabaseObjects: """ Query WfModule info. Raise `WfModule.DoesNotExist` or `Workflow.DoesNotExist` if the step was deleted. Catch a `ModuleError` from migrate_params() and return it as part of the `DatabaseObjects`. """ with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock: # raise WfModule.DoesNotExist wf_module = WfModule.live_in_workflow( workflow_lock.workflow).get(id=wf_module_id) # module_zipfile try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) except KeyError: module_zipfile = None # migrated_params_or_error if module_zipfile is None: migrated_params_or_error = {} else: try: migrated_params_or_error = cjwstate.params.get_migrated_params( wf_module, module_zipfile=module_zipfile) # raise ModuleError except ModuleError as err: migrated_params_or_error = err # stored_object try: stored_object = wf_module.stored_objects.get( stored_at=wf_module.stored_data_version) except StoredObject.DoesNotExist: stored_object = None # input_crr try: # raise WfModule.DoesNotExist -- but we'll catch this one prev_module = wf_module.tab.live_wf_modules.get( order=wf_module.order - 1) input_crr = prev_module.cached_render_result # may be None except WfModule.DoesNotExist: input_crr = None return DatabaseObjects( wf_module, module_zipfile, migrated_params_or_error, stored_object, input_crr, )
def get_migrated_params( wf_module: WfModule, *, module_zipfile: ModuleZipfile = None) -> Dict[str, Any]: """ Read `wf_module.params`, calling migrate_params() or using cache fields. Call this within a `Workflow.cooperative_lock()`. If migrate_params() was already called for this version of the module, return the cached value. See `wf_module.cached_migrated_params`, `wf_module.cached_migrated_params_module_version`. Raise `ModuleError` if migration fails. Raise `KeyError` if the module was deleted. Raise `RuntimeError` (unrecoverable) if there is a problem loading or executing the module. (Modules are validated before import, so this should not happen.) The result may be invalid. Call `validate()` to raise a `ValueError` to detect that case. TODO avoid holding the database lock whilst executing stuff on the kernel. (This will involve auditing and modifying all callers to handle new error cases.) """ if module_zipfile is None: # raise KeyError module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) stale = ( module_zipfile.version == "develop" # works if cached version (and thus cached _result_) is None or (module_zipfile.get_param_schema_version() != wf_module.cached_migrated_params_module_version)) if not stale: return wf_module.cached_migrated_params else: # raise ModuleError params = invoke_migrate_params(module_zipfile, wf_module.params) wf_module.cached_migrated_params = params wf_module.cached_migrated_params_module_version = ( module_zipfile.get_param_schema_version()) try: wf_module.save(update_fields=[ "cached_migrated_params", "cached_migrated_params_module_version", ]) except ValueError: # WfModule was deleted, so we get: # "ValueError: Cannot force an update in save() with no primary key." pass return params
def to_clientside( self, *, force_module_zipfile: Optional[ModuleZipfile] = None ) -> clientside.StepUpdate: # module_zipfile, for params if force_module_zipfile: module_zipfile = force_module_zipfile else: from cjwstate.models.module_registry import MODULE_REGISTRY try: module_zipfile = MODULE_REGISTRY.latest(self.module_id_name) except KeyError: module_zipfile = None if module_zipfile is None: params = {} else: from cjwstate.params import get_migrated_params module_spec = module_zipfile.get_spec() param_schema = module_spec.param_schema # raise ModuleError params = get_migrated_params(self, module_zipfile=module_zipfile) try: param_schema.validate(params) except ValueError: logger.exception( "%s.migrate_params() gave invalid output: %r", self.module_id_name, params, ) params = param_schema.default crr = self._build_cached_render_result_fresh_or_not() if crr is None: crr = clientside.Null return clientside.StepUpdate( id=self.id, slug=self.slug, module_slug=self.module_id_name, tab_slug=self.tab_slug, is_busy=self.is_busy, render_result=crr, files=self._get_clientside_files(module_zipfile), params=params, secrets=self.secret_metadata, is_collapsed=self.is_collapsed, notes=self.notes, is_auto_fetch=self.auto_update_data, fetch_interval=self.update_interval, last_fetched_at=self.last_update_check, is_notify_on_change=self.notifications, last_relevant_delta_id=self.last_relevant_delta_id, versions=self._get_clientside_fetched_version_list(module_zipfile), )
def wrapper(request: HttpRequest, workflow_id: int, wf_module_slug: str, *args, **kwargs): auth_header = request.headers.get("Authorization", "") auth_header_match = AuthTokenHeaderRegex.match(auth_header) if not auth_header_match: return ErrorResponse(403, "authorization-bearer-token-not-provided") bearer_token = auth_header_match.group(1) try: with Workflow.lookup_and_cooperative_lock( id=workflow_id) as workflow_lock: workflow = workflow_lock.workflow try: wf_module = WfModule.live_in_workflow(workflow).get( slug=wf_module_slug) except WfModule.DoesNotExist: return ErrorResponse(404, "step-not-found") try: module_zipfile = MODULE_REGISTRY.latest( wf_module.module_id_name) except KeyError: return ErrorResponse(400, "step-module-deleted") try: file_param_id_name = next( iter(pf.id_name for pf in module_zipfile.get_spec().param_fields if pf.type == "file")) except StopIteration: return ErrorResponse(400, "step-has-no-file-param") api_token = wf_module.file_upload_api_token if not api_token: return ErrorResponse(403, "step-has-no-api-token") bearer_token_hash = hashlib.sha256( bearer_token.encode("utf-8")).digest() api_token_hash = hashlib.sha256( api_token.encode("utf-8")).digest() if bearer_token_hash != api_token_hash or bearer_token != api_token: return ErrorResponse(403, "authorization-bearer-token-invalid") return f( request, workflow_lock, wf_module, file_param_id_name, *args, **kwargs, ) except Workflow.DoesNotExist: return ErrorResponse(404, "workflow-not-found")
def load_from_workflow(cls, workflow: Workflow) -> DependencyGraph: """Create a DependencyGraph using the database. Must be called within a `workflow.cooperative_lock()`. Missing or deleted modules are deemed to have no dependencies. """ from cjwstate.models.module_registry import MODULE_REGISTRY module_zipfiles = MODULE_REGISTRY.all_latest() tabs = [] steps = {} for tab in workflow.live_tabs: tab_step_ids = [] for step in tab.live_steps: tab_step_ids.append(step.id) try: module_zipfile = module_zipfiles[step.module_id_name] except KeyError: steps[step.id] = cls.Step(set()) continue module_spec = module_zipfile.get_spec() schema = module_spec.get_param_schema() # Optimization: don't migrate_params() if we know there are no # tab params. (get_migrated_params() invokes module code, and # we'd prefer for module code to execute only in the renderer.) if all(((not isinstance(dtype, ParamDType.Tab) and not isinstance(dtype, ParamDType.Multitab)) for dtype, v in schema.iter_dfs_dtype_values( schema.coerce(None)))): # There are no tab params. steps[step.id] = cls.Step(set()) continue from cjwstate.params import get_migrated_params params = get_migrated_params(step) # raises ValueError (and we don't handle that right now) schema.validate(params) tab_slugs = frozenset( v for dtype, v in schema.iter_dfs_dtype_values(params) if isinstance(dtype, ParamDType.Tab)) steps[step.id] = cls.Step(tab_slugs) tabs.append(cls.Tab(tab.slug, tab_step_ids)) return cls(tabs, steps)
def create_module_zipfile( module_id: str = "testmodule", *, version: Optional[str] = None, spec_kwargs: Dict[str, Any] = {}, python_code: str = "", html: Optional[str] = None, js_module: str = "", extra_file_contents: Dict[str, bytes] = {}, ) -> ModuleZipfile: """ Create a ModuleZipfile, stored in the database and s3. If `version` is not supplied, generate one using the sha1 of the zipfile. This is usually what you want: s3 reads on overwrites are _eventually_ consistent, so if you 1. write a file; 2. overwrite it; and 3. read it, the read might result in the file from step 1 or the file from step 2. A sha1 version means overwrites will never modify data, solving the problem. """ spec = { "id_name": module_id, "name": "Test Module", "category": "Clean", "parameters": [], **spec_kwargs, } bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr(module_id + ".yaml", json.dumps(spec)) zf.writestr(module_id + ".py", python_code.encode("utf-8")) if html is not None: zf.writestr(module_id + ".html", html.encode("utf-8")) if js_module: zf.writestr(module_id + ".js", js_module.encode("utf-8")) for path, content in extra_file_contents.items(): zf.writestr(path, content) data = bytes(bio.getbuffer()) if version is None: sha1 = hashlib.sha1() sha1.update(data) version = sha1.hexdigest() s3.put_bytes( s3.ExternalModulesBucket, "%s/%s.%s.zip" % (module_id, module_id, version), data, ) ModuleVersion.objects.create(id_name=module_id, source_version_hash=version, spec=spec, js_module=js_module) return MODULE_REGISTRY.latest(module_id)
def amend_create_kwargs(self, *, step, new_values, **kwargs): """Prepare values_for_backward|forward["params"]. Raise ValueError if `values_for_forward["params"]` won't be valid according to the module spec. """ if _step_is_deleted(step): # refreshes from DB return None try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise ValueError("Module %s does not exist" % step.module_id_name) # Old values: store exactly what we had old_values = step.params module_spec = module_zipfile.get_spec() # [2021-07-05, adamhooper] nested here to prevent circular import error # in fetcher. Didn't bother to find the circle. from cjwstate.params import invoke_migrate_params # New values: store _migrated_ old_values, with new_values applied on # top migrated_old_values = invoke_migrate_params(module_zipfile, old_values) # Ensure migrate_params() didn't generate buggy _old_ values before we # add _new_ values. This sanity check may protect users' params by # raising an error early. It's also a way to catch bugs in unit tests. # (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params # returns `{}` -- which is often invalid -- and then the `**new_values` # below overwrites the invalid data. So without this validate(), a unit # test with an invalid migrate_params() may pass, which is wrong.) # # If you're seeing this because your unit test failed, try this: # self.kernel.migrate_params.side_effect = lambda m, p: p module_spec.param_schema.validate( migrated_old_values) # raises ValueError new_values = {**migrated_old_values, **new_values} module_spec.param_schema.validate(new_values) # raises ValueError return { **kwargs, "step": step, "values_for_backward": { "params": old_values }, "values_for_forward": { "params": new_values }, "step_delta_ids": self.affected_step_delta_ids(step), }
def create_application() -> ProtocolTypeRouter: """Create an ASGI application.""" # Load static modules on startup. # # This means starting a kernel and validating all static modules. There are # two good reasons to load during startup: # # 1. In dev mode, this reports errors in modules ASAP -- during startup # 2. In production, this import line costs time -- better to incur that # cost during startup than to incur it when responding to some random # request. cjwstate.modules.init_module_system() if not settings.I_AM_TESTING: # Only the test environment, Django runs migrations itself. We can't # use MODULE_REGISTRY until it migrates. MODULE_REGISTRY.all_latest() return ProtocolTypeRouter({ "websocket": AuthMiddlewareStack(SetCurrentLocaleAsgiMiddleware( create_url_router())) })
def _wf_module_set_secret_and_build_delta( workflow: Workflow, wf_module: WfModule, param: str, secret: str) -> Optional[clientside.Update]: """ Write a new secret to `wf_module`, or raise. Return a `clientside.Update`, or `None` if the database is not modified. Raise Workflow.DoesNotExist if the Workflow was deleted. """ with workflow.cooperative_lock(): # raises Workflow.DoesNotExist try: wf_module.refresh_from_db() except WfModule.DoesNotExist: return None # no-op if wf_module.secrets.get(param, {}).get("secret") == secret: return None # no-op try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) except KeyError: raise HandlerError( f"BadRequest: ModuleZipfile {wf_module.module_id_name} does not exist" ) module_spec = module_zipfile.get_spec() if not any(p.type == "secret" and p.secret_logic.provider == "string" for p in module_spec.param_fields): raise HandlerError( f"BadRequest: param is not a secret string parameter") created_at = timezone.now() created_at_str = ( created_at.strftime("%Y-%m-%dT%H:%M:%S") + "." + created_at.strftime("%f")[0:3] # milliseconds + "Z") wf_module.secrets = { **wf_module.secrets, param: { "name": created_at_str, "secret": secret }, } wf_module.save(update_fields=["secrets"]) return clientside.Update(steps={ wf_module.id: clientside.StepUpdate(secrets=wf_module.secret_metadata) })
def post(self, request: HttpRequest, workflow_id_or_secret_id: Union[int, str]): workflow = lookup_workflow_and_auth(authorized_read, workflow_id_or_secret_id, request) workflow2 = workflow.duplicate(request.user) ctx = JsonizeContext(request.locale_id, MODULE_REGISTRY.all_latest()) json_dict = jsonize_clientside_workflow(workflow2.to_clientside(), ctx, is_init=True) async_to_sync(rabbitmq.queue_render)(workflow2.id, workflow2.last_delta_id) return JsonResponse(json_dict, status=status.CREATED)
def load_from_workflow(cls, workflow: Workflow) -> DependencyGraph: """Create a DependencyGraph using the database. Must be called within a `workflow.cooperative_lock()`. Missing or deleted modules are deemed to have no dependencies. """ from cjwstate.models.module_registry import MODULE_REGISTRY module_zipfiles = MODULE_REGISTRY.all_latest() tabs = [] steps = {} for tab in workflow.live_tabs: tab_step_ids = [] for step in tab.live_steps: tab_step_ids.append(step.id) try: module_zipfile = module_zipfiles[step.module_id_name] except KeyError: steps[step.id] = cls.Step(frozenset()) continue module_spec = module_zipfile.get_spec() schema = module_spec.param_schema # Optimization: don't migrate_params() if we know there are no # tab params. (get_migrated_params() invokes module code, so we # prefer to wait and let it run in the renderer. if not _schema_contains_tabs(schema): steps[step.id] = cls.Step(frozenset()) continue from cjwstate.params import get_migrated_params params = get_migrated_params(step) # raises ValueError (and we don't handle that right now) schema.validate(params) steps[step.id] = cls.Step( gather_param_tab_slugs(schema, params)) tabs.append(cls.Tab(tab.slug, tab_step_ids)) return cls(tabs, steps)
def _add_wf_module_to_tab(step_dict, order, tab, delta_id, lesson): """ Deserialize a WfModule from lesson initial_workflow. Raise `KeyError` if a module ID is invalid. """ id_name = step_dict["module"] slug = step_dict["slug"] # 500 error if bad module id name module_zipfile = MODULE_REGISTRY.latest( id_name) # raise KeyError, RuntimeError module_spec = module_zipfile.get_spec() # All params not set in json get default values # Also, we must have a dict with all param values set or we can't migrate # params later params = {**module_spec.default_params, **step_dict["params"]} # Rewrite 'url' params: if the spec has them as relative, make them the # absolute path -- relative to the lesson URL. if "url" in params: if params["url"].startswith("./"): params["url"] = "".join([ settings.STATIC_URL, ("lessons/" if lesson.course is None else "courses/"), f"{lesson.locale_id}/", (lesson.slug if lesson.course is None else f"{lesson.course.slug}/{lesson.slug}"), params["url"][1:], # include the '/' ]) # 500 error if params are invalid # TODO testme module_spec.get_param_schema().validate(params) # raises ValueError return tab.wf_modules.create( order=order, slug=slug, module_id_name=id_name, is_busy=module_spec.loads_data, # assume we'll send a fetch last_relevant_delta_id=delta_id, params=params, is_collapsed=step_dict.get("collapsed", False), notes=step_dict.get("note", None), )
def post(self, request: HttpRequest, workflow_id: int): workflow = lookup_workflow_and_auth( Workflow.request_authorized_read, workflow_id, request ) workflow2 = workflow.duplicate(request.user) ctx = _get_request_jsonize_context(request, MODULE_REGISTRY.all_latest()) json_dict = jsonize_clientside_workflow( workflow2.to_clientside(), ctx, is_init=True ) server.utils.log_user_event_from_request( request, "Duplicate Workflow", {"name": workflow.name} ) async_to_sync(rabbitmq.queue_render)(workflow2.id, workflow2.last_delta_id) return JsonResponse(json_dict, status=status.CREATED)
def amend_create_kwargs( self, *, workflow, tab, slug, module_id_name, position, param_values, **kwargs ): """Add a step to the tab. Raise KeyError if `module_id_name` is invalid. Raise RuntimeError (unrecoverable) if s3 holds invalid module data. Raise ValueError if `param_values` do not match the module's spec. """ from ..step import Step # ensure slug is unique, or raise ValueError if Step.objects.filter(tab__workflow_id=workflow.id, slug=slug).count() > 0: raise ValueError("slug is not unique. Please pass a unique slug.") # raise KeyError, RuntimeError module_zipfile = MODULE_REGISTRY.latest(module_id_name) module_spec = module_zipfile.get_spec() # Set _all_ params (not just the user-specified ones). Our # dropdown-menu actions only specify the relevant params and expect us # to set the others to defaults. params = {**module_spec.default_params, **param_values} module_spec.get_param_schema().validate(params) # raises ValueError # step starts off "deleted" and gets un-deleted in forward(). step = tab.steps.create( module_id_name=module_id_name, order=position, slug=slug, is_deleted=True, params=params, cached_migrated_params=params, cached_migrated_params_module_version=module_zipfile.version, secrets={}, ) return { **kwargs, "workflow": workflow, "step": step, "step_delta_ids": self.affected_step_delta_ids(step), }
def amend_create_kwargs(cls, *, wf_module, new_values, **kwargs): """ Prepare `old_values` and `new_values`. Raise ValueError if `new_values` won't be valid according to the module spec. """ if cls.wf_module_is_deleted(wf_module): # refreshes from DB return None try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) except KeyError: raise ValueError("Module %s does not exist" % wf_module.module_id_name) # Old values: store exactly what we had old_values = wf_module.params module_spec = module_zipfile.get_spec() param_schema = module_spec.get_param_schema() # New values: store _migrated_ old_values, with new_values applied on # top migrated_old_values = invoke_migrate_params(module_zipfile, old_values) # Ensure migrate_params() didn't generate buggy _old_ values before we # add _new_ values. This sanity check may protect users' params by # raising an error early. It's also a way to catch bugs in unit tests. # (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params # returns `{}` -- which is often invalid -- and then the `**new_values` # below overwrites the invalid data. So without this validate(), a unit # test with an invalid migrate_params() may pass, which is wrong.) # # If you're seeing this because your unit test failed, try this: # self.kernel.migrate_params.side_effect = lambda m, p: p param_schema.validate(migrated_old_values) # raises ValueError new_values = {**migrated_old_values, **new_values} param_schema.validate(new_values) # raises ValueError return { **kwargs, "wf_module": wf_module, "new_values": new_values, "old_values": old_values, "wf_module_delta_ids": cls.affected_wf_module_delta_ids(wf_module), }
def from_workflow(cls, workflow: Workflow) -> Report.ReportWorkflow: module_zipfiles = MODULE_REGISTRY.all_latest() # prefetch would be nice, but it's tricky because A) we need to # filter out is_deleted; and B) we need to filter for modules that # have .html files. all_tabs = [ Report.TabWithIframes.from_tab(tab, module_zipfiles) for tab in workflow.live_tabs ] tabs = [tab for tab in all_tabs if tab.wf_modules] return cls( id=workflow.id, name=workflow.name, owner_name=workbench_user_display(workflow.owner), updated_at=workflow.last_delta.datetime, tabs=tabs, )
def post(self, request: HttpRequest, workflow: Workflow): workflow2 = workflow.duplicate(request.user) ctx = JsonizeContext( request.user, request.session, request.locale_id, dict(MODULE_REGISTRY.all_latest()), ) json_dict = jsonize_clientside_workflow(workflow2.to_clientside(), ctx, is_init=True) server.utils.log_user_event_from_request(request, "Duplicate Workflow", {"name": workflow.name}) async_to_sync(rabbitmq.queue_render)(workflow2.id, workflow2.last_delta_id) return JsonResponse(json_dict, status=status.HTTP_201_CREATED)
def _load_tab_flows(workflow: Workflow, delta_id: int) -> List[TabFlow]: """Query `workflow` for each tab's `TabFlow` (ordered by tab position). Raise `ModuleError` or `ValueError` if migrate_params() fails. Failed migration means the whole execute can't happen. """ ret = [] with workflow.cooperative_lock(): # reloads workflow if workflow.last_delta_id != delta_id: raise UnneededExecution module_zipfiles = MODULE_REGISTRY.all_latest() for tab_model in workflow.live_tabs.all(): steps = [ _build_execute_step(step, module_zipfiles=module_zipfiles) for step in tab_model.live_steps.all() ] ret.append(TabFlow(Tab(tab_model.slug, tab_model.name), steps)) return ret