def test_db_minio_validate_code_with_kernel(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest7", "name": "regtest7 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest7.yaml", json.dumps(mv.spec).encode("utf-8")) zf.writestr( "regtest7.py", b"def render(table, params):\n return table\nfoo()") minio.put_bytes( minio.ExternalModulesBucket, "regtest7/regtest7.b1c2d3.zip", bytes(bio.getbuffer()), ) with self.assertRaises(RuntimeError) as cm: MODULE_REGISTRY.latest("regtest7") self.assertIsInstance(cm.exception.__cause__, ModuleExitedError)
def test_db_s3_validate_spec(self): mv = create_or_replace_from_spec( { "id_name": "regtest8", "name": "regtest8 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr( "regtest8.yaml", json.dumps({ **mv.spec, "parameters": "not an Array" }).encode("utf-8"), ) zf.writestr("regtest8.py", b"def render(table, params):\n return table") s3.put_bytes( s3.ExternalModulesBucket, "regtest8/regtest8.b1c2d3.zip", bytes(bio.getbuffer()), ) with self.assertRaises(RuntimeError) as cm: MODULE_REGISTRY.latest("regtest8") self.assertIsInstance(cm.exception.__cause__, ValueError)
def test_db_minio_syntax_error_is_runtime_error(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest9", "name": "regtest9 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr( "regtest9.yaml", json.dumps({ **mv.spec, "parameters": "not an Array" }).encode("utf-8"), ) zf.writestr("regtest9.py", b"def render(") minio.put_bytes( minio.ExternalModulesBucket, "regtest9/regtest9.b1c2d3.zip", bytes(bio.getbuffer()), ) with self.assertRaises(RuntimeError) as cm: MODULE_REGISTRY.latest("regtest9") self.assertIsInstance(cm.exception.__cause__, SyntaxError)
def test_db_minio_use_cache_for_same_version(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest4", "name": "regtest4 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest4.yaml", json.dumps(mv.spec).encode("utf-8")) zf.writestr("regtest4.py", b"def render(table, params):\n return table") minio.put_bytes( minio.ExternalModulesBucket, "regtest4/regtest4.b1c2d2.zip", bytes(bio.getbuffer()), ) zf1 = MODULE_REGISTRY.latest("regtest4") zf2 = MODULE_REGISTRY.latest("regtest4") self.assertIs(zf2, zf1)
def test_db_minio_refresh_cache_for_new_version(self): v1 = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest5", "name": "regtest5 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest5.yaml", json.dumps(v1.spec).encode("utf-8")) zf.writestr("regtest5.py", b"def render(table, params):\n return table") minio.put_bytes( minio.ExternalModulesBucket, "regtest5/regtest5.b1c2d2.zip", bytes(bio.getbuffer()), ) zipfile1 = MODULE_REGISTRY.latest("regtest5") ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest5", "name": "regtest5 v2", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest5/regtest5.b1c2d3.zip", bytes(bio.getbuffer()), # reuse zipfile to save lines of code ) zipfile2 = MODULE_REGISTRY.latest("regtest5") self.assertIsNot(zipfile2, zipfile1) self.assertEqual(zipfile2.version, "b1c2d3")
def build_init_state(): try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise Http404("Module is not embeddable") try: has_html = module_zipfile.get_optional_html() is not None except (UnicodeDecodeError, FileNotFoundError, zipfile.BadZipFile): has_html = False if not has_html: raise Http404("Module has no HTML") ctx = JsonizeContext( user=AnonymousUser(), user_profile=None, session=None, locale_id=request.locale_id, module_zipfiles={module_zipfile.module_id: module_zipfile}, ) return { "workflow": jsonize_clientside_workflow( workflow.to_clientside( include_tab_slugs=False, include_block_slugs=False, include_acl=False, ), ctx, is_init=True, ), "step": jsonize_clientside_step(step.to_clientside(), ctx), }
def _load_step_and_service(workflow: Workflow, step_id: int, param: str) -> Tuple[Step, oauth.OAuthService]: """Load Step and OAuthService from the database, or raise. Raise Step.DoesNotExist if the Step is deleted or missing. Raise SecretDoesNotExist if the Step does not have the given param. Invoke this within a Workflow.cooperative_lock(). """ # raises Step.DoesNotExist step = Step.live_in_workflow(workflow).get(pk=step_id) # raises KeyError, RuntimeError try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise SecretDoesNotExist( f"Module {step.module_id_name} does not exist") module_spec = module_zipfile.get_spec() for field in module_spec.param_fields: if (isinstance(field, ParamField.Secret) and field.id_name == param and (isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a) or isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2))): service_name = field.secret_logic.service service = oauth.OAuthService.lookup_or_none(service_name) if service is None: raise OauthServiceNotConfigured( f'OAuth not configured for "{service_name}" service') return step, service else: raise SecretDoesNotExist( f"Param {param} does not point to an OAuth secret")
def test_db_minio_latest_load_deprecated_simple(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest2", "name": "regtest2 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest2/b1c2d2/regtest2.py", "def render(table, params):\n return table", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest2/b1c2d2/regtest2.yaml", json.dumps(mv.spec).encode("utf-8"), ) zf = MODULE_REGISTRY.latest("regtest2") self.assertEqual(zf.get_spec(), ModuleSpec(**mv.spec)) self.assertIsNone(zf.get_optional_html())
def test_db_minio_latest_load_deprecated_html(self): mv = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest3", "name": "regtest3 v2", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest3/b1c2d2/regtest3.py", "def render(table, params):\n return table", ) minio.put_bytes( minio.ExternalModulesBucket, "regtest3/b1c2d2/regtest3.yaml", json.dumps(mv.spec).encode("utf-8"), ) html = "<!DOCTYPE html><html><head><title>Hi</title></head><body>Hello, world!</body></html>" minio.put_bytes( minio.ExternalModulesBucket, "regtest3/b1c2d2/regtest3.html", html.encode("utf-8"), ) zf = MODULE_REGISTRY.latest("regtest3") self.assertEqual(zf.get_optional_html(), html)
def wfmodule_output(request: HttpRequest, wf_module: WfModule, format=None): try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) html = module_zipfile.get_optional_html() except KeyError: html = None return HttpResponse(content=html)
def _lookup_service(step: Step, param: str) -> oauth.OAuthService: """Find the OAuthService that manages `param` on `step`. Raise `HandlerError` if we cannot. """ module_id = step.module_id_name try: module_zipfile = MODULE_REGISTRY.latest(module_id) except KeyError: raise HandlerError(f"BadRequest: module {module_id} not found") module_spec = module_zipfile.get_spec() for field in module_spec.param_fields: if ( field.id_name == param and isinstance(field, ParamField.Secret) and ( isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth1a) or isinstance(field.secret_logic, ParamField.Secret.Logic.Oauth2) ) ): service_name = field.secret_logic.service service = oauth.OAuthService.lookup_or_none(service_name) if not service: allowed_services = ", ".join(settings.OAUTH_SERVICES.keys()) raise HandlerError(f"AuthError: we only support {allowed_services}") return service else: raise HandlerError(f"Module {module_id} has no oauth {param} parameter")
def load_database_objects(workflow_id: int, wf_module_id: int) -> DatabaseObjects: """ Query WfModule info. Raise `WfModule.DoesNotExist` or `Workflow.DoesNotExist` if the step was deleted. Catch a `ModuleError` from migrate_params() and return it as part of the `DatabaseObjects`. """ with Workflow.lookup_and_cooperative_lock(id=workflow_id) as workflow_lock: # raise WfModule.DoesNotExist wf_module = WfModule.live_in_workflow( workflow_lock.workflow).get(id=wf_module_id) # module_zipfile try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) except KeyError: module_zipfile = None # migrated_params_or_error if module_zipfile is None: migrated_params_or_error = {} else: try: migrated_params_or_error = cjwstate.params.get_migrated_params( wf_module, module_zipfile=module_zipfile) # raise ModuleError except ModuleError as err: migrated_params_or_error = err # stored_object try: stored_object = wf_module.stored_objects.get( stored_at=wf_module.stored_data_version) except StoredObject.DoesNotExist: stored_object = None # input_crr try: # raise WfModule.DoesNotExist -- but we'll catch this one prev_module = wf_module.tab.live_wf_modules.get( order=wf_module.order - 1) input_crr = prev_module.cached_render_result # may be None except WfModule.DoesNotExist: input_crr = None return DatabaseObjects( wf_module, module_zipfile, migrated_params_or_error, stored_object, input_crr, )
def get_migrated_params( wf_module: WfModule, *, module_zipfile: ModuleZipfile = None) -> Dict[str, Any]: """ Read `wf_module.params`, calling migrate_params() or using cache fields. Call this within a `Workflow.cooperative_lock()`. If migrate_params() was already called for this version of the module, return the cached value. See `wf_module.cached_migrated_params`, `wf_module.cached_migrated_params_module_version`. Raise `ModuleError` if migration fails. Raise `KeyError` if the module was deleted. Raise `RuntimeError` (unrecoverable) if there is a problem loading or executing the module. (Modules are validated before import, so this should not happen.) The result may be invalid. Call `validate()` to raise a `ValueError` to detect that case. TODO avoid holding the database lock whilst executing stuff on the kernel. (This will involve auditing and modifying all callers to handle new error cases.) """ if module_zipfile is None: # raise KeyError module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) stale = ( module_zipfile.version == "develop" # works if cached version (and thus cached _result_) is None or (module_zipfile.get_param_schema_version() != wf_module.cached_migrated_params_module_version)) if not stale: return wf_module.cached_migrated_params else: # raise ModuleError params = invoke_migrate_params(module_zipfile, wf_module.params) wf_module.cached_migrated_params = params wf_module.cached_migrated_params_module_version = ( module_zipfile.get_param_schema_version()) try: wf_module.save(update_fields=[ "cached_migrated_params", "cached_migrated_params_module_version", ]) except ValueError: # WfModule was deleted, so we get: # "ValueError: Cannot force an update in save() with no primary key." pass return params
def to_clientside( self, *, force_module_zipfile: Optional[ModuleZipfile] = None ) -> clientside.StepUpdate: # module_zipfile, for params if force_module_zipfile: module_zipfile = force_module_zipfile else: from cjwstate.models.module_registry import MODULE_REGISTRY try: module_zipfile = MODULE_REGISTRY.latest(self.module_id_name) except KeyError: module_zipfile = None if module_zipfile is None: params = {} else: from cjwstate.params import get_migrated_params module_spec = module_zipfile.get_spec() param_schema = module_spec.param_schema # raise ModuleError params = get_migrated_params(self, module_zipfile=module_zipfile) try: param_schema.validate(params) except ValueError: logger.exception( "%s.migrate_params() gave invalid output: %r", self.module_id_name, params, ) params = param_schema.default crr = self._build_cached_render_result_fresh_or_not() if crr is None: crr = clientside.Null return clientside.StepUpdate( id=self.id, slug=self.slug, module_slug=self.module_id_name, tab_slug=self.tab_slug, is_busy=self.is_busy, render_result=crr, files=self._get_clientside_files(module_zipfile), params=params, secrets=self.secret_metadata, is_collapsed=self.is_collapsed, notes=self.notes, is_auto_fetch=self.auto_update_data, fetch_interval=self.update_interval, last_fetched_at=self.last_update_check, is_notify_on_change=self.notifications, last_relevant_delta_id=self.last_relevant_delta_id, versions=self._get_clientside_fetched_version_list(module_zipfile), )
def wrapper(request: HttpRequest, workflow_id: int, wf_module_slug: str, *args, **kwargs): auth_header = request.headers.get("Authorization", "") auth_header_match = AuthTokenHeaderRegex.match(auth_header) if not auth_header_match: return ErrorResponse(403, "authorization-bearer-token-not-provided") bearer_token = auth_header_match.group(1) try: with Workflow.lookup_and_cooperative_lock( id=workflow_id) as workflow_lock: workflow = workflow_lock.workflow try: wf_module = WfModule.live_in_workflow(workflow).get( slug=wf_module_slug) except WfModule.DoesNotExist: return ErrorResponse(404, "step-not-found") try: module_zipfile = MODULE_REGISTRY.latest( wf_module.module_id_name) except KeyError: return ErrorResponse(400, "step-module-deleted") try: file_param_id_name = next( iter(pf.id_name for pf in module_zipfile.get_spec().param_fields if pf.type == "file")) except StopIteration: return ErrorResponse(400, "step-has-no-file-param") api_token = wf_module.file_upload_api_token if not api_token: return ErrorResponse(403, "step-has-no-api-token") bearer_token_hash = hashlib.sha256( bearer_token.encode("utf-8")).digest() api_token_hash = hashlib.sha256( api_token.encode("utf-8")).digest() if bearer_token_hash != api_token_hash or bearer_token != api_token: return ErrorResponse(403, "authorization-bearer-token-invalid") return f( request, workflow_lock, wf_module, file_param_id_name, *args, **kwargs, ) except Workflow.DoesNotExist: return ErrorResponse(404, "workflow-not-found")
def create_module_zipfile( module_id: str = "testmodule", *, version: Optional[str] = None, spec_kwargs: Dict[str, Any] = {}, python_code: str = "", html: Optional[str] = None, js_module: str = "", extra_file_contents: Dict[str, bytes] = {}, ) -> ModuleZipfile: """ Create a ModuleZipfile, stored in the database and s3. If `version` is not supplied, generate one using the sha1 of the zipfile. This is usually what you want: s3 reads on overwrites are _eventually_ consistent, so if you 1. write a file; 2. overwrite it; and 3. read it, the read might result in the file from step 1 or the file from step 2. A sha1 version means overwrites will never modify data, solving the problem. """ spec = { "id_name": module_id, "name": "Test Module", "category": "Clean", "parameters": [], **spec_kwargs, } bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr(module_id + ".yaml", json.dumps(spec)) zf.writestr(module_id + ".py", python_code.encode("utf-8")) if html is not None: zf.writestr(module_id + ".html", html.encode("utf-8")) if js_module: zf.writestr(module_id + ".js", js_module.encode("utf-8")) for path, content in extra_file_contents.items(): zf.writestr(path, content) data = bytes(bio.getbuffer()) if version is None: sha1 = hashlib.sha1() sha1.update(data) version = sha1.hexdigest() s3.put_bytes( s3.ExternalModulesBucket, "%s/%s.%s.zip" % (module_id, module_id, version), data, ) ModuleVersion.objects.create(id_name=module_id, source_version_hash=version, spec=spec, js_module=js_module) return MODULE_REGISTRY.latest(module_id)
def amend_create_kwargs(self, *, step, new_values, **kwargs): """Prepare values_for_backward|forward["params"]. Raise ValueError if `values_for_forward["params"]` won't be valid according to the module spec. """ if _step_is_deleted(step): # refreshes from DB return None try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise ValueError("Module %s does not exist" % step.module_id_name) # Old values: store exactly what we had old_values = step.params module_spec = module_zipfile.get_spec() # [2021-07-05, adamhooper] nested here to prevent circular import error # in fetcher. Didn't bother to find the circle. from cjwstate.params import invoke_migrate_params # New values: store _migrated_ old_values, with new_values applied on # top migrated_old_values = invoke_migrate_params(module_zipfile, old_values) # Ensure migrate_params() didn't generate buggy _old_ values before we # add _new_ values. This sanity check may protect users' params by # raising an error early. It's also a way to catch bugs in unit tests. # (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params # returns `{}` -- which is often invalid -- and then the `**new_values` # below overwrites the invalid data. So without this validate(), a unit # test with an invalid migrate_params() may pass, which is wrong.) # # If you're seeing this because your unit test failed, try this: # self.kernel.migrate_params.side_effect = lambda m, p: p module_spec.param_schema.validate( migrated_old_values) # raises ValueError new_values = {**migrated_old_values, **new_values} module_spec.param_schema.validate(new_values) # raises ValueError return { **kwargs, "step": step, "values_for_backward": { "params": old_values }, "values_for_forward": { "params": new_values }, "step_delta_ids": self.affected_step_delta_ids(step), }
def _wf_module_set_secret_and_build_delta( workflow: Workflow, wf_module: WfModule, param: str, secret: str) -> Optional[clientside.Update]: """ Write a new secret to `wf_module`, or raise. Return a `clientside.Update`, or `None` if the database is not modified. Raise Workflow.DoesNotExist if the Workflow was deleted. """ with workflow.cooperative_lock(): # raises Workflow.DoesNotExist try: wf_module.refresh_from_db() except WfModule.DoesNotExist: return None # no-op if wf_module.secrets.get(param, {}).get("secret") == secret: return None # no-op try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) except KeyError: raise HandlerError( f"BadRequest: ModuleZipfile {wf_module.module_id_name} does not exist" ) module_spec = module_zipfile.get_spec() if not any(p.type == "secret" and p.secret_logic.provider == "string" for p in module_spec.param_fields): raise HandlerError( f"BadRequest: param is not a secret string parameter") created_at = timezone.now() created_at_str = ( created_at.strftime("%Y-%m-%dT%H:%M:%S") + "." + created_at.strftime("%f")[0:3] # milliseconds + "Z") wf_module.secrets = { **wf_module.secrets, param: { "name": created_at_str, "secret": secret }, } wf_module.save(update_fields=["secrets"]) return clientside.Update(steps={ wf_module.id: clientside.StepUpdate(secrets=wf_module.secret_metadata) })
def amend_create_kwargs( self, *, workflow, tab, slug, module_id_name, position, param_values, **kwargs ): """Add a step to the tab. Raise KeyError if `module_id_name` is invalid. Raise RuntimeError (unrecoverable) if s3 holds invalid module data. Raise ValueError if `param_values` do not match the module's spec. """ from ..step import Step # ensure slug is unique, or raise ValueError if Step.objects.filter(tab__workflow_id=workflow.id, slug=slug).count() > 0: raise ValueError("slug is not unique. Please pass a unique slug.") # raise KeyError, RuntimeError module_zipfile = MODULE_REGISTRY.latest(module_id_name) module_spec = module_zipfile.get_spec() # Set _all_ params (not just the user-specified ones). Our # dropdown-menu actions only specify the relevant params and expect us # to set the others to defaults. params = {**module_spec.default_params, **param_values} module_spec.get_param_schema().validate(params) # raises ValueError # step starts off "deleted" and gets un-deleted in forward(). step = tab.steps.create( module_id_name=module_id_name, order=position, slug=slug, is_deleted=True, params=params, cached_migrated_params=params, cached_migrated_params_module_version=module_zipfile.version, secrets={}, ) return { **kwargs, "workflow": workflow, "step": step, "step_delta_ids": self.affected_step_delta_ids(step), }
def _add_wf_module_to_tab(step_dict, order, tab, delta_id, lesson): """ Deserialize a WfModule from lesson initial_workflow. Raise `KeyError` if a module ID is invalid. """ id_name = step_dict["module"] slug = step_dict["slug"] # 500 error if bad module id name module_zipfile = MODULE_REGISTRY.latest( id_name) # raise KeyError, RuntimeError module_spec = module_zipfile.get_spec() # All params not set in json get default values # Also, we must have a dict with all param values set or we can't migrate # params later params = {**module_spec.default_params, **step_dict["params"]} # Rewrite 'url' params: if the spec has them as relative, make them the # absolute path -- relative to the lesson URL. if "url" in params: if params["url"].startswith("./"): params["url"] = "".join([ settings.STATIC_URL, ("lessons/" if lesson.course is None else "courses/"), f"{lesson.locale_id}/", (lesson.slug if lesson.course is None else f"{lesson.course.slug}/{lesson.slug}"), params["url"][1:], # include the '/' ]) # 500 error if params are invalid # TODO testme module_spec.get_param_schema().validate(params) # raises ValueError return tab.wf_modules.create( order=order, slug=slug, module_id_name=id_name, is_busy=module_spec.loads_data, # assume we'll send a fetch last_relevant_delta_id=delta_id, params=params, is_collapsed=step_dict.get("collapsed", False), notes=step_dict.get("note", None), )
def amend_create_kwargs(cls, *, wf_module, new_values, **kwargs): """ Prepare `old_values` and `new_values`. Raise ValueError if `new_values` won't be valid according to the module spec. """ if cls.wf_module_is_deleted(wf_module): # refreshes from DB return None try: module_zipfile = MODULE_REGISTRY.latest(wf_module.module_id_name) except KeyError: raise ValueError("Module %s does not exist" % wf_module.module_id_name) # Old values: store exactly what we had old_values = wf_module.params module_spec = module_zipfile.get_spec() param_schema = module_spec.get_param_schema() # New values: store _migrated_ old_values, with new_values applied on # top migrated_old_values = invoke_migrate_params(module_zipfile, old_values) # Ensure migrate_params() didn't generate buggy _old_ values before we # add _new_ values. This sanity check may protect users' params by # raising an error early. It's also a way to catch bugs in unit tests. # (DbTestCaseWithModuleRegistryAndMockKernel default migrate_params # returns `{}` -- which is often invalid -- and then the `**new_values` # below overwrites the invalid data. So without this validate(), a unit # test with an invalid migrate_params() may pass, which is wrong.) # # If you're seeing this because your unit test failed, try this: # self.kernel.migrate_params.side_effect = lambda m, p: p param_schema.validate(migrated_old_values) # raises ValueError new_values = {**migrated_old_values, **new_values} param_schema.validate(new_values) # raises ValueError return { **kwargs, "wf_module": wf_module, "new_values": new_values, "old_values": old_values, "wf_module_delta_ids": cls.affected_wf_module_delta_ids(wf_module), }
def test_db_minio_latest_order_by_last_update_time(self): # old version ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest1", "name": "regtest1 v1", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d3", ) time.sleep(0.000002) # guarantee new timestamp # new version v2 = ModuleVersion.create_or_replace_from_spec( { "id_name": "regtest1", "name": "regtest1 v2", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="b1c2d2", ) bio = io.BytesIO() with zipfile.ZipFile(bio, mode="w") as zf: zf.writestr("regtest1.yaml", json.dumps(v2.spec).encode("utf-8")) zf.writestr("regtest1.py", b"def render(table, params):\n return table") minio.put_bytes( minio.ExternalModulesBucket, "regtest1/regtest1.b1c2d2.zip", bytes(bio.getbuffer()), ) zf = MODULE_REGISTRY.latest("regtest1") self.assertEqual(zf.get_spec(), ModuleSpec(**v2.spec))
def embed(request, wfmodule_id): try: wf_module = WfModule.objects.get(pk=wfmodule_id, is_deleted=False) except WfModule.DoesNotExist: wf_module = None if wf_module: if not wf_module.workflow: wf_module = None elif not wf_module.workflow.request_authorized_read(request): wf_module = None else: try: module_zipfile = MODULE_REGISTRY.latest( wf_module.module_id_name) if not module_zipfile.get_spec().html_output: wf_module = None except KeyError: wf_module = None if wf_module: ctx = JsonizeContext( request.user, request.session, request.locale_id, {module_zipfile.module_id: module_zipfile}, ) init_state = { "workflow": jsonize_clientside_workflow( wf_module.workflow.to_clientside(include_tab_slugs=False), ctx, is_init=True, ), "wf_module": jsonize_clientside_step(wf_module.to_clientside(), ctx), } else: init_state = {"workflow": None, "wf_module": None} return TemplateResponse(request, "embed.html", {"initState": init_state})
def locked_and_loaded_step( workflow_id: int, step_slug: str ) -> ContextManager[Tuple[DbObjectCooperativeLock, Step, str]]: """Yield `WorkflowLock`, `step` and `file_param_id_name`. SECURITY: the caller may want to test the Step's `file_upload_api_token`. Raise UploadError(404, "workflow-not-found") on missing/deleted Workflow. Raise UploadError(404, "step-not-found") on missing/deleted Step. Raise UploadError(400, "step-module-deleted") on code-less Step. Raise UploadError(400, "step-has-no-file-param") on a Step with no File param. """ try: with Workflow.lookup_and_cooperative_lock( id=workflow_id) as workflow_lock: workflow = workflow_lock.workflow try: step = Step.live_in_workflow(workflow).get(slug=step_slug) except Step.DoesNotExist: raise UploadError(404, "step-not-found") try: module_zipfile = MODULE_REGISTRY.latest(step.module_id_name) except KeyError: raise UploadError(400, "step-module-deleted") try: file_param_id_name = next( iter(pf.id_name for pf in module_zipfile.get_spec().param_fields if pf.type == "file")) except StopIteration: raise UploadError(400, "step-has-no-file-param") yield workflow_lock, step, file_param_id_name except Workflow.DoesNotExist: raise UploadError(404, "workflow-not-found")
def import_module_from_url(url: str) -> clientside.Module: """Import zipfile from a URL. Return a `ModuleZipFile` on success. Raise `WorkbenchModuleImportError` if import fails, meaning: * The URL is not a URL we handle * There's an HTTP error * The ModuleZipfile is invalid """ match = GITHUB_URL_PATTERN.match(url) if match: clientside_module = import_module_from_github( match.group("owner"), match.group("repo") ) elif TEST_ZIP_URL_PATTERN.match(url): clientside_module = import_module_from_test_zip_url(url) else: raise WorkbenchModuleImportError( "Please supply a GitHub URL with owner=CJWorkbench" ) return clientside_module, MODULE_REGISTRY.latest(clientside_module.spec.id_name)
def to_clientside(self) -> clientside.StepUpdate: # params from cjwstate.models.module_registry import MODULE_REGISTRY try: module_zipfile = MODULE_REGISTRY.latest(self.module_id_name) except KeyError: module_zipfile = None if module_zipfile is None: params = {} else: from cjwstate.params import get_migrated_params module_spec = module_zipfile.get_spec() param_schema = module_spec.get_param_schema() # raise ModuleError params = get_migrated_params(self, module_zipfile=module_zipfile) try: param_schema.validate(params) except ValueError: logger.exception( "%s.migrate_params() gave invalid output: %r", self.module_id_name, params, ) params = param_schema.coerce(params) crr = self._build_cached_render_result_fresh_or_not() if crr is None: crr = clientside.Null return clientside.StepUpdate( id=self.id, slug=self.slug, module_slug=self.module_id_name, tab_slug=self.tab_slug, is_busy=self.is_busy, render_result=crr, files=[ clientside.UploadedFile( name=name, uuid=uuid, size=size, created_at=created_at ) for name, uuid, size, created_at in self.uploaded_files.order_by( "-created_at" ).values_list("name", "uuid", "size", "created_at") ], params=params, secrets=self.secret_metadata, is_collapsed=self.is_collapsed, notes=self.notes, is_auto_fetch=self.auto_update_data, fetch_interval=self.update_interval, last_fetched_at=self.last_update_check, is_notify_on_change=self.notifications, has_unseen_notification=self.has_unseen_notification, last_relevant_delta_id=self.last_relevant_delta_id, versions=clientside.FetchedVersionList( versions=[ clientside.FetchedVersion(created_at=created_at, is_seen=is_seen) for created_at, is_seen in self.stored_objects.order_by( "-stored_at" ).values_list("stored_at", "read") ], selected=self.stored_data_version, ), )
def _load_workflow_and_step_sync( request: HttpRequest, workflow_id_or_secret_id: Union[int, str], step_slug: str, accessing: Literal["all", "chart", "table"], ) -> Tuple[Workflow, Step]: """Load (Workflow, Step) from database, or raise Http404 or PermissionDenied. `Step.tab` will be loaded. (`Step.tab.workflow_id` is needed to access the render cache.) To avoid PermissionDenied: * The workflow must be public; OR * The user must be workflow owner, editor or viewer; OR * The user must be workflow report-viewer and the step must be a chart or table in the report. """ try: if isinstance(workflow_id_or_secret_id, int): search = {"id": workflow_id_or_secret_id} has_secret = False else: search = {"secret_id": workflow_id_or_secret_id} has_secret = True with Workflow.lookup_and_cooperative_lock(**search) as workflow_lock: workflow = workflow_lock.workflow if (has_secret or workflow.public or workflow.request_authorized_owner(request)): need_report_auth = False elif request.user is None or request.user.is_anonymous: raise PermissionDenied() else: try: acl_entry = workflow.acl.filter( email=request.user.email).get() except AclEntry.DoesNotExist: raise PermissionDenied() if acl_entry.role in {Role.VIEWER, Role.EDITOR}: need_report_auth = False elif acl_entry.role == Role.REPORT_VIEWER: need_report_auth = True else: raise PermissionDenied() # role we don't handle yet step = (Step.live_in_workflow( workflow.id).select_related("tab").get(slug=step_slug) ) # or Step.DoesNotExist if need_report_auth: # user is report-viewer if workflow.has_custom_report: if (accessing == "chart" and workflow.blocks.filter(step_id=step.id).exists()): pass # the step is a chart elif (accessing == "table" and workflow.blocks.filter(tab_id=step.tab_id).exists() and not step.tab.live_steps.filter( order__gt=step.order)): pass # step is a table (last step of a report-included tab) else: raise PermissionDenied() else: # Auto-report: all Charts are allowed; everything else is not try: if accessing == "chart" and (MODULE_REGISTRY.latest( step.module_id_name).get_spec().html_output): pass else: raise PermissionDenied() except KeyError: # not a module raise PermissionDenied() return workflow, step except (Workflow.DoesNotExist, Step.DoesNotExist): raise Http404()
def _load_module_zipfile(module_id_name: str) -> ModuleZipfile: """Return a ModuleZipfile or raise HandlerError.""" try: return MODULE_REGISTRY.latest(module_id_name) except KeyError: raise HandlerError("KeyError: ModuleVersion not found")
def test_latest_internal(self): zf = MODULE_REGISTRY.latest("pythoncode") self.assertEqual(zf.get_spec().id_name, "pythoncode")