def test_invalid_visible_if_menu_options(): with pytest.raises( ValueError, match=r"Param 'a' has visible_if values \{'x'\} not in 'b' options" ): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "string", "visible_if": {"id_name": "b", "value": ["a", "x"]}, }, { "id_name": "b", "type": "menu", "options": [ {"value": "a", "label": "A"}, {"value": "c", "label": "C"}, ], }, ], } )
def test_valid_visible_if_menu_options(): # does not raise load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "string", "visible_if": {"id_name": "b", "value": ["a", "b"]}, }, { "id_name": "b", "type": "menu", "options": [ {"value": "a", "label": "A"}, "separator", {"value": "b", "label": "B"}, {"value": "c", "label": "C"}, ], }, ], } )
def test_validate_gdrivefile_invalid_secret(): with pytest.raises( ValueError, match="Param 'b' 'secret_parameter' does not refer to a 'google'" ): load_spec( { "id_name": "twitter", # only twitter is allowed a twitter secret "name": "Name", "category": "Clean", "parameters": [ { "id_name": "twitter_credentials", "type": "secret", "secret_logic": { "provider": "oauth1a", "service": "twitter", }, }, { "id_name": "b", "type": "gdrivefile", "secret_parameter": "twitter_credentials", }, ], } )
def test_missing_radio_options(): with pytest.raises(ValueError): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [{"id_name": "radio", "type": "radio"}], } )
def test_multicolumn_tab_parameter(): # does not raise load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ {"id_name": "a", "type": "column", "tab_parameter": "b"}, {"id_name": "b", "type": "tab"}, ], } )
def test_schema_errors(): with pytest.raises( ValueError, match=r"'id_name' is a required property.*'category' is a required property.*'not a link at all' is not a 'uri'.*'NotABoolean' is not of type 'boolean'", ): load_spec( { "name": "Hello", "link": "not a link at all", "loads_data": "NotABoolean", "parameters": [], } )
def test_validate_gdrivefile_missing_secret(): with pytest.raises( ValueError, match="Param 'b' has a 'secret_parameter' that is not a 'secret'" ): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ {"id_name": "b", "type": "gdrivefile", "secret_parameter": "a"} ], } )
def test_unique_params(): with pytest.raises(ValueError, match="Param 'dup' appears twice"): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ {"id_name": "dup", "type": "string"}, {"id_name": "original", "type": "string"}, {"id_name": "dup", "type": "string"}, ], } )
def test_validate_allow_secret_based_on_module_id_name(): load_spec( { "id_name": "twitter", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "secret", "secret_logic": {"provider": "oauth1a", "service": "twitter"}, } ], } )
def test_multicolumn_non_tab_parameter(): with pytest.raises( ValueError, match="Param 'a' has a 'tab_parameter' that is not a 'tab'" ): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ {"id_name": "a", "type": "column", "tab_parameter": "b"}, {"id_name": "b", "type": "string"}, # Not a 'tab' ], } )
def test_happy_path(self): with tempdir_context() as tempdir: zip_path = tempdir / "importmodule.1.zip" with zipfile.ZipFile(zip_path, mode="w") as zf: zf.writestr( "importmodule.yaml", json.dumps( dict( id_name="importmodule", name="Importable module", category="Clean", parameters=[], ) ).encode("utf-8"), ) zf.writestr( "importmodule.py", b"def render(table, params): return table" ) clientside_module = import_zipfile(zip_path) self.assertEqual( clientside_module, clientside.Module( spec=load_spec( dict( id_name="importmodule", name="Importable module", category="Clean", parameters=[], ) ), js_module="", ), )
def create_or_replace_from_spec(spec, *, source_version_hash="", js_module="") -> "ModuleVersion": load_spec(dict(spec)) # raises ValueError module_version, _ = ModuleVersion.objects.update_or_create( id_name=spec["id_name"], source_version_hash=source_version_hash, defaults={ "spec": dict(spec), "js_module": js_module }, ) return module_version
def test_parameter_type_string(self): spec = load_spec({ "id_name": "testme", "name": "Test Module", "category": "Clean", "parameters": [{ "id_name": "hello", "type": "string", "name": "Hello there!", "placeholder": "Hey", "multiline": False, "default": "H", }], }) result = find_spec_messages(spec) expected = { "_spec.name": "Test Module", "_spec.parameters.hello.name": "Hello there!", "_spec.parameters.hello.placeholder": "Hey", "_spec.parameters.hello.default": "H", } self.assertDictEqual(result, expected)
def test_parameter_type_multicolumn(self): spec = load_spec({ "id_name": "testme", "name": "Test Module", "category": "Clean", "parameters": [ { "id_name": "hello", "type": "multicolumn", "name": "Hello there!", "placeholder": "Fill me", "column_types": ["text"], "tab_parameter": "tab", }, { "id_name": "tab", "type": "tab", "name": "Hello there 2!" }, ], }) result = find_spec_messages(spec) expected = { "_spec.name": "Test Module", "_spec.parameters.hello.name": "Hello there!", "_spec.parameters.hello.placeholder": "Fill me", "_spec.parameters.tab.name": "Hello there 2!", } self.assertDictEqual(result, expected)
def test_parameter_type_list(self): spec = load_spec({ "id_name": "testme", "name": "Test Module", "category": "Clean", "parameters": [{ "id_name": "hello", "type": "list", "name": "Hello there!", "child_parameters": [{ "id_name": "hello2", "type": "statictext", "name": "Hello there 2!", }], }], }) result = find_spec_messages(spec) expected = { "_spec.name": "Test Module", "_spec.parameters.hello.name": "Hello there!", "_spec.parameters.hello.child_parameters.hello2.name": "Hello there 2!", } self.assertDictEqual(result, expected)
def test_parameter_type_gdrivefile(self): spec = load_spec({ "id_name": "googlesheets", "name": "Test Module", "category": "Clean", "parameters": [ { "id_name": "google", "type": "secret", "secret_logic": { "provider": "oauth2", "service": "google" }, }, { "id_name": "hello2", "type": "gdrivefile", "secret_parameter": "google", }, ], }) result = find_spec_messages(spec) expected = {"_spec.name": "Test Module"} self.assertDictEqual(result, expected)
def test_param_schema_explicit(): spec = load_spec( dict( id_name="x", name="x", category="Clean", parameters=[{"id_name": "whee", "type": "custom"}], param_schema={ "id_name": { "type": "dict", "properties": { "x": {"type": "integer"}, "y": {"type": "string", "default": "X"}, }, } }, ) ) assert spec.param_schema == ParamSchema.Dict( { "id_name": ParamSchema.Dict( {"x": ParamSchema.Integer(), "y": ParamSchema.String(default="X")} ) } )
def test_everything_except_parameters(self): spec = load_spec({ "id_name": "testme", "name": "Test Module", "category": "Clean", "parameters": [], "description": "I do that", "deprecated": { "message": "Please use something else", "end_date": "2030-12-31", }, "icon": "url", "link": "http://example.com/module", "loads_data": False, "uses_data": True, "html_output": False, "has_zen_mode": True, "row_action_menu_entry_title": "Solve your problem", "help_url": "/testme", }) result = find_spec_messages(spec) expected = { "_spec.name": "Test Module", "_spec.description": "I do that", "_spec.deprecated.message": "Please use something else", "_spec.row_action_menu_entry_title": "Solve your problem", } self.assertDictEqual(result, expected)
def test_param_schema_implicit(): spec = load_spec( dict( id_name="googlesheets", name="x", category="Clean", parameters=[ {"id_name": "foo", "type": "string", "default": "X"}, { "id_name": "bar", "type": "secret", "secret_logic": {"provider": "oauth2", "service": "google"}, }, { "id_name": "baz", "type": "menu", "options": [ {"value": "a", "label": "A"}, "separator", {"value": "c", "label": "C"}, ], "default": "c", }, ], ) ) assert spec.param_schema == ParamSchema.Dict( { "foo": ParamSchema.String(default="X"), # secret is not in param_schema "baz": ParamSchema.Enum(choices=frozenset({"a", "c"}), default="c"), } )
def run_in_sandbox(compiled_module: CompiledModule, function: str, args: List[Any]) -> None: """Run `function` with `args`, and write the (Thrift) result to `sys.stdout`.""" # TODO sandbox -- will need an OS `clone()` with namespace, cgroups, .... # Run the user's code in a new (programmatic) module. # # This gives the user code a blank namespace -- exactly what we want. module_name = f"rawmodule.{compiled_module.module_slug}" user_code_module = types.ModuleType(module_name) sys.modules[module_name] = user_code_module # simulate "import" exec(compiled_module.code_object, user_code_module.__dict__) # And now ... now we're unsafe! Because `code_object` may be malicious, any # line of code from here on out gives undefined behavior. Luckily, a parent # is catching all possibile outcomes.... # Now override the pieces of the _default_ module with the user-supplied # ones. That way, when the default `render_pandas()` calls `render()`, that # `render()` is the user-code `render()` (if supplied). # # Good thing we've forked! This totally messes with global variables. module = cjwkernel.pandas.module for fn in ( "fetch", "fetch_arrow", "fetch_pandas", "fetch_thrift", "migrate_params", "migrate_params_thrift", "render", "render_arrow", "render_arrow_v1", "render_pandas", "render_thrift", ): if fn in user_code_module.__dict__: module.__dict__[fn] = user_code_module.__dict__[fn] # Set ModuleSpec global parameter -- module frameworks use it for params module.__dict__["ModuleSpec"] = load_spec(compiled_module.module_spec_dict) if function == "render_thrift": result = module.render_thrift(*args) elif function == "migrate_params_thrift": result = module.migrate_params_thrift(*args) elif function == "validate_thrift": result = module.validate_thrift(*args) elif function == "fetch_thrift": result = module.fetch_thrift(*args) else: raise NotImplementedError transport = thrift.transport.TTransport.TFileObjectTransport( sys.__stdout__.buffer) protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(transport) if result is not None: result.write(protocol) transport.flush()
def test_invalid_visible_if(): with pytest.raises( ValueError, match="Param 'a' has visible_if id_name 'b', which does not exist" ): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "string", "visible_if": {"id_name": "b", "value": True}, } ], } )
def test_multicolumn_missing_tab_parameter(): with pytest.raises( ValueError, match="Param 'a' has a 'tab_parameter' that is not in 'parameters'" ): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "column", "tab_parameter": "b", # does not exist } ], } )
def test_validate_disallow_secret_based_on_module_id_name(): with pytest.raises(ValueError, match="Denied access to global 'twitter' secrets"): load_spec( { "id_name": "eviltwitter", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "secret", "secret_logic": { "provider": "oauth1a", "service": "twitter", }, } ], } )
def test_only_required(self): spec = load_spec({ "id_name": "testme", "name": "Test Module", "category": "Clean", "parameters": [], }) result = find_spec_messages(spec) expected = {"_spec.name": "Test Module"} self.assertDictEqual(result, expected)
def test_uses_data_default_false_if_loads_data_true(): spec = load_spec( dict( id_name="id_name", name="Name", category="Add data", parameters=[], loads_data=True, ) ) assert not spec.uses_data
def test_uses_data_override(): spec = load_spec( dict( id_name="id_name", name="Name", category="Add data", parameters=[], loads_data=True, uses_data=True, ) ) assert spec.uses_data
def test_parameter_type_file(self): spec = load_spec({ "id_name": "testme", "name": "Test Module", "category": "Clean", "parameters": [{ "id_name": "hello", "type": "file" }], }) result = find_spec_messages(spec) expected = {"_spec.name": "Test Module"} self.assertDictEqual(result, expected)
def test_validate_menu_with_default(): # does not raise load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "menu", "placeholder": "Select something", "options": [ {"value": "x", "label": "X"}, "separator", {"value": "y", "label": "Y"}, {"value": "z", "label": "Z"}, ], "default": "y", } ], } )
def test_validate_menu_invalid_default(): with pytest.raises( ValueError, match="Param 'a' has a 'default' that is not in its 'options'" ): load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "menu", "options": [{"value": "x", "label": "X"}], "default": "y", }, { # Previously, we gave the wrong id_name "id_name": "not-a", "type": "string", }, ], } )
def test_valid_visible_if(): # does not raise spec = load_spec( { "id_name": "id", "name": "Name", "category": "Clean", "parameters": [ { "id_name": "a", "type": "string", "visible_if": {"id_name": "b", "value": True}, }, {"id_name": "b", "type": "string"}, ], } ) assert spec.param_fields[0].visible_if == {"id_name": "b", "value": True}