def test_invalid_visible_if_menu_options():
with pytest.raises(
ValueError, match=r"Param 'a' has visible_if values \{'x'\} not in 'b' options"
):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "string",
"visible_if": {"id_name": "b", "value": ["a", "x"]},
},
{
"id_name": "b",
"type": "menu",
"options": [
{"value": "a", "label": "A"},
{"value": "c", "label": "C"},
],
},
],
}
)
def test_valid_visible_if_menu_options():
# does not raise
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "string",
"visible_if": {"id_name": "b", "value": ["a", "b"]},
},
{
"id_name": "b",
"type": "menu",
"options": [
{"value": "a", "label": "A"},
"separator",
{"value": "b", "label": "B"},
{"value": "c", "label": "C"},
],
},
],
}
)
def test_validate_gdrivefile_invalid_secret():
with pytest.raises(
ValueError, match="Param 'b' 'secret_parameter' does not refer to a 'google'"
):
load_spec(
{
"id_name": "twitter", # only twitter is allowed a twitter secret
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "twitter_credentials",
"type": "secret",
"secret_logic": {
"provider": "oauth1a",
"service": "twitter",
},
},
{
"id_name": "b",
"type": "gdrivefile",
"secret_parameter": "twitter_credentials",
},
],
}
)
def test_missing_radio_options():
with pytest.raises(ValueError):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [{"id_name": "radio", "type": "radio"}],
}
)
def test_multicolumn_tab_parameter():
# does not raise
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{"id_name": "a", "type": "column", "tab_parameter": "b"},
{"id_name": "b", "type": "tab"},
],
}
)
def test_schema_errors():
with pytest.raises(
ValueError,
match=r"'id_name' is a required property.*'category' is a required property.*'not a link at all' is not a 'uri'.*'NotABoolean' is not of type 'boolean'",
):
load_spec(
{
"name": "Hello",
"link": "not a link at all",
"loads_data": "NotABoolean",
"parameters": [],
}
)
def test_validate_gdrivefile_missing_secret():
with pytest.raises(
ValueError, match="Param 'b' has a 'secret_parameter' that is not a 'secret'"
):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{"id_name": "b", "type": "gdrivefile", "secret_parameter": "a"}
],
}
)
def test_unique_params():
with pytest.raises(ValueError, match="Param 'dup' appears twice"):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{"id_name": "dup", "type": "string"},
{"id_name": "original", "type": "string"},
{"id_name": "dup", "type": "string"},
],
}
)
def test_validate_allow_secret_based_on_module_id_name():
load_spec(
{
"id_name": "twitter",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "secret",
"secret_logic": {"provider": "oauth1a", "service": "twitter"},
}
],
}
)
def test_multicolumn_non_tab_parameter():
with pytest.raises(
ValueError, match="Param 'a' has a 'tab_parameter' that is not a 'tab'"
):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{"id_name": "a", "type": "column", "tab_parameter": "b"},
{"id_name": "b", "type": "string"}, # Not a 'tab'
],
}
)
def test_happy_path(self):
with tempdir_context() as tempdir:
zip_path = tempdir / "importmodule.1.zip"
with zipfile.ZipFile(zip_path, mode="w") as zf:
zf.writestr(
"importmodule.yaml",
json.dumps(
dict(
id_name="importmodule",
name="Importable module",
category="Clean",
parameters=[],
)
).encode("utf-8"),
)
zf.writestr(
"importmodule.py", b"def render(table, params): return table"
)
clientside_module = import_zipfile(zip_path)
self.assertEqual(
clientside_module,
clientside.Module(
spec=load_spec(
dict(
id_name="importmodule",
name="Importable module",
category="Clean",
parameters=[],
)
),
js_module="",
),
)
def create_or_replace_from_spec(spec,
*,
source_version_hash="",
js_module="") -> "ModuleVersion":
load_spec(dict(spec)) # raises ValueError
module_version, _ = ModuleVersion.objects.update_or_create(
id_name=spec["id_name"],
source_version_hash=source_version_hash,
defaults={
"spec": dict(spec),
"js_module": js_module
},
)
return module_version
def test_parameter_type_string(self):
spec = load_spec({
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [{
"id_name": "hello",
"type": "string",
"name": "Hello there!",
"placeholder": "Hey",
"multiline": False,
"default": "H",
}],
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.placeholder": "Hey",
"_spec.parameters.hello.default": "H",
}
self.assertDictEqual(result, expected)
def test_parameter_type_multicolumn(self):
spec = load_spec({
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [
{
"id_name": "hello",
"type": "multicolumn",
"name": "Hello there!",
"placeholder": "Fill me",
"column_types": ["text"],
"tab_parameter": "tab",
},
{
"id_name": "tab",
"type": "tab",
"name": "Hello there 2!"
},
],
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.placeholder": "Fill me",
"_spec.parameters.tab.name": "Hello there 2!",
}
self.assertDictEqual(result, expected)
def test_parameter_type_list(self):
spec = load_spec({
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [{
"id_name":
"hello",
"type":
"list",
"name":
"Hello there!",
"child_parameters": [{
"id_name": "hello2",
"type": "statictext",
"name": "Hello there 2!",
}],
}],
})
result = find_spec_messages(spec)
expected = {
"_spec.name":
"Test Module",
"_spec.parameters.hello.name":
"Hello there!",
"_spec.parameters.hello.child_parameters.hello2.name":
"Hello there 2!",
}
self.assertDictEqual(result, expected)
def test_parameter_type_gdrivefile(self):
spec = load_spec({
"id_name":
"googlesheets",
"name":
"Test Module",
"category":
"Clean",
"parameters": [
{
"id_name": "google",
"type": "secret",
"secret_logic": {
"provider": "oauth2",
"service": "google"
},
},
{
"id_name": "hello2",
"type": "gdrivefile",
"secret_parameter": "google",
},
],
})
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_param_schema_explicit():
spec = load_spec(
dict(
id_name="x",
name="x",
category="Clean",
parameters=[{"id_name": "whee", "type": "custom"}],
param_schema={
"id_name": {
"type": "dict",
"properties": {
"x": {"type": "integer"},
"y": {"type": "string", "default": "X"},
},
}
},
)
)
assert spec.param_schema == ParamSchema.Dict(
{
"id_name": ParamSchema.Dict(
{"x": ParamSchema.Integer(), "y": ParamSchema.String(default="X")}
)
}
)
def test_everything_except_parameters(self):
spec = load_spec({
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [],
"description": "I do that",
"deprecated": {
"message": "Please use something else",
"end_date": "2030-12-31",
},
"icon": "url",
"link": "http://example.com/module",
"loads_data": False,
"uses_data": True,
"html_output": False,
"has_zen_mode": True,
"row_action_menu_entry_title": "Solve your problem",
"help_url": "/testme",
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.description": "I do that",
"_spec.deprecated.message": "Please use something else",
"_spec.row_action_menu_entry_title": "Solve your problem",
}
self.assertDictEqual(result, expected)
def test_param_schema_implicit():
spec = load_spec(
dict(
id_name="googlesheets",
name="x",
category="Clean",
parameters=[
{"id_name": "foo", "type": "string", "default": "X"},
{
"id_name": "bar",
"type": "secret",
"secret_logic": {"provider": "oauth2", "service": "google"},
},
{
"id_name": "baz",
"type": "menu",
"options": [
{"value": "a", "label": "A"},
"separator",
{"value": "c", "label": "C"},
],
"default": "c",
},
],
)
)
assert spec.param_schema == ParamSchema.Dict(
{
"foo": ParamSchema.String(default="X"),
# secret is not in param_schema
"baz": ParamSchema.Enum(choices=frozenset({"a", "c"}), default="c"),
}
)
def run_in_sandbox(compiled_module: CompiledModule, function: str,
args: List[Any]) -> None:
"""Run `function` with `args`, and write the (Thrift) result to `sys.stdout`."""
# TODO sandbox -- will need an OS `clone()` with namespace, cgroups, ....
# Run the user's code in a new (programmatic) module.
#
# This gives the user code a blank namespace -- exactly what we want.
module_name = f"rawmodule.{compiled_module.module_slug}"
user_code_module = types.ModuleType(module_name)
sys.modules[module_name] = user_code_module # simulate "import"
exec(compiled_module.code_object, user_code_module.__dict__)
# And now ... now we're unsafe! Because `code_object` may be malicious, any
# line of code from here on out gives undefined behavior. Luckily, a parent
# is catching all possibile outcomes....
# Now override the pieces of the _default_ module with the user-supplied
# ones. That way, when the default `render_pandas()` calls `render()`, that
# `render()` is the user-code `render()` (if supplied).
#
# Good thing we've forked! This totally messes with global variables.
module = cjwkernel.pandas.module
for fn in (
"fetch",
"fetch_arrow",
"fetch_pandas",
"fetch_thrift",
"migrate_params",
"migrate_params_thrift",
"render",
"render_arrow",
"render_arrow_v1",
"render_pandas",
"render_thrift",
):
if fn in user_code_module.__dict__:
module.__dict__[fn] = user_code_module.__dict__[fn]
# Set ModuleSpec global parameter -- module frameworks use it for params
module.__dict__["ModuleSpec"] = load_spec(compiled_module.module_spec_dict)
if function == "render_thrift":
result = module.render_thrift(*args)
elif function == "migrate_params_thrift":
result = module.migrate_params_thrift(*args)
elif function == "validate_thrift":
result = module.validate_thrift(*args)
elif function == "fetch_thrift":
result = module.fetch_thrift(*args)
else:
raise NotImplementedError
transport = thrift.transport.TTransport.TFileObjectTransport(
sys.__stdout__.buffer)
protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(transport)
if result is not None:
result.write(protocol)
transport.flush()
def test_invalid_visible_if():
with pytest.raises(
ValueError, match="Param 'a' has visible_if id_name 'b', which does not exist"
):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "string",
"visible_if": {"id_name": "b", "value": True},
}
],
}
)
def test_multicolumn_missing_tab_parameter():
with pytest.raises(
ValueError, match="Param 'a' has a 'tab_parameter' that is not in 'parameters'"
):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "column",
"tab_parameter": "b", # does not exist
}
],
}
)
def test_validate_disallow_secret_based_on_module_id_name():
with pytest.raises(ValueError, match="Denied access to global 'twitter' secrets"):
load_spec(
{
"id_name": "eviltwitter",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "secret",
"secret_logic": {
"provider": "oauth1a",
"service": "twitter",
},
}
],
}
)
def test_only_required(self):
spec = load_spec({
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [],
})
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_uses_data_default_false_if_loads_data_true():
spec = load_spec(
dict(
id_name="id_name",
name="Name",
category="Add data",
parameters=[],
loads_data=True,
)
)
assert not spec.uses_data
def test_uses_data_override():
spec = load_spec(
dict(
id_name="id_name",
name="Name",
category="Add data",
parameters=[],
loads_data=True,
uses_data=True,
)
)
assert spec.uses_data
def test_parameter_type_file(self):
spec = load_spec({
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [{
"id_name": "hello",
"type": "file"
}],
})
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_validate_menu_with_default():
# does not raise
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "menu",
"placeholder": "Select something",
"options": [
{"value": "x", "label": "X"},
"separator",
{"value": "y", "label": "Y"},
{"value": "z", "label": "Z"},
],
"default": "y",
}
],
}
)
def test_validate_menu_invalid_default():
with pytest.raises(
ValueError, match="Param 'a' has a 'default' that is not in its 'options'"
):
load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "menu",
"options": [{"value": "x", "label": "X"}],
"default": "y",
},
{
# Previously, we gave the wrong id_name
"id_name": "not-a",
"type": "string",
},
],
}
)
def test_valid_visible_if():
# does not raise
spec = load_spec(
{
"id_name": "id",
"name": "Name",
"category": "Clean",
"parameters": [
{
"id_name": "a",
"type": "string",
"visible_if": {"id_name": "b", "value": True},
},
{"id_name": "b", "type": "string"},
],
}
)
assert spec.param_fields[0].visible_if == {"id_name": "b", "value": True}
