def test_param_schema_explicit(self):
spec = ModuleSpec(
id_name="x",
name="x",
category="Clean",
parameters=[{
"id_name": "whee",
"type": "custom"
}],
param_schema={
"id_name": {
"type": "dict",
"properties": {
"x": {
"type": "integer"
},
"y": {
"type": "string",
"default": "X"
},
},
}
},
)
self.assertEqual(
spec.get_param_schema(),
ParamDType.Dict({
"id_name":
ParamDType.Dict({
"x": ParamDType.Integer(),
"y": ParamDType.String(default="X")
})
}),
)
def test_uses_data_default_true_if_loads_data_false(self):
spec = ModuleSpec(
id_name="id_name",
name="Name",
category="Add data",
parameters=[],
loads_data=True,
)
self.assertFalse(spec.get_uses_data())
def test_uses_data_override(self):
spec = ModuleSpec(
id_name="idname",
name="Name",
category="Add data",
parameters=[],
loads_data=True,
uses_data=True,
)
self.assertTrue(spec.get_uses_data())
def test_param_schema_implicit(self):
spec = ModuleSpec(
id_name="googlesheets",
name="x",
category="Clean",
parameters=[
{
"id_name": "foo",
"type": "string",
"default": "X"
},
{
"id_name": "bar",
"type": "secret",
"secret_logic": {
"provider": "oauth2",
"service": "google"
},
},
{
"id_name":
"baz",
"type":
"menu",
"options": [
{
"value": "a",
"label": "A"
},
"separator",
{
"value": "c",
"label": "C"
},
],
"default":
"c",
},
],
)
self.assertEqual(
spec.get_param_schema(),
ParamDType.Dict({
"foo":
ParamDType.String(default="X"),
# secret is not in param_schema
"baz":
ParamDType.Enum(choices=frozenset({"a", "c"}), default="c"),
}),
)
def test_parameter_type_gdrivefile(self):
spec = ModuleSpec(
**{
"id_name":
"googlesheets",
"name":
"Test Module",
"category":
"Clean",
"parameters": [
{
"id_name": "google",
"type": "secret",
"secret_logic": {
"provider": "oauth2",
"service": "google"
},
},
{
"id_name": "hello2",
"type": "gdrivefile",
"secret_parameter": "google",
},
],
})
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_parameter_type_multicolumn(self):
spec = ModuleSpec(
**{
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [
{
"id_name": "hello",
"type": "multicolumn",
"name": "Hello there!",
"placeholder": "Fill me",
"column_types": ["text"],
"tab_parameter": "tab",
},
{
"id_name": "tab",
"type": "tab",
"name": "Hello there 2!"
},
],
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.placeholder": "Fill me",
"_spec.parameters.tab.name": "Hello there 2!",
}
self.assertDictEqual(result, expected)
def test_everything_except_parameters(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [],
"description": "I do that",
"deprecated": {
"message": "Please use something else",
"end_date": "2030-12-31",
},
"icon": "url",
"link": "http://example.com/module",
"loads_data": False,
"uses_data": True,
"html_output": False,
"has_zen_mode": True,
"row_action_menu_entry_title": "Solve your problem",
"help_url": "/testme",
"parameters_version": 3,
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.description": "I do that",
"_spec.deprecated.message": "Please use something else",
"_spec.row_action_menu_entry_title": "Solve your problem",
}
self.assertDictEqual(result, expected)
def test_ignore_empty(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"description": "",
"category": "Clean",
"parameters": [
{
"id_name": "hello",
"type": "list",
"name": "",
"child_parameters": [
{"id_name": "hello2", "type": "statictext", "name": ""}
],
},
{
"id_name": "hello3",
"type": "string",
"name": "Hello there!",
"placeholder": "",
"multiline": True,
"default": "",
},
],
}
)
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello3.name": "Hello there!",
}
self.assertDictEqual(result, expected)
def test_parameter_type_secret_string(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [
{
"id_name": "hello",
"type": "secret",
"secret_logic": {
"provider": "string",
"label": "Secret",
"pattern": "[A-Z]{10,12}",
"placeholder": "AAAAAAAAAAAA",
"help": "Find it there",
"help_url_prompt": "Take me there",
"help_url": "https://example.com/get_secret",
},
}
],
}
)
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.secret_logic.label": "Secret",
"_spec.parameters.hello.secret_logic.help": "Find it there",
"_spec.parameters.hello.secret_logic.help_url_prompt": "Take me there",
"_spec.parameters.hello.secret_logic.help_url": "https://example.com/get_secret",
}
self.assertDictEqual(result, expected)
def test_parameter_type_radio(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [
{
"id_name": "hello",
"type": "radio",
"name": "Hello there!",
"options": [
{"label": "First", "value": "first"},
{"label": "Second", "value": True},
],
"default": "first",
}
],
}
)
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.options.first.label": "First",
"_spec.parameters.hello.options.True.label": "Second",
}
self.assertDictEqual(result, expected)
def test_parameter_type_list(self):
spec = ModuleSpec(
**{
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [{
"id_name":
"hello",
"type":
"list",
"name":
"Hello there!",
"child_parameters": [{
"id_name": "hello2",
"type": "statictext",
"name": "Hello there 2!",
}],
}],
})
result = find_spec_messages(spec)
expected = {
"_spec.name":
"Test Module",
"_spec.parameters.hello.name":
"Hello there!",
"_spec.parameters.hello.child_parameters.hello2.name":
"Hello there 2!",
}
self.assertDictEqual(result, expected)
def test_happy_path(self):
with tempdir_context() as tempdir:
zip_path = tempdir / "importmodule.1.zip"
with zipfile.ZipFile(zip_path, mode="w") as zf:
zf.writestr(
"importmodule.yaml",
json.dumps(
dict(
id_name="importmodule",
name="Importable module",
category="Clean",
parameters=[],
)).encode("utf-8"),
)
zf.writestr("importmodule.py",
b"def render(table, params): return table")
clientside_module = import_zipfile(zip_path)
self.assertEqual(
clientside_module,
clientside.Module(
spec=ModuleSpec(
id_name="importmodule",
name="Importable module",
category="Clean",
parameters=[],
),
js_module="",
),
)
def test_parameter_type_menu(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [
{
"id_name": "hello",
"type": "menu",
"name": "Hello there!",
"placeholder": "Choose something...",
"options": [
{"label": "First", "value": "first"},
"separator",
{"label": "Second", "value": "second"},
],
"default": "first",
}
],
}
)
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.placeholder": "Choose something...",
"_spec.parameters.hello.options.first.label": "First",
"_spec.parameters.hello.options.second.label": "Second",
}
self.assertDictEqual(result, expected)
def test_parameter_type_string(self):
spec = ModuleSpec(
**{
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [{
"id_name": "hello",
"type": "string",
"name": "Hello there!",
"placeholder": "Hey",
"multiline": False,
"default": "H",
}],
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.placeholder": "Hey",
"_spec.parameters.hello.default": "H",
}
self.assertDictEqual(result, expected)
def test_db_minio_latest_load_deprecated_simple(self):
mv = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest2",
"name": "regtest2 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest2/b1c2d2/regtest2.py",
"def render(table, params):\n return table",
)
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest2/b1c2d2/regtest2.yaml",
json.dumps(mv.spec).encode("utf-8"),
)
zf = MODULE_REGISTRY.latest("regtest2")
self.assertEqual(zf.get_spec(), ModuleSpec(**mv.spec))
self.assertIsNone(zf.get_optional_html())
def test_only_required(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [],
})
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_parameter_type_file(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [{"id_name": "hello", "type": "file"}],
}
)
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_default_params(self):
spec = ModuleSpec(
id_name="x",
name="x",
category="Clean",
parameters=[{
"id_name": "foo",
"type": "string",
"default": "X"
}],
)
self.assertEqual(spec.default_params, {"foo": "X"})
def test_parameter_type_custom(self):
spec = ModuleSpec(
**{
"id_name": "testme",
"name": "Test Module",
"category": "Clean",
"parameters": [
{"id_name": "hello", "type": "custom", "name": "Hello there!"}
],
}
)
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
}
self.assertDictEqual(result, expected)
def test_parameter_type_secret_oauth1a(self):
spec = ModuleSpec(
**{
"id_name": "twitter",
"name": "Test Module",
"category": "Clean",
"parameters": [
{
"id_name": "hello",
"type": "secret",
"secret_logic": {"provider": "oauth1a", "service": "twitter"},
}
],
}
)
result = find_spec_messages(spec)
expected = {"_spec.name": "Test Module"}
self.assertDictEqual(result, expected)
def test_db_minio_latest_order_by_last_update_time(self):
# old version
ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest1",
"name": "regtest1 v1",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d3",
)
time.sleep(0.000002) # guarantee new timestamp
# new version
v2 = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "regtest1",
"name": "regtest1 v2",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="b1c2d2",
)
bio = io.BytesIO()
with zipfile.ZipFile(bio, mode="w") as zf:
zf.writestr("regtest1.yaml", json.dumps(v2.spec).encode("utf-8"))
zf.writestr("regtest1.py",
b"def render(table, params):\n return table")
minio.put_bytes(
minio.ExternalModulesBucket,
"regtest1/regtest1.b1c2d2.zip",
bytes(bio.getbuffer()),
)
zf = MODULE_REGISTRY.latest("regtest1")
self.assertEqual(zf.get_spec(), ModuleSpec(**v2.spec))
def test_parameter_type_numberformat(self):
spec = ModuleSpec(
**{
"id_name":
"testme",
"name":
"Test Module",
"category":
"Clean",
"parameters": [{
"id_name": "hello",
"type": "numberformat",
"name": "Hello there!",
"placeholder": "Fill me",
}],
})
result = find_spec_messages(spec)
expected = {
"_spec.name": "Test Module",
"_spec.parameters.hello.name": "Hello there!",
"_spec.parameters.hello.placeholder": "Fill me",
}
self.assertDictEqual(result, expected)
