def test_error_message3(self):
import traceback
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
@command
def expected(data, arg):
return f"{data} {arg}"
assert evaluate("a/b/-/world/expected-x").get() == "hello world x"
try:
evaluate("a/b/-/expected").get()
assert False
except Exception as e:
traceback.print_exc()
assert e.query == "a/b/-/expected"
assert e.position.offset == 6
try:
evaluate("a/b/-/expected-x-y").get()
assert False
except Exception as e:
assert e.query == "a/b/-/expected-x-y"
assert e.position.offset == 6
def test_tsplit(self):
import importlib
import liquer.ext.lq_pandas # register pandas commands and state type
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.lq_pandas)
importlib.reload(liquer.ext.basic)
set_var("server", "http://localhost")
set_var("api_path", "/q/")
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test_hxl.csv")
df = evaluate(f"df_from-{filename}/tsplit_df-a").get()
assert "a" in df.columns
assert "b" not in df.columns
assert list(df.a) == ["#indicator +num +aaa", "1", "3"]
assert list(df["query"])[1:] == [
f"df_from-{filename}/teq-a-1",
f"df_from-{filename}/teq-a-3",
]
assert list(df["link"])[1:] == [
f"http://localhost/q/df_from-{filename}/teq-a-1",
f"http://localhost/q/df_from-{filename}/teq-a-3",
]
def test_evaluate_chaining_volatile(self):
import importlib
from liquer import evaluate
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
@first_command(volatile=True)
def test_volatile():
return 123
@first_command
def test_nonvolatile():
return 234
@command
def test_callable2(x): # has state as a first argument
return x * 10
state1 = evaluate("test_volatile/test_callable2")
assert state1.get() == 1230
assert state1.is_volatile()
state2 = evaluate("test_nonvolatile/test_callable2")
assert state2.get() == 2340
assert not state2.is_volatile()
def setup_class(cls):
from liquer.commands import reset_command_registry
import liquer.ext.lq_pandas # register pandas commands and state type
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.lq_pandas)
def test_registration_namespace(self):
import importlib
from liquer import evaluate
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
assert "flag" in command_registry().as_dict()["root"]
try:
@command(ns="new")
def flag(state, name):
return f"New definition of flag called with {name}"
redefined = True
except:
redefined = False
assert redefined
# assert evaluate("/flag-test-f/flag-test/state_variable-test").get() == True
# assert evaluate("/flag-test-f/ns-root/flag-test/state_variable-test").get() == True
assert evaluate(
"/flag-test-f/ns-new/flag-test/state_variable-test").get() == False
# Cleanup
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
def test_status(self):
import liquer.store as st
reset_command_registry()
@first_command
def hello(x):
return f"Hello, {x}"
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- hello-RECIPES/hello1.txt
subdir:
- hello-subdir/hello2.txt
""",
{},
)
store = RecipeSpecStore(substore)
assert "hello1.txt" in store.get_bytes(store.STATUS_FILE).decode("utf-8")
assert "recipe " in store.get_bytes(store.STATUS_FILE).decode("utf-8")
assert store.get_bytes("hello1.txt") == b"Hello, RECIPES"
assert "hello1.txt" in store.get_bytes(store.STATUS_FILE).decode("utf-8")
assert "ready " in store.get_bytes(store.STATUS_FILE).decode("utf-8")
assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir"
assert "hello2.txt" in store.get_bytes("subdir/"+store.STATUS_FILE).decode("utf-8")
assert "ready " in store.get_bytes("subdir/"+store.STATUS_FILE).decode("utf-8")
def test_vars_context(self):
reset_command_registry()
set_var("test_var", "INITIAL")
@first_command
def varcommand(context=None):
is_initial = context.vars.test_var == "INITIAL"
context.vars.test_var = "MODIFIED"
return is_initial
@command
def varcommand_state(state,context=None):
is_initial = state.vars.get("test_var") == "INITIAL"
state.vars["test_var"] = "MODIFIED"
return is_initial
@command
def check1(state, context=None):
print(f"Check1: ", state.vars["test_var"])
return state.vars["test_var"] == "MODIFIED"
@command
def check2(state, context=None):
print(f"Check2: ", context.vars.test_var)
return context.vars.test_var == "MODIFIED"
assert get_context().evaluate("varcommand").get() == True
assert get_context().evaluate("varcommand_state").get() == True
assert get_context().evaluate("check1").get() == False
assert get_context().evaluate("check2").get() == False
assert get_context().evaluate("varcommand/check1").get() == True
assert get_context().evaluate("varcommand/check2").get() == True
assert get_context().evaluate("varcommand_state/check1").get() == True
assert get_context().evaluate("varcommand_state/check2").get() == True
def test_ridge(self):
import importlib
import liquer.ext.basic
import liquer.ext.lq_pandas
import liquer.ext.lq_sklearn_regression
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.lq_pandas)
importlib.reload(liquer.ext.lq_sklearn_regression)
@first_command
def test1():
return pd.DataFrame(dict(x=[1, 2, 3], Y=[10, 20, 30]))
@first_command
def test2():
return pd.DataFrame(dict(x=[1, 2, 3], Y=[30, 40, 50]))
@first_command
def test3():
return pd.DataFrame(dict(x1=[1, 2, 3], x2=[0, 0, 1], Y=[30, 40, 55]))
df = evaluate("ns-sklearn/test1/ridge").get()
assert list(df.x)[0] == pytest.approx(9.52381)
df = evaluate("ns-sklearn/test2/ridge").get()
assert list(df.x)[0] == pytest.approx(9.52381)
assert list(df.intercept)[0] == pytest.approx(20.952381)
df = evaluate("ns-sklearn/test3/ridge").get()
assert list(df.x1)[0] == pytest.approx(9.562842)
assert list(df.x2)[0] == pytest.approx(4.918033)
assert list(df.intercept)[0] == pytest.approx(20.901639)
def test_evaluate_chaining_exceptions(self):
import importlib
from liquer import evaluate
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
@command(ABC="def", ns="testns", context_menu="menu")
def test_callable1(state,
a: int,
b=123): # has state as a first argument
return a + b
@command
def test_callable2(state): # has state as a first argument
return state
state1 = evaluate("ns-testns/test_callable1-1")
assert state1.get() == 124
assert state1.metadata["attributes"]["ABC"] == "def"
assert state1.metadata["attributes"]["ns"] == "testns"
assert state1.metadata["attributes"]["context_menu"] == "menu"
state2 = evaluate("ns-testns/test_callable1-1/test_callable2")
assert state2.get() == 124
assert state2.metadata["attributes"]["ABC"] == "def"
assert state2.metadata["attributes"].get("ns") != "testns"
assert "context_menu" not in state2.metadata["attributes"]
def test_recipe_spec_store(self):
import liquer.store as st
import liquer.recipes as r
from liquer.cache import MemoryCache, set_cache, get_cache
reset_command_registry()
set_cache(None)
@first_command
def hello():
return "Hello"
store = r.RecipeSpecStore(st.MemoryStore())
store.store(
"results/recipes.yaml",
b"""
subdir:
- hello/hello.txt
""",
{},
)
assert "results/subdir/hello.txt" in store.recipes()
assert store.contains("results")
assert store.contains("results/subdir")
assert store.contains("results/subdir/hello.txt")
assert store.is_dir("results")
assert store.is_dir("results/subdir")
assert not store.is_dir("results/subdir/hello.txt")
assert store.get_bytes("results/subdir/hello.txt") == b"Hello"
def test_as_dict(self):
reset_command_registry()
@command
def somecommand(x: int): # has state as a first argument
return 123 + x
assert "somecommand" in command_registry().as_dict()["root"]
def test_duplicate_registration(self):
reset_command_registry()
def somecommand(x: int): # has state as a first argument
return 123 + x
command(somecommand)
command(somecommand)
assert "somecommand" in command_registry().as_dict()["root"]
def setup_class(cls):
from liquer.commands import reset_command_registry
import liquer.ext.lq_pil # register PIL commands and state type
import liquer.ext.basic # to get ns
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.lq_pil)
def test_state_command(self):
reset_command_registry()
@command
def statecommand(state): # has state as a first argument
assert isinstance(state, State)
return 123 + state.get()
assert command_registry().evaluate_command(
State().with_data(1), ["statecommand"]).get() == 124
def test_nonstate_command(self):
reset_command_registry()
@command
def nonstatecommand(x: int): # has state as a first argument
assert x == 1
return 123 + x
assert command_registry().evaluate_command(
State().with_data(1), ["nonstatecommand"]).get() == 124
def test_first_command(self):
reset_command_registry()
@first_command
def test_callable(a: int, b=123):
return a + b
result = command_registry().executables["root"]["test_callable"](
State(), "1")
assert result.get() == 124
def setup_class(cls):
import importlib
from liquer.commands import reset_command_registry
import liquer.ext.basic
import liquer.ext.meta
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.meta)
set_store(MemoryStore())
def test_command(self):
reset_command_registry()
@command
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
ns, _command, command_metadata = command_registry().resolve_command(State(),"test_callable")
m = Dependencies()
m.add_command_dependency(ns, command_metadata)
assert "ns-root/test_callable" in m.as_dict()["commands"]
def test_changing_attributes(self):
reset_command_registry()
def somecommand(x: int): # has state as a first argument
return 123 + x
command(somecommand)
assert "abc" not in command_registry(
).metadata["root"]["somecommand"].attributes
command(abc="def")(somecommand)
assert "def" == command_registry(
).metadata["root"]["somecommand"].attributes["abc"]
def test_concat_recipe(self):
import pandas as pd
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.lq_pandas)
import liquer.store as st
@first_command
def hello(offset=0):
return pd.DataFrame(
dict(a=[1 + offset, 2 + offset], b=[3 + offset, 4 + offset]))
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- filename: hello.parquet
type: pandas_concat
concat:
- hello
- query: hello-10
column: test
value: extra
""",
{},
)
store = RecipeSpecStore(substore)
set_store(store)
assert "hello.parquet" in store.keys()
df = evaluate("hello.parquet/-/dr").get()
assert sorted(df.columns) == ["a", "b", "test"]
assert list(df.a) == [1, 2, 11, 12]
assert list(df.b) == [3, 4, 13, 14]
assert list(df.test) == [None, None, "extra", "extra"]
assert store.get_metadata(
"hello.parquet")["status"] == Status.READY.value
assert store.get_metadata(
"hello.parquet")["recipes_key"] == "recipes.yaml"
assert store.get_metadata("hello.parquet")["has_recipe"] == True
assert store.get_metadata("hello.parquet")["recipes_directory"] == ""
assert store.get_metadata("hello.parquet")[
"recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet"
assert store.get_metadata("hello.parquet")["data_characteristics"][
"description"] == "Dataframe with 3 columns and 4 rows."
set_store(None)
reset_command_registry()
def test_evaluate_command(self):
reset_command_registry()
@command
def test_callable(state,
a: int,
b=123): # has state as a first argument
return a + b
cmd = ["test_callable", "1"]
result = command_registry().evaluate_command(State(), cmd)
assert result.get() == 124
assert result.commands[-1] == cmd
def test_evaluate_action_with_arguments(self):
reset_command_registry()
@command
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
context = get_context()
action = ActionRequest.from_arguments("test_callable", "1")
result = context.evaluate_action(State(), action, extra_parameters=[234])
assert result.is_volatile()
assert result.get() == 235
assert result.metadata["commands"][-1] == ["test_callable", "1"]
def test_evaluate_command_with_attributes(self):
reset_command_registry()
@command(ABC="def")
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
context = get_context()
action = ActionRequest.from_arguments("test_callable", "1")
result = context.evaluate_action(State(), action)
assert result.get() == 124
assert result.metadata["commands"][-1] == ["test_callable", "1"]
assert result.metadata["attributes"]["ABC"] == "def"
def test_store(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
assert evaluate("a/b/-/world").get() == "hello world"
def test_clean_recipes(self):
import importlib
from liquer import evaluate
import liquer.ext.basic
import liquer.ext.meta
import liquer.store as st
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.meta)
@first_command
def hello(x):
return f"Hello, {x}"
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- hello-RECIPES/hello1.txt
subdir:
- hello-subdir/hello2.txt
""",
{},
)
store = RecipeSpecStore(substore)
store_backup = st.get_store()
st.set_store(store)
try:
assert store.get_metadata("hello1.txt")["status"] == Status.RECIPE.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value
assert store.get_bytes("hello1.txt") == b"Hello, RECIPES"
assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir"
assert store.get_metadata("hello1.txt")["status"] == Status.READY.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.READY.value
assert evaluate("-R-meta/subdir/-/ns-meta/clean_recipes").get()["removed"] == ["subdir/hello2.txt"]
assert store.get_metadata("hello1.txt")["status"] == Status.READY.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value
finally:
st.set_store(store_backup)
def test_evaluate_query_with_arguments(self):
reset_command_registry()
@command
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
result = get_context().evaluate("test_callable-1")
assert not result.is_volatile()
assert result.get() == 124
result = get_context().evaluate("test_callable-1", extra_parameters=[234])
assert result.is_volatile()
assert result.get() == 235
def test_state_command(self):
reset_command_registry()
@command
def statecommand(state): # has state as a first argument
assert isinstance(state, State)
return 123 + state.get()
assert (
get_context()
.evaluate_action(State().with_data(1), ActionRequest("statecommand"))
.get()
== 124
)
def test_subquery(self):
reset_command_registry()
@first_command
def a():
return 123
@first_command
def b(context):
return context.evaluate("a").get() * 10
state = evaluate("b")
assert state.get() == 1230
assert state.metadata["direct_subqueries"][0]["query"] == "a"
def test_link(self):
reset_command_registry()
@first_command
def value(x):
return int(x)
@command
def add(x, y):
return int(x) + int(y)
assert evaluate("value-1/add-2").get() == 3
assert evaluate("value-1/add-~X~/value-2~E").get() == 3
assert evaluate("value-1/add-~X~add-2~E").get() == 4
def test_link_error(self):
reset_command_registry()
@first_command
def make_error():
raise Exception("Error in make_error")
@command
def concat(x, y=1):
return str(x) + str(y)
assert evaluate("concat-2/concat-3").get() == "None23"
with pytest.raises(Exception):
assert evaluate("concat-4/concat-~X~concat-5/make_error~E").get()