def test_error_message3(self): import traceback import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" @command def expected(data, arg): return f"{data} {arg}" assert evaluate("a/b/-/world/expected-x").get() == "hello world x" try: evaluate("a/b/-/expected").get() assert False except Exception as e: traceback.print_exc() assert e.query == "a/b/-/expected" assert e.position.offset == 6 try: evaluate("a/b/-/expected-x-y").get() assert False except Exception as e: assert e.query == "a/b/-/expected-x-y" assert e.position.offset == 6
def test_sql_recipe(self): import liquer.store as st @first_command def hello(): return pd.DataFrame(dict(a=[1, 2], b=[3, 4])) @command def c(df): return ",".join(str(x) for x in df.c) substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - filename: hello.parquet type: parquet_sql register: - hello/hello.parquet sql: SELECT a,b,a+b AS c FROM hello - hello.parquet/-/dr-dataframe-parquet/c/c.txt """, {}, ) store = RecipeSpecStore(substore) set_store(store) assert "hello.parquet" in store.keys() assert "c.txt" in store.keys() assert store.get_bytes("c.txt") == b"4,6" assert store.get_metadata( "hello.parquet")["status"] == Status.READY.value assert store.get_metadata("hello.parquet")["dependencies"]["recipe"][ "version"] == "md5:c065cb611597fc0e953e5ad72a769db8"
def test_recipe_spec_store(self): import liquer.store as st import liquer.recipes as r from liquer.cache import MemoryCache, set_cache, get_cache reset_command_registry() set_cache(None) @first_command def hello(): return "Hello" store = r.RecipeSpecStore(st.MemoryStore()) store.store( "results/recipes.yaml", b""" subdir: - hello/hello.txt """, {}, ) assert "results/subdir/hello.txt" in store.recipes() assert store.contains("results") assert store.contains("results/subdir") assert store.contains("results/subdir/hello.txt") assert store.is_dir("results") assert store.is_dir("results/subdir") assert not store.is_dir("results/subdir/hello.txt") assert store.get_bytes("results/subdir/hello.txt") == b"Hello"
def test_status(self): import liquer.store as st reset_command_registry() @first_command def hello(x): return f"Hello, {x}" substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - hello-RECIPES/hello1.txt subdir: - hello-subdir/hello2.txt """, {}, ) store = RecipeSpecStore(substore) assert "hello1.txt" in store.get_bytes(store.STATUS_FILE).decode("utf-8") assert "recipe " in store.get_bytes(store.STATUS_FILE).decode("utf-8") assert store.get_bytes("hello1.txt") == b"Hello, RECIPES" assert "hello1.txt" in store.get_bytes(store.STATUS_FILE).decode("utf-8") assert "ready " in store.get_bytes(store.STATUS_FILE).decode("utf-8") assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir" assert "hello2.txt" in store.get_bytes("subdir/"+store.STATUS_FILE).decode("utf-8") assert "ready " in store.get_bytes("subdir/"+store.STATUS_FILE).decode("utf-8")
def test_storing(self): import liquer.store as st store = st.MemoryStore() m = StoreSyncMetadata(store, "a/b") assert store.get_metadata("a/b")["key"] == "a/b" assert "Hello" not in [ x["message"] for x in store.get_metadata("a/b")["log"] ] m.info("Hello") assert "Hello" in [ x["message"] for x in store.get_metadata("a/b")["log"] ]
def test_concat_recipe(self): import pandas as pd import liquer.ext.basic from liquer.commands import reset_command_registry reset_command_registry() # prevent double-registration importlib.reload(liquer.ext.basic) importlib.reload(liquer.ext.lq_pandas) import liquer.store as st @first_command def hello(offset=0): return pd.DataFrame( dict(a=[1 + offset, 2 + offset], b=[3 + offset, 4 + offset])) substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - filename: hello.parquet type: pandas_concat concat: - hello - query: hello-10 column: test value: extra """, {}, ) store = RecipeSpecStore(substore) set_store(store) assert "hello.parquet" in store.keys() df = evaluate("hello.parquet/-/dr").get() assert sorted(df.columns) == ["a", "b", "test"] assert list(df.a) == [1, 2, 11, 12] assert list(df.b) == [3, 4, 13, 14] assert list(df.test) == [None, None, "extra", "extra"] assert store.get_metadata( "hello.parquet")["status"] == Status.READY.value assert store.get_metadata( "hello.parquet")["recipes_key"] == "recipes.yaml" assert store.get_metadata("hello.parquet")["has_recipe"] == True assert store.get_metadata("hello.parquet")["recipes_directory"] == "" assert store.get_metadata("hello.parquet")[ "recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet" assert store.get_metadata("hello.parquet")["data_characteristics"][ "description"] == "Dataframe with 3 columns and 4 rows." set_store(None) reset_command_registry()
def test_store(self): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" assert evaluate("a/b/-/world").get() == "hello world"
def test_clean_recipes(self): import importlib from liquer import evaluate import liquer.ext.basic import liquer.ext.meta import liquer.store as st from liquer.commands import reset_command_registry reset_command_registry() # prevent double-registration # Hack to enforce registering of the commands importlib.reload(liquer.ext.basic) importlib.reload(liquer.ext.meta) @first_command def hello(x): return f"Hello, {x}" substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - hello-RECIPES/hello1.txt subdir: - hello-subdir/hello2.txt """, {}, ) store = RecipeSpecStore(substore) store_backup = st.get_store() st.set_store(store) try: assert store.get_metadata("hello1.txt")["status"] == Status.RECIPE.value assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value assert store.get_bytes("hello1.txt") == b"Hello, RECIPES" assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir" assert store.get_metadata("hello1.txt")["status"] == Status.READY.value assert store.get_metadata("subdir/hello2.txt")["status"] == Status.READY.value assert evaluate("-R-meta/subdir/-/ns-meta/clean_recipes").get()["removed"] == ["subdir/hello2.txt"] assert store.get_metadata("hello1.txt")["status"] == Status.READY.value assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value finally: st.set_store(store_backup)
def test_store_evaluate_and_save1(self): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" evaluate_and_save("a/b/-/world/hello.txt", target_resource_directory="results") assert store.get_bytes("results/hello.txt") == b"hello world"
def test_recipe_store(self): import liquer.store as st import liquer.recipes as r from liquer.cache import MemoryCache, set_cache, get_cache reset_command_registry() set_cache(None) @first_command def hello(): return "Hello" store = r.RecipeStore(st.MemoryStore()) store.mount_recipe("my/hello.txt", "hello") assert store.contains("my/hello.txt") assert store.get_bytes("my/hello.txt") == b"Hello"
def test_recipe_error_in_query_metadata(self): import liquer.store as st import liquer.recipes as r from liquer.cache import MemoryCache, set_cache, get_cache reset_command_registry() set_cache(None) @first_command def hello(): raise Exception("Hello error") @command def world(x): return str(x) + "world" store = r.RecipeSpecStore(st.MemoryStore()) set_store(store) store.store( "results/recipes.yaml", b""" subdir: - hello/hello.txt """, {}, ) assert "results/subdir/hello.txt" in store.recipes() try: assert store.get_bytes("results/subdir/hello.txt") except KeyNotFoundStoreException: pass assert store.get_metadata("results/subdir/hello.txt")["is_error"] assert store.get_metadata( "results/subdir/hello.txt")["log"][-1]["message"] == "Hello error" child_messages = [ x["message"] for x in evaluate("results/subdir/hello.txt/-/world/hello.txt"). metadata["child_log"] ] print(child_messages) assert "Hello error" in child_messages set_store(None) reset_command_registry()
def test_resource_link(self, tmpdir): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" @first_command def value(x): return f"<{x}>" assert evaluate("a/b/-/world").get() == "hello world" assert evaluate("-R/a/b/-/world").get() == "hello world" assert evaluate("value-~X~-R/a/b/-/world~E").get() == "<hello world>"
def test_sql_error(self): import liquer.store as st @first_command def hello1(): return pd.DataFrame(dict(a=[1, 2], b=[3, 4])) @command def c1(df): return ",".join(str(x) for x in df.c) substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - filename: hello.parquet type: parquet_sql register: - hello1/hello.parquet sql: SELECT xxx,b,a+b AS c FROM hello - hello.parquet/-/dr-dataframe-parquet/c1/c.txt """, {}, ) store = RecipeSpecStore(substore) set_store(store) assert "hello.parquet" in store.keys() assert "c.txt" in store.keys() try: store.get_bytes("c.txt") except: pass assert store.get_metadata( "hello.parquet")["status"] == Status.ERROR.value assert store.get_metadata( "hello.parquet")["recipes_key"] == "recipes.yaml" assert store.get_metadata("hello.parquet")["has_recipe"] == True assert store.get_metadata("hello.parquet")["recipes_directory"] == "" assert store.get_metadata("hello.parquet")[ "recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet"
def test_ignore(self): import liquer.store as st reset_command_registry() @first_command def hello(x): return f"Hello, {x}" substore = st.MemoryStore() substore.store(".a/recipes.yaml", "", {}) substore.store("a/b", "", {}) substore.store("a/.b", "", {}) substore.store(".a/b", "", {}) substore.store(".a/.b", "", {}) store = RecipeSpecStore(substore) assert "a/b" in store.keys() assert "a/.b" not in store.keys() assert ".a/b" not in store.keys() assert ".a/.b" not in store.keys()
def test_error_message1(self): import traceback import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" assert evaluate("a/b/-/world").get() == "hello world" try: evaluate("a/b/-/undefined").get() assert False except Exception as e: assert e.query == "a/b/-/undefined" assert e.position.offset == 6
def test_meta_store(self): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {"x": 123}) @command def world(data): return data.decode("utf-8") + " world" @command def get_x(metadata): return metadata.get("x") assert evaluate("-R/a/b/-/world").get() == "hello world" assert evaluate("-R-meta/a/b/-/get_x").get() == 123 print(evaluate("-R-meta/a/b").get()) assert evaluate("-R-meta/a/b").get()["key"] == "a/b"
def test_recipes(self): import liquer.store as st reset_command_registry() @first_command def hello(x): return f"Hello, {x}" substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - hello-RECIPES/hello1.txt subdir: - hello-subdir/hello2.txt """, {}, ) substore.store( "x/recipes.yaml", """ RECIPES: - hello-RECIPES_X/hello3.txt subdir: - hello-subdir_x/hello4.txt """, {}, ) store = RecipeSpecStore(substore) assert "hello1.txt" in store.keys() assert "subdir/hello2.txt" in store.keys() assert "x/hello3.txt" in store.keys() assert "x/subdir/hello4.txt" in store.keys() assert store.get_bytes("hello1.txt") == b"Hello, RECIPES" assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir" assert store.get_bytes("x/hello3.txt") == b"Hello, RECIPES_X" assert store.get_bytes("x/subdir/hello4.txt") == b"Hello, subdir_x"
def test_recipes_advanced(self): import liquer.store as st import liquer.constants as consts reset_command_registry() @first_command def hello(x): return f"Hello, {x}" @first_command def error(): raise Exception("Error") substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - query: hello-RECIPES/hello1.txt title: "Hello 1" description: "This is hello 1." - query: error/error.txt title: "Error example" description: "Should fail." subdir: - query: hello-subdir/hello.txt filename: hello2.txt title: "Hello 2" description: "This is hello 2." """, {}, ) store = RecipeSpecStore(substore) assert "hello1.txt" in store.keys() assert "subdir/hello2.txt" in store.keys() assert store.get_metadata("hello1.txt")["status"] == consts.Status.RECIPE.value assert store.get_metadata("hello1.txt")["title"] == "Hello 1" assert store.get_metadata("hello1.txt")["description"] == "This is hello 1." assert store.get_bytes("hello1.txt") == b"Hello, RECIPES" assert store.get_metadata("hello1.txt")["status"] == consts.Status.READY.value assert store.get_metadata("hello1.txt")["title"] == "Hello 1" assert store.get_metadata("hello1.txt")["description"] == "This is hello 1." assert store.get_metadata("error.txt")["status"] == consts.Status.RECIPE.value try: assert store.get_bytes("error.txt") == b"Hello, RECIPES" except: pass print(store.get_metadata("error.txt")) assert store.get_metadata("error.txt")["status"] == consts.Status.ERROR.value assert store.get_metadata("subdir/hello2.txt")["status"] == consts.Status.RECIPE.value assert store.get_metadata("subdir/hello2.txt")["title"] == "Hello 2" assert ( store.get_metadata("subdir/hello2.txt")["description"] == "This is hello 2." ) assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir" assert store.get_metadata("subdir/hello2.txt")["status"] == consts.Status.READY.value assert store.get_metadata("subdir/hello2.txt")["title"] == "Hello 2" assert ( store.get_metadata("subdir/hello2.txt")["description"] == "This is hello 2." )