def test_error_message3(self): import traceback import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" @command def expected(data, arg): return f"{data} {arg}" assert evaluate("a/b/-/world/expected-x").get() == "hello world x" try: evaluate("a/b/-/expected").get() assert False except Exception as e: traceback.print_exc() assert e.query == "a/b/-/expected" assert e.position.offset == 6 try: evaluate("a/b/-/expected-x-y").get() assert False except Exception as e: assert e.query == "a/b/-/expected-x-y" assert e.position.offset == 6
def test_sql_recipe(self): import liquer.store as st @first_command def hello(): return pd.DataFrame(dict(a=[1, 2], b=[3, 4])) @command def c(df): return ",".join(str(x) for x in df.c) substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - filename: hello.parquet type: parquet_sql register: - hello/hello.parquet sql: SELECT a,b,a+b AS c FROM hello - hello.parquet/-/dr-dataframe-parquet/c/c.txt """, {}, ) store = RecipeSpecStore(substore) set_store(store) assert "hello.parquet" in store.keys() assert "c.txt" in store.keys() assert store.get_bytes("c.txt") == b"4,6" assert store.get_metadata( "hello.parquet")["status"] == Status.READY.value assert store.get_metadata("hello.parquet")["dependencies"]["recipe"][ "version"] == "md5:c065cb611597fc0e953e5ad72a769db8"
def setup_class(cls): import importlib from liquer.commands import reset_command_registry import liquer.ext.basic import liquer.ext.meta reset_command_registry() # prevent double-registration # Hack to enforce registering of the commands importlib.reload(liquer.ext.basic) importlib.reload(liquer.ext.meta) set_store(MemoryStore())
def test_image(self): store = MemoryStore() set_store(store) @first_command def image(): return Image.new(mode="RGB", size=(200, 300)) assert evaluate("image").get().size == (200,300) evaluate_and_save("image/ns-pil/resize-400-600-bilinear/test.png",target_resource_directory="x") image = evaluate("x/test.png/-/dr").get() assert image.size == (400,600)
def test_concat_recipe(self): import pandas as pd import liquer.ext.basic from liquer.commands import reset_command_registry reset_command_registry() # prevent double-registration importlib.reload(liquer.ext.basic) importlib.reload(liquer.ext.lq_pandas) import liquer.store as st @first_command def hello(offset=0): return pd.DataFrame( dict(a=[1 + offset, 2 + offset], b=[3 + offset, 4 + offset])) substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - filename: hello.parquet type: pandas_concat concat: - hello - query: hello-10 column: test value: extra """, {}, ) store = RecipeSpecStore(substore) set_store(store) assert "hello.parquet" in store.keys() df = evaluate("hello.parquet/-/dr").get() assert sorted(df.columns) == ["a", "b", "test"] assert list(df.a) == [1, 2, 11, 12] assert list(df.b) == [3, 4, 13, 14] assert list(df.test) == [None, None, "extra", "extra"] assert store.get_metadata( "hello.parquet")["status"] == Status.READY.value assert store.get_metadata( "hello.parquet")["recipes_key"] == "recipes.yaml" assert store.get_metadata("hello.parquet")["has_recipe"] == True assert store.get_metadata("hello.parquet")["recipes_directory"] == "" assert store.get_metadata("hello.parquet")[ "recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet" assert store.get_metadata("hello.parquet")["data_characteristics"][ "description"] == "Dataframe with 3 columns and 4 rows." set_store(None) reset_command_registry()
def test_workbook(self): store = MemoryStore() set_store(store) filename = encode_token( os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv") evaluate_and_save(f"df_from-{filename}/test.xlsx", target_resource_directory="testdir") df = evaluate("testdir/test.xlsx/-/workbook/workbook_sheet_df").get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1, 3] assert list(df.b) == [2, 4]
def test_store(self): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" assert evaluate("a/b/-/world").get() == "hello world"
def test_clean_recipes(self): import importlib from liquer import evaluate import liquer.ext.basic import liquer.ext.meta import liquer.store as st from liquer.commands import reset_command_registry reset_command_registry() # prevent double-registration # Hack to enforce registering of the commands importlib.reload(liquer.ext.basic) importlib.reload(liquer.ext.meta) @first_command def hello(x): return f"Hello, {x}" substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - hello-RECIPES/hello1.txt subdir: - hello-subdir/hello2.txt """, {}, ) store = RecipeSpecStore(substore) store_backup = st.get_store() st.set_store(store) try: assert store.get_metadata("hello1.txt")["status"] == Status.RECIPE.value assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value assert store.get_bytes("hello1.txt") == b"Hello, RECIPES" assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir" assert store.get_metadata("hello1.txt")["status"] == Status.READY.value assert store.get_metadata("subdir/hello2.txt")["status"] == Status.READY.value assert evaluate("-R-meta/subdir/-/ns-meta/clean_recipes").get()["removed"] == ["subdir/hello2.txt"] assert store.get_metadata("hello1.txt")["status"] == Status.READY.value assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value finally: st.set_store(store_backup)
def test_store_evaluate_and_save1(self): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" evaluate_and_save("a/b/-/world/hello.txt", target_resource_directory="results") assert store.get_bytes("results/hello.txt") == b"hello world"
def test_get_stored_metadata(self): from liquer.store import set_store, MemoryStore from liquer.cache import set_cache, MemoryCache, NoCache store = MemoryStore() store.store_metadata("a/b", dict(test="stored value 1")) set_store(store) cache = MemoryCache() cache.store_metadata(dict(query="c/d", test="stored value 2")) set_cache(cache) assert get_stored_metadata("-R/a/b")["test"] == "stored value 1" assert get_stored_metadata("a/b") is None # this represents a query assert get_stored_metadata("c/d")["test"] == "stored value 2" set_store(None) set_cache(NoCache())
def test_resource_link(self, tmpdir): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" @first_command def value(x): return f"<{x}>" assert evaluate("a/b/-/world").get() == "hello world" assert evaluate("-R/a/b/-/world").get() == "hello world" assert evaluate("value-~X~-R/a/b/-/world~E").get() == "<hello world>"
def test_recipe_error_in_query_metadata(self): import liquer.store as st import liquer.recipes as r from liquer.cache import MemoryCache, set_cache, get_cache reset_command_registry() set_cache(None) @first_command def hello(): raise Exception("Hello error") @command def world(x): return str(x) + "world" store = r.RecipeSpecStore(st.MemoryStore()) set_store(store) store.store( "results/recipes.yaml", b""" subdir: - hello/hello.txt """, {}, ) assert "results/subdir/hello.txt" in store.recipes() try: assert store.get_bytes("results/subdir/hello.txt") except KeyNotFoundStoreException: pass assert store.get_metadata("results/subdir/hello.txt")["is_error"] assert store.get_metadata( "results/subdir/hello.txt")["log"][-1]["message"] == "Hello error" child_messages = [ x["message"] for x in evaluate("results/subdir/hello.txt/-/world/hello.txt"). metadata["child_log"] ] print(child_messages) assert "Hello error" in child_messages set_store(None) reset_command_registry()
def test_pptx(self): store = MemoryStore() set_store(store) @first_command def make_presentation(): prs = Presentation() title_slide_layout = prs.slide_layouts[0] slide = prs.slides.add_slide(title_slide_layout) title = slide.shapes.title subtitle = slide.placeholders[1] title.text = "Hello, World!" subtitle.text = "python-pptx was here!" return prs @command def add_slide(prs, title="Title"): bullet_slide_layout = prs.slide_layouts[1] slide = prs.slides.add_slide(bullet_slide_layout) shapes = slide.shapes title_shape = shapes.title body_shape = shapes.placeholders[1] title_shape.text = title tf = body_shape.text_frame tf.text = 'Find the bullet slide layout' p = tf.add_paragraph() p.text = 'Use _TextFrame.text for first bullet' p.level = 1 return prs evaluate_and_save(f"make_presentation/add_slide/test.pptx", target_resource_directory="testdir") assert store.get_metadata( "testdir/test.pptx")["type_identifier"] == "pptx_presentation" prs = evaluate("testdir/test.pptx/-/dr/add_slide-Slide2").get() assert len(prs.slides) == 3
def test_sql_error(self): import liquer.store as st @first_command def hello1(): return pd.DataFrame(dict(a=[1, 2], b=[3, 4])) @command def c1(df): return ",".join(str(x) for x in df.c) substore = st.MemoryStore() substore.store( "recipes.yaml", """ RECIPES: - filename: hello.parquet type: parquet_sql register: - hello1/hello.parquet sql: SELECT xxx,b,a+b AS c FROM hello - hello.parquet/-/dr-dataframe-parquet/c1/c.txt """, {}, ) store = RecipeSpecStore(substore) set_store(store) assert "hello.parquet" in store.keys() assert "c.txt" in store.keys() try: store.get_bytes("c.txt") except: pass assert store.get_metadata( "hello.parquet")["status"] == Status.ERROR.value assert store.get_metadata( "hello.parquet")["recipes_key"] == "recipes.yaml" assert store.get_metadata("hello.parquet")["has_recipe"] == True assert store.get_metadata("hello.parquet")["recipes_directory"] == "" assert store.get_metadata("hello.parquet")[ "recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet"
def test_dr(self): import pandas as pd import liquer.ext.basic from liquer.commands import reset_command_registry reset_command_registry() # prevent double-registration importlib.reload(liquer.ext.basic) importlib.reload(liquer.ext.lq_pandas) store = MemoryStore() set_store(store) store.store("data.csv", b"a,b\n1,2\n3,4", {}) df = evaluate("data.csv/-/dr").get() assert isinstance(df, pd.DataFrame) assert len(df) == 2 assert len(df.columns) == 2 assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1, 3] assert list(df.b) == [2, 4]
def test_error_message1(self): import traceback import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {}) @command def world(data): return data.decode("utf-8") + " world" assert evaluate("a/b/-/world").get() == "hello world" try: evaluate("a/b/-/undefined").get() assert False except Exception as e: assert e.query == "a/b/-/undefined" assert e.position.offset == 6
def test_meta_store(self): import liquer.store as st reset_command_registry() st.set_store(st.MemoryStore()) store = st.get_store() store.store("a/b", b"hello", {"x": 123}) @command def world(data): return data.decode("utf-8") + " world" @command def get_x(metadata): return metadata.get("x") assert evaluate("-R/a/b/-/world").get() == "hello world" assert evaluate("-R-meta/a/b/-/get_x").get() == 123 print(evaluate("-R-meta/a/b").get()) assert evaluate("-R-meta/a/b").get()["key"] == "a/b"
def teardown_class(cls): from liquer.commands import reset_command_registry reset_command_registry() set_store(None)