def test_error_message3(self):
import traceback
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
@command
def expected(data, arg):
return f"{data} {arg}"
assert evaluate("a/b/-/world/expected-x").get() == "hello world x"
try:
evaluate("a/b/-/expected").get()
assert False
except Exception as e:
traceback.print_exc()
assert e.query == "a/b/-/expected"
assert e.position.offset == 6
try:
evaluate("a/b/-/expected-x-y").get()
assert False
except Exception as e:
assert e.query == "a/b/-/expected-x-y"
assert e.position.offset == 6
def test_sql_recipe(self):
import liquer.store as st
@first_command
def hello():
return pd.DataFrame(dict(a=[1, 2], b=[3, 4]))
@command
def c(df):
return ",".join(str(x) for x in df.c)
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- filename: hello.parquet
type: parquet_sql
register:
- hello/hello.parquet
sql: SELECT a,b,a+b AS c FROM hello
- hello.parquet/-/dr-dataframe-parquet/c/c.txt
""",
{},
)
store = RecipeSpecStore(substore)
set_store(store)
assert "hello.parquet" in store.keys()
assert "c.txt" in store.keys()
assert store.get_bytes("c.txt") == b"4,6"
assert store.get_metadata(
"hello.parquet")["status"] == Status.READY.value
assert store.get_metadata("hello.parquet")["dependencies"]["recipe"][
"version"] == "md5:c065cb611597fc0e953e5ad72a769db8"
def setup_class(cls):
import importlib
from liquer.commands import reset_command_registry
import liquer.ext.basic
import liquer.ext.meta
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.meta)
set_store(MemoryStore())
def test_image(self):
store = MemoryStore()
set_store(store)
@first_command
def image():
return Image.new(mode="RGB", size=(200, 300))
assert evaluate("image").get().size == (200,300)
evaluate_and_save("image/ns-pil/resize-400-600-bilinear/test.png",target_resource_directory="x")
image = evaluate("x/test.png/-/dr").get()
assert image.size == (400,600)
def test_concat_recipe(self):
import pandas as pd
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.lq_pandas)
import liquer.store as st
@first_command
def hello(offset=0):
return pd.DataFrame(
dict(a=[1 + offset, 2 + offset], b=[3 + offset, 4 + offset]))
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- filename: hello.parquet
type: pandas_concat
concat:
- hello
- query: hello-10
column: test
value: extra
""",
{},
)
store = RecipeSpecStore(substore)
set_store(store)
assert "hello.parquet" in store.keys()
df = evaluate("hello.parquet/-/dr").get()
assert sorted(df.columns) == ["a", "b", "test"]
assert list(df.a) == [1, 2, 11, 12]
assert list(df.b) == [3, 4, 13, 14]
assert list(df.test) == [None, None, "extra", "extra"]
assert store.get_metadata(
"hello.parquet")["status"] == Status.READY.value
assert store.get_metadata(
"hello.parquet")["recipes_key"] == "recipes.yaml"
assert store.get_metadata("hello.parquet")["has_recipe"] == True
assert store.get_metadata("hello.parquet")["recipes_directory"] == ""
assert store.get_metadata("hello.parquet")[
"recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet"
assert store.get_metadata("hello.parquet")["data_characteristics"][
"description"] == "Dataframe with 3 columns and 4 rows."
set_store(None)
reset_command_registry()
def test_workbook(self):
store = MemoryStore()
set_store(store)
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv")
evaluate_and_save(f"df_from-{filename}/test.xlsx",
target_resource_directory="testdir")
df = evaluate("testdir/test.xlsx/-/workbook/workbook_sheet_df").get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1, 3]
assert list(df.b) == [2, 4]
def test_store(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
assert evaluate("a/b/-/world").get() == "hello world"
def test_clean_recipes(self):
import importlib
from liquer import evaluate
import liquer.ext.basic
import liquer.ext.meta
import liquer.store as st
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.meta)
@first_command
def hello(x):
return f"Hello, {x}"
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- hello-RECIPES/hello1.txt
subdir:
- hello-subdir/hello2.txt
""",
{},
)
store = RecipeSpecStore(substore)
store_backup = st.get_store()
st.set_store(store)
try:
assert store.get_metadata("hello1.txt")["status"] == Status.RECIPE.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value
assert store.get_bytes("hello1.txt") == b"Hello, RECIPES"
assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir"
assert store.get_metadata("hello1.txt")["status"] == Status.READY.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.READY.value
assert evaluate("-R-meta/subdir/-/ns-meta/clean_recipes").get()["removed"] == ["subdir/hello2.txt"]
assert store.get_metadata("hello1.txt")["status"] == Status.READY.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value
finally:
st.set_store(store_backup)
def test_store_evaluate_and_save1(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
evaluate_and_save("a/b/-/world/hello.txt",
target_resource_directory="results")
assert store.get_bytes("results/hello.txt") == b"hello world"
def test_get_stored_metadata(self):
from liquer.store import set_store, MemoryStore
from liquer.cache import set_cache, MemoryCache, NoCache
store = MemoryStore()
store.store_metadata("a/b", dict(test="stored value 1"))
set_store(store)
cache = MemoryCache()
cache.store_metadata(dict(query="c/d", test="stored value 2"))
set_cache(cache)
assert get_stored_metadata("-R/a/b")["test"] == "stored value 1"
assert get_stored_metadata("a/b") is None # this represents a query
assert get_stored_metadata("c/d")["test"] == "stored value 2"
set_store(None)
set_cache(NoCache())
def test_resource_link(self, tmpdir):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
@first_command
def value(x):
return f"<{x}>"
assert evaluate("a/b/-/world").get() == "hello world"
assert evaluate("-R/a/b/-/world").get() == "hello world"
assert evaluate("value-~X~-R/a/b/-/world~E").get() == "<hello world>"
def test_recipe_error_in_query_metadata(self):
import liquer.store as st
import liquer.recipes as r
from liquer.cache import MemoryCache, set_cache, get_cache
reset_command_registry()
set_cache(None)
@first_command
def hello():
raise Exception("Hello error")
@command
def world(x):
return str(x) + "world"
store = r.RecipeSpecStore(st.MemoryStore())
set_store(store)
store.store(
"results/recipes.yaml",
b"""
subdir:
- hello/hello.txt
""",
{},
)
assert "results/subdir/hello.txt" in store.recipes()
try:
assert store.get_bytes("results/subdir/hello.txt")
except KeyNotFoundStoreException:
pass
assert store.get_metadata("results/subdir/hello.txt")["is_error"]
assert store.get_metadata(
"results/subdir/hello.txt")["log"][-1]["message"] == "Hello error"
child_messages = [
x["message"]
for x in evaluate("results/subdir/hello.txt/-/world/hello.txt").
metadata["child_log"]
]
print(child_messages)
assert "Hello error" in child_messages
set_store(None)
reset_command_registry()
def test_pptx(self):
store = MemoryStore()
set_store(store)
@first_command
def make_presentation():
prs = Presentation()
title_slide_layout = prs.slide_layouts[0]
slide = prs.slides.add_slide(title_slide_layout)
title = slide.shapes.title
subtitle = slide.placeholders[1]
title.text = "Hello, World!"
subtitle.text = "python-pptx was here!"
return prs
@command
def add_slide(prs, title="Title"):
bullet_slide_layout = prs.slide_layouts[1]
slide = prs.slides.add_slide(bullet_slide_layout)
shapes = slide.shapes
title_shape = shapes.title
body_shape = shapes.placeholders[1]
title_shape.text = title
tf = body_shape.text_frame
tf.text = 'Find the bullet slide layout'
p = tf.add_paragraph()
p.text = 'Use _TextFrame.text for first bullet'
p.level = 1
return prs
evaluate_and_save(f"make_presentation/add_slide/test.pptx",
target_resource_directory="testdir")
assert store.get_metadata(
"testdir/test.pptx")["type_identifier"] == "pptx_presentation"
prs = evaluate("testdir/test.pptx/-/dr/add_slide-Slide2").get()
assert len(prs.slides) == 3
def test_sql_error(self):
import liquer.store as st
@first_command
def hello1():
return pd.DataFrame(dict(a=[1, 2], b=[3, 4]))
@command
def c1(df):
return ",".join(str(x) for x in df.c)
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- filename: hello.parquet
type: parquet_sql
register:
- hello1/hello.parquet
sql: SELECT xxx,b,a+b AS c FROM hello
- hello.parquet/-/dr-dataframe-parquet/c1/c.txt
""",
{},
)
store = RecipeSpecStore(substore)
set_store(store)
assert "hello.parquet" in store.keys()
assert "c.txt" in store.keys()
try:
store.get_bytes("c.txt")
except:
pass
assert store.get_metadata(
"hello.parquet")["status"] == Status.ERROR.value
assert store.get_metadata(
"hello.parquet")["recipes_key"] == "recipes.yaml"
assert store.get_metadata("hello.parquet")["has_recipe"] == True
assert store.get_metadata("hello.parquet")["recipes_directory"] == ""
assert store.get_metadata("hello.parquet")[
"recipe_name"] == "recipes.yaml/-Ryaml/RECIPES/0#hello.parquet"
def test_dr(self):
import pandas as pd
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.lq_pandas)
store = MemoryStore()
set_store(store)
store.store("data.csv", b"a,b\n1,2\n3,4", {})
df = evaluate("data.csv/-/dr").get()
assert isinstance(df, pd.DataFrame)
assert len(df) == 2
assert len(df.columns) == 2
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1, 3]
assert list(df.b) == [2, 4]
def test_error_message1(self):
import traceback
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
assert evaluate("a/b/-/world").get() == "hello world"
try:
evaluate("a/b/-/undefined").get()
assert False
except Exception as e:
assert e.query == "a/b/-/undefined"
assert e.position.offset == 6
def test_meta_store(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {"x": 123})
@command
def world(data):
return data.decode("utf-8") + " world"
@command
def get_x(metadata):
return metadata.get("x")
assert evaluate("-R/a/b/-/world").get() == "hello world"
assert evaluate("-R-meta/a/b/-/get_x").get() == 123
print(evaluate("-R-meta/a/b").get())
assert evaluate("-R-meta/a/b").get()["key"] == "a/b"
def teardown_class(cls):
from liquer.commands import reset_command_registry
reset_command_registry()
set_store(None)
