def test_dependencies() -> None:
"""Test cached result."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
cmd2 = p.python_funsie(uncapitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="uncap")
operation = _graph.make_op(
db,
cmd,
inp={"inp": _graph.constant_artefact(db, store, "bla bla")},
opt=opt)
operation2 = _graph.make_op(
db,
cmd2,
inp={"inp": _graph.Artefact.grab(db, operation.out["inp"])},
opt=opt)
status = run_op(db, store, operation2.hash)
assert status == RunStatus.unmet_dependencies
status = run_op(db, store, operation.hash)
assert status == RunStatus.executed
status = run_op(db, store, operation2.hash)
assert status == RunStatus.executed
def test_template() -> None:
"""Basic test of chevron templating."""
with Fun(MockServer()):
db, store = get_connection()
t = "Hello, {{ mustache }}!"
result = template(t, {"mustache": "world"})
run_op(db, store, result.parent)
assert take(result) == b"Hello, world!"
def test_truncate() -> None:
"""Test truncation."""
with Fun(MockServer()):
db, store = get_connection()
inp = "\n".join([f"{k}" for k in range(10)])
dat1 = put(inp.encode())
trunc = utils.truncate(dat1, 2, 3)
run_op(db, store, trunc.parent)
assert take(trunc) == ("\n".join(inp.split("\n")[2:-3])).encode()
def test_wait() -> None:
"""Test waiting on things."""
with Fun(MockServer()):
db, store = _context.get_connection()
s = ui.shell("cp file1 file2", inp={"file1": "wawa"}, out=["file2"])
with pytest.raises(TimeoutError):
ui.wait_for(s.stdout, timeout=0)
run_op(db, store, s.op.hash)
ui.wait_for(s.stdout, timeout=0)
ui.wait_for(s, timeout=0)
def test_shell_run() -> None:
"""Test shell command."""
with Fun(MockServer()):
db, store = _context.get_connection()
s = ui.shell("cp file1 file2", inp={"file1": b"wawa"}, out=["file2"])
run_op(db, store, s.hash)
assert _graph.get_data(db, store, s.stderr) == b""
assert _graph.get_data(db, store, s.returncode) == 0
assert _graph.get_data(db, store, s.inp["file1"]) == b"wawa"
assert _graph.get_data(db, store, s.stdout) == b""
assert ui.take(s.out["file2"]) == b"wawa"
def test_not_generated() -> None:
"""What happens when an artefact is not generated?"""
with Fun(MockServer()):
db, store = get_connection()
s = funsies.shell("cp file1 file2",
inp=dict(file1="bla"),
out=["file3"])
run_op(db, store, s.op.hash)
assert funsies.take(s.returncode) == 0
with pytest.raises(UnwrapError):
funsies.take(s.out["file3"])
def test_morph() -> None:
"""Test store for caching."""
with Fun(MockServer()):
db, store = _context.get_connection()
dat = ui.put(b"bla bla")
morph = fp.morph(lambda x: x.decode().upper().encode(), dat)
run_op(db, store, morph.parent)
assert ui.take(morph) == b"BLA BLA"
dat = ui.put("bla bla")
morph = fp.morph(lambda x: x.upper(), dat, name="CAPITALIZE_THIS")
run_op(db, store, morph.parent)
assert ui.take(morph) == "BLA BLA"
def test_concat() -> None:
"""Test concatenation."""
with Fun(MockServer()):
db, store = get_connection()
dat1 = put(b"bla")
dat2 = put(b"bla")
cat = utils.concat(dat1, dat2)
run_op(db, store, cat.parent)
assert take(cat) == b"blabla"
cat = utils.concat(dat1, dat1, dat1, join=b" ")
run_op(db, store, cat.parent)
assert take(cat) == b"bla bla bla"
def test_error_propagation_morph() -> None:
"""Test propagation of errors."""
with Fun(MockServer()):
db, store = get_connection()
s1 = funsies.shell("cp file1 file3",
inp=dict(file1="bla"),
out=["file2"])
def fun_strict(inp: bytes) -> bytes:
return inp
def fun_lax(inp: Result[bytes]) -> bytes:
return b"bla bla"
s2 = funsies.morph(fun_strict, s1.out["file2"])
s3 = funsies.morph(fun_lax, s1.out["file2"])
s4 = funsies.morph(fun_lax, s1.out["file2"], strict=False)
run_op(db, store, s1.op.hash)
run_op(db, store, s2.parent)
out = funsies.take(s2, strict=False)
assert isinstance(out, Error)
assert out.source == s1.op.hash
print(s3.parent)
run_op(db, store, s3.parent)
out = funsies.take(s3, strict=False)
assert isinstance(out, Error)
assert out.source == s1.op.hash
run_op(db, store, s4.parent)
out = funsies.take(s4)
assert out == b"bla bla"
def test_shell_run2() -> None:
"""Test shell command output side cases."""
with Fun(MockServer()):
db, store = _context.get_connection()
s = ui.shell("cp file1 file2", "cat file2", inp={"file1": b"wawa"})
run_op(db, store, s.hash)
assert _graph.get_data(db, store, s.inp["file1"]) == b"wawa"
with pytest.raises(Exception):
_graph.get_data(db, store, s.stdout)
with pytest.raises(Exception):
_graph.get_data(db, store, s.stderr)
with pytest.raises(Exception):
_graph.get_data(db, store, s.returncode)
assert ui.take(s.stdouts[1]) == b"wawa"
def test_reduce() -> None:
"""Test store for caching."""
with Fun(MockServer()):
db, store = _context.get_connection()
dat = ui.put("bla bla")
morph = fp.morph(lambda x: x.upper(), dat)
def join(x: str, y: str) -> str:
return x + y
red = fp.reduce(join, morph, dat)
run_op(db, store, morph.parent)
run_op(db, store, red.parent)
assert ui.take(red) == "BLA BLAbla bla"
def test_error_propagation() -> None:
"""Test propagation of errors."""
with Fun(MockServer()):
db, store = get_connection()
s1 = funsies.shell("cp file1 file3",
inp=dict(file1="bla"),
out=["file2"])
s2 = funsies.shell("cat file1 file2",
inp=dict(file1="a file", file2=s1.out["file2"]))
run_op(db, store, s1.op.hash)
run_op(db, store, s2.op.hash)
out = funsies.take(s2.stdout, strict=False)
print(out)
assert isinstance(out, Error)
assert out.source == s1.op.hash
def test_cached_run() -> None:
"""Test cached result."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
inp = {"inp": _graph.constant_artefact(db, store, "bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
status = run_op(db, store, operation.hash)
assert status == RunStatus.using_cached
def test_py() -> None:
"""Test multiple output py()."""
with Fun(MockServer()):
db, store = _context.get_connection()
dat = ui.put("Bla Bla")
def fun(a: str) -> Tuple[str, str]:
return a.upper(), a.lower()
x1, x2 = fp.py(
fun,
dat,
out=[Encoding.json, Encoding.json],
strict=True,
)
run_op(db, store, x1.parent)
assert ui.take(x1) == "BLA BLA"
assert ui.take(x2) == "bla bla"
def test_multi_reduce() -> None:
"""Test store for caching."""
with Fun(MockServer()):
db, store = _context.get_connection()
dat = ui.put("bla bla")
morph = fp.morph(lambda x: x.upper(), dat)
def join(*x: str) -> str:
out = ""
for el in x:
out += el
return out
red = fp.reduce(join, morph, dat, "|wat")
run_op(db, store, morph.parent)
run_op(db, store, red.parent)
assert ui.take(red) == "BLA BLAbla bla|wat"
def test_error_propagation_shell() -> None:
"""Test propagation of errors."""
db = Redis()
store = RedisStorage(db)
s1 = funsies.shell(
"cp file1 file3",
inp=dict(file1="bla"),
out=["file2"],
connection=(db, store),
opt=options(),
)
s2 = funsies.shell(
"cat file2",
inp=dict(file2=s1.out["file2"]),
connection=(db, store),
opt=options(),
)
s3 = funsies.shell(
"cat file2",
inp=dict(file2=s1.out["file2"]),
strict=False,
connection=(db, store),
opt=options(),
)
run_op(db, store, s1.op.hash)
run_op(db, store, s2.op.hash)
with pytest.raises(UnwrapError):
funsies.take(s2.stderr, connection=(db, store))
run_op(db, store, s3.op.hash)
assert funsies.take(s3.stderr, connection=(db, store)) != b""
assert isinstance(funsies.take(s3.returncode, connection=(db, store)), int)
assert funsies.take(s3.returncode, connection=(db, store)) != 0
def test_shell_run() -> None:
"""Test run on a shell command."""
with Fun(MockServer()):
db, store = get_connection()
cmd = shell("cat file1", inp={"file1": b"bla bla"}, out=["bla"])
_ = run_op(db, store, cmd.hash)
with tempfile.TemporaryDirectory() as d:
debug.shell(cmd, d)
n = os.listdir(d)
assert "stdout0" in n
assert "stderr0" in n
assert "input_files" in n
assert "output_files" in n
with open(os.path.join(d, "errors.json"), "r") as f:
assert "MissingOutput" in f.read()
def test_subdag() -> None:
"""Test run of a subdag function."""
# funsies
import funsies as f
opt = options()
serv = MockServer()
with f.Fun(serv):
db, store = _context.get_connection()
def map_reduce(inputs: Dict[str, bytes]) -> Dict[str, _graph.Artefact]:
"""Basic map reduce."""
inp_data = inputs["inp"].split(b" ")
for el in inp_data:
out = f.morph(lambda x: x.upper(), el, opt=options())
return {"out": f.utils.concat(out, join="-")}
cmd = sub.subdag_funsie(map_reduce, {"inp": Encoding.blob},
{"out": Encoding.blob})
inp = {"inp": _graph.constant_artefact(db, store, b"bla bla blo lol")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.subdag_ready
# test output data
dat = _graph.get_data(
db,
store,
_graph.Artefact[bytes].grab(db, operation.out["out"]),
do_resolve_link=False,
)
assert isinstance(dat, f.errors.Error)
assert dat.kind == "UnresolvedLink"
datl = _graph.get_data(
db,
store,
_graph.Artefact[bytes].grab(db, operation.out["out"]),
do_resolve_link=True,
)
assert isinstance(datl, f.errors.Error)
assert datl.kind == "NotFound"
def test_shell_run() -> None:
"""Test run on a shell command."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = s.shell_funsie(["cat file1"], {"file1": Encoding.blob}, [])
inp = {"file1": _graph.constant_artefact(db, store, b"bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
# check data is good
dat = _graph.get_data(
db, store, _graph.Artefact[bytes].grab(db, operation.inp["file1"]))
assert dat == b"bla bla"
dat = _graph.get_data(
db, store, _graph.Artefact[bytes].grab(db,
operation.out[f"{s.STDOUT}0"]))
assert dat == b"bla bla"
def test_template_complicated() -> None:
"""Test templating with funky types."""
with Fun(MockServer()):
db, store = get_connection()
t = "wazzaa, {{ mustache }}!"
result = template(t, {"mustache": put(b"people")})
run_op(db, store, result.parent)
assert take(result) == b"wazzaa, people!"
t = "{{a}}{{b}}{{c}}"
result = template(t, dict(a=2, b="cool", c="4me"))
run_op(db, store, result.parent)
assert take(result) == b"2cool4me"
t = ""
result = template(t, dict(a=2, b="cool", c="4me"))
run_op(db, store, result.parent)
assert take(result) == b""
def test_pyfunc_run() -> None:
"""Test run on a python function."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
inp = {"inp": _graph.constant_artefact(db, store, "bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
# check data is good
dat = _graph.get_data(db, store,
_graph.Artefact[str].grab(db, operation.inp["inp"]))
assert dat == "bla bla"
dat = _graph.get_data(db, store,
_graph.Artefact[str].grab(db, operation.out["inp"]))
assert dat == "BLA BLA"
def test_error_tolerant() -> None:
"""Test error tolerant funsie."""
def error_tolerant_fun(inp: Result[bytes]) -> bytes:
if isinstance(inp, Error):
return b"err"
else:
return b""
with Fun(MockServer()):
db, store = get_connection()
s1 = funsies.shell("cp file1 file3",
inp=dict(file1="bla"),
out=["file2"])
s2 = funsies.morph(error_tolerant_fun, s1.out["file2"], strict=False)
with pytest.raises(RuntimeError):
# Test operation not found
run_op(db, store, s2.hash)
run_op(db, store, s1.op)
run_op(db, store, s2.parent)
assert funsies.take(s2) == b"err"