def test_operation_pack() -> None:
"""Test packing and unpacking of operations."""
opt = options()
server = MockServer()
db, store = server.new_connection()
a = _graph.constant_artefact(db, store, b"bla bla")
b = _graph.constant_artefact(db, store, b"bla bla bla")
fun = f.Funsie(
how=f.FunsieHow.shell,
what="cat infile",
inp={"infile": Encoding.blob},
out={"out": Encoding.json},
extra={},
)
op = _graph.make_op(db, fun, {"infile": a}, opt)
op2 = _graph.Operation.grab(db, op.hash)
assert op == op2
with pytest.raises(AttributeError):
op = _graph.make_op(db, fun, {}, opt)
with pytest.raises(AttributeError):
# no inputs
op = _graph.make_op(db, fun, {}, opt)
with pytest.raises(AttributeError):
# too many inputs
op = _graph.make_op(db, fun, {"infile": a, "infile2": b}, opt)
with pytest.raises(RuntimeError):
op = _graph.Operation.grab(db, hash_t("b"))
def test_dependencies() -> None:
"""Test cached result."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
cmd2 = p.python_funsie(uncapitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="uncap")
operation = _graph.make_op(
db,
cmd,
inp={"inp": _graph.constant_artefact(db, store, "bla bla")},
opt=opt)
operation2 = _graph.make_op(
db,
cmd2,
inp={"inp": _graph.Artefact.grab(db, operation.out["inp"])},
opt=opt)
status = run_op(db, store, operation2.hash)
assert status == RunStatus.unmet_dependencies
status = run_op(db, store, operation.hash)
assert status == RunStatus.executed
status = run_op(db, store, operation2.hash)
assert status == RunStatus.executed
def test_subdag() -> None:
"""Test that subdags execute properly."""
def cap(inp: bytes) -> bytes:
return inp.upper()
def map_reduce(
inputs: dict[str, bytes]) -> dict[str, _graph.Artefact[bytes]]:
"""Basic map reduce."""
inp_data = inputs["inp"].split(b" ")
out: list[_graph.Artefact[bytes]] = []
for el in inp_data:
out += [morph(cap, el, opt=options(distributed=False))]
return {"out": concat(*out, join="-")}
with Fun(MockServer(), defaults=options(distributed=False)) as db:
dat = put(b"bla bla lol what")
inp = {"inp": dat}
cmd = _subdag.subdag_funsie(map_reduce, {"inp": Encoding.blob},
{"out": Encoding.blob})
operation = _graph.make_op(db, cmd, inp, options())
out = _graph.Artefact[bytes].grab(db, operation.out["out"])
final = shell(
"cat file1 file2",
inp=dict(file1=out, file2=b"something"),
)
execute(final)
data = take(final.stdout)
assert data == b"BLA-BLA-LOL-WHATsomething"
def test_cached_instances() -> None:
"""Test cached result from running same code twice."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
inp = {"inp": _graph.constant_artefact(db, store, "bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
inp = {"inp": _graph.constant_artefact(db, store, "bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
assert status == RunStatus.using_cached
def test_subdag() -> None:
"""Test run of a subdag function."""
# funsies
import funsies as f
opt = options()
serv = MockServer()
with f.Fun(serv):
db, store = _context.get_connection()
def map_reduce(inputs: Dict[str, bytes]) -> Dict[str, _graph.Artefact]:
"""Basic map reduce."""
inp_data = inputs["inp"].split(b" ")
for el in inp_data:
out = f.morph(lambda x: x.upper(), el, opt=options())
return {"out": f.utils.concat(out, join="-")}
cmd = sub.subdag_funsie(map_reduce, {"inp": Encoding.blob},
{"out": Encoding.blob})
inp = {"inp": _graph.constant_artefact(db, store, b"bla bla blo lol")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.subdag_ready
# test output data
dat = _graph.get_data(
db,
store,
_graph.Artefact[bytes].grab(db, operation.out["out"]),
do_resolve_link=False,
)
assert isinstance(dat, f.errors.Error)
assert dat.kind == "UnresolvedLink"
datl = _graph.get_data(
db,
store,
_graph.Artefact[bytes].grab(db, operation.out["out"]),
do_resolve_link=True,
)
assert isinstance(datl, f.errors.Error)
assert datl.kind == "NotFound"
def test_shell_run() -> None:
"""Test run on a shell command."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = s.shell_funsie(["cat file1"], {"file1": Encoding.blob}, [])
inp = {"file1": _graph.constant_artefact(db, store, b"bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
# check data is good
dat = _graph.get_data(
db, store, _graph.Artefact[bytes].grab(db, operation.inp["file1"]))
assert dat == b"bla bla"
dat = _graph.get_data(
db, store, _graph.Artefact[bytes].grab(db,
operation.out[f"{s.STDOUT}0"]))
assert dat == b"bla bla"
def test_pyfunc_run() -> None:
"""Test run on a python function."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
inp = {"inp": _graph.constant_artefact(db, store, "bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
# check data is good
dat = _graph.get_data(db, store,
_graph.Artefact[str].grab(db, operation.inp["inp"]))
assert dat == "bla bla"
dat = _graph.get_data(db, store,
_graph.Artefact[str].grab(db, operation.out["inp"]))
assert dat == "BLA BLA"
