def test_artefact_wrong_type() -> None:
"""Test storing non-bytes in implicit artefacts."""
server = MockServer()
db, store = server.new_connection()
art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
_graph.set_data(
db,
store,
art.hash,
_serdes.encode(art.kind, "what"),
_graph.ArtefactStatus.done,
)
out = _graph.get_data(db, store, art)
assert isinstance(out, Error)
assert out.kind == ErrorKind.WrongType
art = _graph.variable_artefact(db, hash_t("2"), "file", Encoding.json)
_graph.set_data(
db,
store,
art.hash,
_serdes.encode(art.kind, ["what", 1]),
_graph.ArtefactStatus.done,
)
out = _graph.get_data(db, store, art)
assert out == ["what", 1]
def test_shell_run() -> None:
"""Test shell command."""
with Fun(MockServer()):
db, store = _context.get_connection()
s = ui.shell("cp file1 file2", inp={"file1": b"wawa"}, out=["file2"])
run_op(db, store, s.hash)
assert _graph.get_data(db, store, s.stderr) == b""
assert _graph.get_data(db, store, s.returncode) == 0
assert _graph.get_data(db, store, s.inp["file1"]) == b"wawa"
assert _graph.get_data(db, store, s.stdout) == b""
assert ui.take(s.out["file2"]) == b"wawa"
def test_subdag() -> None:
"""Test run of a subdag function."""
# funsies
import funsies as f
opt = options()
serv = MockServer()
with f.Fun(serv):
db, store = _context.get_connection()
def map_reduce(inputs: Dict[str, bytes]) -> Dict[str, _graph.Artefact]:
"""Basic map reduce."""
inp_data = inputs["inp"].split(b" ")
for el in inp_data:
out = f.morph(lambda x: x.upper(), el, opt=options())
return {"out": f.utils.concat(out, join="-")}
cmd = sub.subdag_funsie(map_reduce, {"inp": Encoding.blob},
{"out": Encoding.blob})
inp = {"inp": _graph.constant_artefact(db, store, b"bla bla blo lol")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.subdag_ready
# test output data
dat = _graph.get_data(
db,
store,
_graph.Artefact[bytes].grab(db, operation.out["out"]),
do_resolve_link=False,
)
assert isinstance(dat, f.errors.Error)
assert dat.kind == "UnresolvedLink"
datl = _graph.get_data(
db,
store,
_graph.Artefact[bytes].grab(db, operation.out["out"]),
do_resolve_link=True,
)
assert isinstance(datl, f.errors.Error)
assert datl.kind == "NotFound"
def test_artefact_add_implicit() -> None:
"""Test adding implicit artefacts."""
options()
server = MockServer()
db, store = server.new_connection()
art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
out = _graph.get_data(db, store, art)
assert isinstance(out, Error)
assert out.kind == ErrorKind.NotFound
def test_artefact_add_large() -> None:
"""Test adding large artefacts to a Redis store."""
db = Redis()
store = RedisStorage(db, block_size=8)
art = _graph.variable_artefact(db, hash_t("1"), "file", cons.Encoding.blob)
data = b"12345" * 100
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert db.llen(cons.join(cons.ARTEFACTS, art.hash, "data")) == 63
assert data == data2
def test_artefact_add() -> None:
"""Test adding const artefacts."""
options()
server = MockServer()
db, store = server.new_connection()
a = _graph.constant_artefact(db, store, b"bla bla")
b = _graph.Artefact[bytes].grab(db, a.hash)
c = _graph.get_data(db, store, a)
assert b is not None
assert a == b
assert c == b"bla bla"
def test_artefact_disk() -> None:
"""Test saving an artefact to disk and restoring it."""
with tempfile.TemporaryDirectory() as td:
db = Redis()
store = DiskStorage(td)
art = _graph.variable_artefact(db, hash_t("12345678"), "file",
cons.Encoding.blob)
data = b"12345" * 100
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert data == data2
def test_artefact_replace_large() -> None:
"""Test replacing large artefacts."""
db = Redis()
store = RedisStorage(db, block_size=8)
art = _graph.variable_artefact(db, hash_t("1"), "file", cons.Encoding.blob)
data = b"12345" * 100
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
store.block_size = 1000000
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert data == data2
def test_artefact_disk_json() -> None:
"""Test saving an artefact to disk and restoring it."""
with tempfile.TemporaryDirectory() as td:
db = Redis()
store = DiskStorage(td)
art = _graph.variable_artefact(db, hash_t("12345678"), "file",
cons.Encoding.json)
dat = [0, 1, 2, 3, "a string"]
bdat = _serdes.encode(_serdes.kind(dat), dat)
_graph.set_data(db, store, art.hash, bdat, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert dat == data2
def test_shell_run() -> None:
"""Test run on a shell command."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = s.shell_funsie(["cat file1"], {"file1": Encoding.blob}, [])
inp = {"file1": _graph.constant_artefact(db, store, b"bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
# check data is good
dat = _graph.get_data(
db, store, _graph.Artefact[bytes].grab(db, operation.inp["file1"]))
assert dat == b"bla bla"
dat = _graph.get_data(
db, store, _graph.Artefact[bytes].grab(db,
operation.out[f"{s.STDOUT}0"]))
assert dat == b"bla bla"
def test_pyfunc_run() -> None:
"""Test run on a python function."""
opt = options()
serv = MockServer()
db, store = serv.new_connection()
cmd = p.python_funsie(capitalize, {"inp": Encoding.json},
{"inp": Encoding.json},
name="capit")
inp = {"inp": _graph.constant_artefact(db, store, "bla bla")}
operation = _graph.make_op(db, cmd, inp, opt)
status = run_op(db, store, operation.hash)
# test return values
assert status == RunStatus.executed
# check data is good
dat = _graph.get_data(db, store,
_graph.Artefact[str].grab(db, operation.inp["inp"]))
assert dat == "bla bla"
dat = _graph.get_data(db, store,
_graph.Artefact[str].grab(db, operation.out["inp"]))
assert dat == "BLA BLA"
def test_shell_run2() -> None:
"""Test shell command output side cases."""
with Fun(MockServer()):
db, store = _context.get_connection()
s = ui.shell("cp file1 file2", "cat file2", inp={"file1": b"wawa"})
run_op(db, store, s.hash)
assert _graph.get_data(db, store, s.inp["file1"]) == b"wawa"
with pytest.raises(Exception):
_graph.get_data(db, store, s.stdout)
with pytest.raises(Exception):
_graph.get_data(db, store, s.stderr)
with pytest.raises(Exception):
_graph.get_data(db, store, s.returncode)
assert ui.take(s.stdouts[1]) == b"wawa"