def test_artefact_wrong_type() -> None:
"""Test storing non-bytes in implicit artefacts."""
server = MockServer()
db, store = server.new_connection()
art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
_graph.set_data(
db,
store,
art.hash,
_serdes.encode(art.kind, "what"),
_graph.ArtefactStatus.done,
)
out = _graph.get_data(db, store, art)
assert isinstance(out, Error)
assert out.kind == ErrorKind.WrongType
art = _graph.variable_artefact(db, hash_t("2"), "file", Encoding.json)
_graph.set_data(
db,
store,
art.hash,
_serdes.encode(art.kind, ["what", 1]),
_graph.ArtefactStatus.done,
)
out = _graph.get_data(db, store, art)
assert out == ["what", 1]
def test_artefact_disk_json() -> None:
"""Test saving an artefact to disk and restoring it."""
with tempfile.TemporaryDirectory() as td:
db = Redis()
store = DiskStorage(td)
art = _graph.variable_artefact(db, hash_t("12345678"), "file",
cons.Encoding.json)
dat = [0, 1, 2, 3, "a string"]
bdat = _serdes.encode(_serdes.kind(dat), dat)
_graph.set_data(db, store, art.hash, bdat, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert dat == data2
def test_serde_blob() -> None:
"""Test 'blob' ser/deser."""
assert _serdes.encode(Encoding.blob, b"bla") == b"bla"
assert isinstance(_serdes.encode(Encoding.blob, "bla bla bla"), Error)
def test_integration(reference: str, nworkers: int) -> None:
"""Test full integration."""
# make a temp file and copy reference database
dir = tempfile.mkdtemp()
if not make_reference:
shutil.copy(os.path.join(ref_dir, reference, "appendonly.aof"), dir)
shutil.copytree(os.path.join(ref_dir, reference, "data"),
os.path.join(dir, "data"))
shutil.copy(os.path.join(ref_dir, "redis.conf"), dir)
# data url
datadir = f"file://{os.path.join(dir, 'data')}"
# Dictionary for test data
test_data: dict[str, Any] = {}
def update_data(a: dict[int, int], b: list[int]) -> dict[int, int]:
for i in b:
a[i] = a.get(i, 0) + 1
return a
def sum_data(x: dict[int, int]) -> int:
return sum([int(k) * v for k, v in x.items()])
def make_secret(x: int) -> str:
return secrets.token_hex(x)
# Start funsie script
with ManagedFun(
nworkers=nworkers,
directory=dir,
data_url=datadir,
redis_args=["redis.conf"],
) as db:
integers = put([5, 4, 8, 9, 9, 10, 1, 3])
init_data = put({100: 9})
test_data["init_data"] = init_data
nbytes = put(4)
s1 = reduce(update_data, init_data, integers)
num = morph(sum_data, s1)
date = shell("date").stdout
test_data["date"] = date
rand = morph(make_secret, nbytes)
s4 = template(
"date:{{date}}\n" + "some random bytes:{{random}}\n" +
"a number: {{num}}\n" + "a string: {{string}}\n",
{
"date": date,
"random": rand,
"num": num,
"string": "wazza"
},
name="a template",
)
test_data["s4"] = s4
execute(s4)
wait_for(s4, 5)
# check that the db doesn't itself include data
for k in db.keys():
assert b"data" not in k
if make_reference:
folder = os.path.join(ref_dir, reference)
os.makedirs(folder, exist_ok=True)
for name, artefact in test_data.items():
with open(os.path.join(folder, name), "wb") as f:
execute(artefact)
wait_for(artefact, 10.0)
out = take(artefact)
data2 = _serdes.encode(artefact.kind, out)
assert isinstance(data2, bytes)
f.write(data2)
shutil.copy(
os.path.join(dir, "appendonly.aof"),
os.path.join(folder, "appendonly.aof"),
)
shutil.copytree(
os.path.join(dir, "data"),
os.path.join(folder, "data"),
)
else:
# Test against reference dbs
for name, artefact in test_data.items():
execute(artefact)
wait_for(artefact, 10.0)
with open(os.path.join(ref_dir, reference, name), "rb") as f:
data = f.read()
out = take(artefact)
data_ref = _serdes.encode(artefact.kind, out)
assert isinstance(data_ref, bytes)
assert data == data_ref
# delete tempdir
shutil.rmtree(dir)
def test_integration(reference: str, nworkers: int) -> None:
"""Test full integration."""
# make a temp file and copy reference database
dir = tempfile.mkdtemp()
if not make_reference:
shutil.copy(os.path.join(ref_dir, reference, "appendonly.aof"), dir)
shutil.copy(os.path.join(ref_dir, "redis.conf"), dir)
# Dictionary for test data
test_data = {}
# Start funsie script
with ManagedFun(nworkers=nworkers,
directory=dir,
redis_args=["redis.conf"]):
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell(
"cat file1 file2; grep 'bla' file2 file1 > file3; date >> file3",
inp=dict(file1=step1, file2=dat),
out=["file2", "file3"],
)
echo = shell("sleep 1", "date")
merge = reduce(
join_bytes,
step2.out["file3"],
echo.stdouts[1],
name="merger",
)
def tolist(x: bytes, y: bytes) -> Dict[int, str]:
return {1: x.decode(), 8: y.decode()}
A = py(tolist, merge, echo.stdouts[1])
test_data["test1"] = A
def raises(inp: bytes) -> bytes:
raise RuntimeError("an error was raised")
def error_count(*inp: Result[bytes]) -> bytes:
out = utils.match_results(inp, lambda x: 0, lambda x: 1)
return str(sum(out)).encode()
err = morph(raises, dat)
count = reduce(error_count,
dat,
dat,
err,
dat,
err,
err,
echo.stdouts[0],
strict=False)
cat = utils.concat(merge,
dat,
err,
count,
echo.stdouts[1],
strict=False)
test_data["test2"] = cat
execute(step1)
wait_for(step1, timeout=10.0)
execute(step2)
wait_for(step2, timeout=10.0)
assert take(step1) == b"BLA BLA"
assert take(step2.stdout) == b"BLA BLAbla bla"
if make_reference:
folder = os.path.join(ref_dir, reference)
os.makedirs(folder, exist_ok=True)
for name, artefact in test_data.items():
with open(os.path.join(folder, name), "wb") as f:
execute(artefact)
wait_for(artefact, 10.0)
out = take(artefact)
data2 = _serdes.encode(artefact.kind, out)
assert isinstance(data2, bytes)
f.write(data2)
shutil.copy(
os.path.join(dir, "appendonly.aof"),
os.path.join(folder, "appendonly.aof"),
)
else:
# Test against reference dbs
for name, artefact in test_data.items():
execute(artefact)
wait_for(artefact, 10.0)
with open(os.path.join(ref_dir, reference, name), "rb") as f:
data = f.read()
out = take(artefact)
data_ref = _serdes.encode(artefact.kind, out)
assert isinstance(data_ref, bytes)
assert data == data_ref
shutil.rmtree(dir)