def test_subgraph() -> None:
"""Test that we can isolate the required operators for parametrization."""
with Fun(MockServer(), options(distributed=False)) as db:
dat = put(b"bla bla")
step1 = morph(capitalize, dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
# random not included ops
stepA = shell("echo 'bla'")
_ = concat(dat, dat)
_ = morph(capitalize, b"another word")
final = shell("cat file1 file2",
inp={
"file1": stepA.stdout,
"file2": step2.stdout
})
ops = _p._parametrize_subgraph(db, {"input": dat},
{"output": final.stdout})
assert len(ops) == 3
assert step1.parent in ops
assert step2.hash in ops
assert final.hash in ops
# get edges
edges = _p._subgraph_edges(db, ops)
print(edges)
def test_dag_dump() -> None:
"""Test simple DAG dump to file."""
with Fun(MockServer(), options(distributed=False)) as db:
dat = put(b"bla bla")
dat2 = put(b"blaXbla")
errorstep = morph(raises, dat2)
step1 = morph(upper, dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
step2b = utils.concat(step2.stdout, errorstep, strict=False)
step3 = shell("cat file1", inp=dict(file1=step2b))
step4 = shell("cat file1", inp=dict(file1=step1))
step4b = shell("cat file2", inp=dict(file2=step4.stdout))
out = utils.concat(step1, dat, step2.stdout, step3.stdout)
_dag.build_dag(db, out.hash)
execute(step2b)
execute(step4b)
wait_for(step4b, 1.0)
reset(step4)
nodes, artefacts, labels, links = _graphviz.export(
db, [out.hash, step4b.hash])
dot = _graphviz.format_dot(nodes, artefacts, labels, links,
[out.hash, step4b.hash])
assert len(dot) > 0
assert len(nodes) == 8
assert len(labels) == 8
# TODO pass through dot for testing?
with open("g.dot", "w") as f:
f.write(dot)
def test_toposort() -> None:
"""Test that we can topologically sort the subset."""
with Fun(MockServer(), options(distributed=False)) as db:
dat = put(b"bla bla")
step1 = morph(capitalize, dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
# random not included ops
stepA = shell("echo 'bla'")
_ = concat(dat, dat)
_ = morph(capitalize, b"another word")
final = shell("cat file1 file2",
inp={
"file1": stepA.stdout,
"file2": step2.stdout
})
ops = _p._parametrize_subgraph(db, {"input": dat},
{"output": final.stdout})
edges = _p._subgraph_edges(db, ops)
sorted_ops = _p._subgraph_toposort(ops, edges)
assert sorted_ops[0] == step1.parent
assert sorted_ops[1] == step2.hash
assert sorted_ops[2] == final.hash
def test_double_execution(nworkers: int) -> None:
"""Test multiple executions of the same task."""
# This test will fail if a job is re-executed multiple times.
# external
from rq.job import get_current_job
def track_runs(inp: bytes) -> bytes:
job = get_current_job()
db: Redis[bytes] = job.connection
val = db.incrby("sentinel", 1)
time.sleep(0.5)
return str(val).encode()
with f.ManagedFun(nworkers=nworkers):
# wait_for_workers(db, nworkers)
dat = f.put(b"bla bla")
step1 = f.morph(track_runs, dat)
step1a = f.shell(
"cat file1",
inp=dict(file1=step1),
)
step1b = f.shell(
"cat file2",
inp=dict(file2=step1),
)
f.execute(step1a)
f.execute(step1b)
f.wait_for(step1a, timeout=10.0)
f.wait_for(step1b, timeout=10.0)
assert f.take(step1a.stdout) == b"1"
def test_error_propagation_shell() -> None:
"""Test propagation of errors."""
db = Redis()
store = RedisStorage(db)
s1 = funsies.shell(
"cp file1 file3",
inp=dict(file1="bla"),
out=["file2"],
connection=(db, store),
opt=options(),
)
s2 = funsies.shell(
"cat file2",
inp=dict(file2=s1.out["file2"]),
connection=(db, store),
opt=options(),
)
s3 = funsies.shell(
"cat file2",
inp=dict(file2=s1.out["file2"]),
strict=False,
connection=(db, store),
opt=options(),
)
run_op(db, store, s1.op.hash)
run_op(db, store, s2.op.hash)
with pytest.raises(UnwrapError):
funsies.take(s2.stderr, connection=(db, store))
run_op(db, store, s3.op.hash)
assert funsies.take(s3.stderr, connection=(db, store)) != b""
assert isinstance(funsies.take(s3.returncode, connection=(db, store)), int)
assert funsies.take(s3.returncode, connection=(db, store)) != 0
def test_dag_execute_same_root() -> None:
"""Test execution of two dags that share the same origin."""
with Fun(MockServer(), defaults=options(distributed=False)):
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
step2b = shell("cat file1", inp=dict(file1=step1))
execute(step2)
out = take(step2.stdout)
assert out == b"BLA BLAbla bla"
execute(step2b)
out = take(step2b.stdout)
assert out == b"BLA BLA"
def test_error_propagation() -> None:
"""Test propagation of errors."""
with Fun(MockServer()):
db, store = get_connection()
s1 = funsies.shell("cp file1 file3",
inp=dict(file1="bla"),
out=["file2"])
s2 = funsies.shell("cat file1 file2",
inp=dict(file1="a file", file2=s1.out["file2"]))
run_op(db, store, s1.op.hash)
run_op(db, store, s2.op.hash)
out = funsies.take(s2.stdout, strict=False)
print(out)
assert isinstance(out, Error)
assert out.source == s1.op.hash
def test_parametric() -> None:
"""Test that parametric DAGs work."""
with Fun(MockServer(), options(distributed=False)) as db:
dat = put(b"bla bla")
step1 = morph(capitalize, dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
final = shell("cat file1 file3",
inp={
"file1": step1,
"file3": step2.stdout
})
param = _p.make_parametric(db, "param", {"input": dat},
{"output": final.stdout})
param2 = _p.Parametric.grab(db, param.hash)
assert param == param2
def test_error_propagation_morph() -> None:
"""Test propagation of errors."""
with Fun(MockServer()):
db, store = get_connection()
s1 = funsies.shell("cp file1 file3",
inp=dict(file1="bla"),
out=["file2"])
def fun_strict(inp: bytes) -> bytes:
return inp
def fun_lax(inp: Result[bytes]) -> bytes:
return b"bla bla"
s2 = funsies.morph(fun_strict, s1.out["file2"])
s3 = funsies.morph(fun_lax, s1.out["file2"])
s4 = funsies.morph(fun_lax, s1.out["file2"], strict=False)
run_op(db, store, s1.op.hash)
run_op(db, store, s2.parent)
out = funsies.take(s2, strict=False)
assert isinstance(out, Error)
assert out.source == s1.op.hash
print(s3.parent)
run_op(db, store, s3.parent)
out = funsies.take(s3, strict=False)
assert isinstance(out, Error)
assert out.source == s1.op.hash
run_op(db, store, s4.parent)
out = funsies.take(s4)
assert out == b"bla bla"
def test_subdag() -> None:
"""Test that subdags execute properly."""
def cap(inp: bytes) -> bytes:
return inp.upper()
def map_reduce(
inputs: dict[str, bytes]) -> dict[str, _graph.Artefact[bytes]]:
"""Basic map reduce."""
inp_data = inputs["inp"].split(b" ")
out: list[_graph.Artefact[bytes]] = []
for el in inp_data:
out += [morph(cap, el, opt=options(distributed=False))]
return {"out": concat(*out, join="-")}
with Fun(MockServer(), defaults=options(distributed=False)) as db:
dat = put(b"bla bla lol what")
inp = {"inp": dat}
cmd = _subdag.subdag_funsie(map_reduce, {"inp": Encoding.blob},
{"out": Encoding.blob})
operation = _graph.make_op(db, cmd, inp, options())
out = _graph.Artefact[bytes].grab(db, operation.out["out"])
final = shell(
"cat file1 file2",
inp=dict(file1=out, file2=b"something"),
)
execute(final)
data = take(final.stdout)
assert data == b"BLA-BLA-LOL-WHATsomething"
def test_dag_execute2() -> None:
"""Test execution of a _dag."""
with Fun(MockServer(), defaults=options(distributed=False)):
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
step11 = shell("echo 'bla'")
final = shell("cat file1 file2",
inp={
"file1": step11.stdout,
"file2": step2.stdout
})
output = final.stdout
# make queue
execute(output)
out = take(output)
assert out == b"bla\nBLA BLAbla bla"
def test_parametric_eval() -> None:
"""Test that parametric evaluate properly."""
with Fun(MockServer(), options(distributed=False)) as db:
dat = put(b"bla bla")
step1 = morph(capitalize, dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
final = shell("cat file1 file3",
inp={
"file1": step1,
"file3": step2.stdout
})
execute(final.stdout)
# b'BLA BLABLA BLAbla bla'
param = _p.make_parametric(db, "param", {"input": dat},
{"output": final.stdout})
dat2 = put(b"lol lol")
out = param.evaluate(db, {"input": dat2})
execute(out["output"])
assert take(out["output"]) == b"LOL LOLLOL LOLlol lol"
def test_not_generated() -> None:
"""What happens when an artefact is not generated?"""
with Fun(MockServer()):
db, store = get_connection()
s = funsies.shell("cp file1 file2",
inp=dict(file1="bla"),
out=["file3"])
run_op(db, store, s.op.hash)
assert funsies.take(s.returncode) == 0
with pytest.raises(UnwrapError):
funsies.take(s.out["file3"])
def optimize_conformer_xtb(structure: Artefact[bytes]) -> Artefact[bytes]:
"""Optimize a structure using xtb."""
# perform an xtb optimization
optim = f.shell(
"xtb input.xyz --opt vtight",
inp={"input.xyz": structure},
out=[".xtboptok", "xtbopt.xyz"],
)
# barrier for convergence (fails if .xtboptok is not found)
struct: Artefact[bytes] = f.reduce(lambda x, y: x, optim.out["xtbopt.xyz"],
optim.out[".xtboptok"])
return struct
def test_dag_cached() -> None:
"""Test that DAG caching works."""
serv = MockServer()
with Fun(serv, defaults=options(distributed=False)):
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2b = shell("echo 'not'", inp=dict(file1=step1))
merge = shell("cat file1 file2",
inp=dict(file1=step1, file2=step2b.stdout),
out=["file2"])
execute(merge)
with Fun(serv, defaults=options(distributed=False, evaluate=False)):
# Same as above, should run through with no evaluation
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2b = shell("echo 'not'", inp=dict(file1=step1))
merge = shell("cat file1 file2",
inp=dict(file1=step1, file2=step2b.stdout),
out=["file2"])
execute(merge)
with Fun(serv, defaults=options(distributed=False, evaluate=False)):
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
# DIFFERENT HERE: Trigger re-evaluation and raise
step2b = shell("echo 'knot'", inp=dict(file1=step1))
merge = shell("cat file1 file2",
inp=dict(file1=step1, file2=step2b.stdout),
out=["file2"])
with pytest.raises(RuntimeError):
execute(merge)
def test_artefact_disk_distributed() -> None:
"""Test whether artefacts on disk works on different nodes."""
# funsies
import funsies as f
with tempfile.TemporaryDirectory() as td:
with f.ManagedFun(nworkers=1, data_url=f"file://{td}"):
dat = f.put(b"bla bla")
step1 = f.morph(lambda x: x.decode().upper().encode(), dat)
step2 = f.shell("cat file1 file2",
inp=dict(file1=step1, file2=dat))
step2b = f.shell("cat file1", inp=dict(file1=step1))
f.execute(step2)
f.wait_for(step2, 1.0)
out = f.take(step2.stdout)
assert out == b"BLA BLAbla bla"
f.execute(step2b)
f.wait_for(step2b, 1.0)
out = f.take(step2b.stdout)
assert out == b"BLA BLA"
def test_shell_norun() -> None:
"""Test run on a shell command that didn't run."""
with Fun(MockServer()):
cmd = shell("cat file1", inp={"file1": b"bla bla"}, out=["bla"])
with tempfile.TemporaryDirectory() as d:
debug.shell(cmd, d)
n = os.listdir(d)
assert "input_files" in n
assert "output_files" in n
with open(os.path.join(d, "errors.json"), "r") as f:
assert "NotFound" in f.read()
def test_dag_efficient() -> None:
"""Test that DAG building doesn't do extra work."""
with Fun(MockServer()) as db:
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell("cat file1 file2",
inp=dict(file1=step1, file2=dat),
out=["file2"])
step2b = shell("echo 'not'", inp=dict(file1=step1))
merge = shell("cat file1 file2",
inp=dict(file1=step1, file2=step2b.stdout),
out=["file2"])
_dag.build_dag(db, step2.stdout.hash)
# check that step2 only has stdout has no dependents
assert len(_dag._dag_dependents(db, step2.stdout.hash,
step2.hash)) == 0
assert len(_dag._dag_dependents(db, step2.stdout.hash,
step1.parent)) == 1
_dag.build_dag(db, merge.hash)
# check that however, the merged one has two dependents for step1
assert len(_dag._dag_dependents(db, merge.hash, step1.parent)) == 2
def test_data_race(nworkers: int) -> None:
"""Test a data race when execute calls are interleaved."""
with f.ManagedFun(nworkers=nworkers):
dat = f.put(b"bla bla")
step1 = f.morph(lambda x: x.decode().upper().encode(), dat)
step2 = f.shell(
"cat file1 file2; grep 'bla' file2 file1 > file3; date >> file3",
inp=dict(file1=step1, file2=dat),
out=["file2", "file3"],
)
f.execute(step1)
f.execute(step2)
f.wait_for(step1, timeout=20.0)
f.wait_for(step2, timeout=20.0)
def test_parametrize() -> None:
"""Test that parametrization works."""
with Fun(MockServer(), options(distributed=False)) as db:
dat = put(b"bla bla")
dat2 = put(b"bla bla bla")
step1 = morph(capitalize, dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
final = shell("cat file1 file3",
inp={
"file1": step1,
"file3": step2.stdout
})
pinp = {"input": dat}
pout = {"final.stdout": final.stdout, "step1": step1}
new_inp = {"input": dat2}
ops = _p._parametrize_subgraph(db, pinp, pout)
edges = _p._subgraph_edges(db, ops)
sorted_ops = _p._subgraph_toposort(ops, edges)
pinp2 = dict([(k, v.hash) for k, v in pinp.items()])
pout2 = dict([(k, v.hash) for k, v in pout.items()])
new_out = _p._do_parametrize(db, sorted_ops, pinp2, pout2, new_inp)
# re-run with dat2, check if the same.
step1 = morph(capitalize, dat2)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat2))
final = shell("cat file1 file3",
inp={
"file1": step1,
"file3": step2.stdout
})
assert new_out["final.stdout"] == final.stdout
assert new_out["step1"] == step1
def test_dag_large() -> None:
"""Test that DAG building doesn't do extra work for large operations."""
with Fun(MockServer()) as db:
outputs = []
for i in range(100):
dat = put(f"bla{i}".encode())
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell(
"cat file1 file2",
inp=dict(file1=step1, file2="something"),
out=["file2"],
)
outputs += [concat(step1, step1, step2.stdout, join=b" ")]
final = concat(*outputs, join=b"\n")
_dag.build_dag(db, final.hash)
assert len(_dag._dag_dependents(db, final.hash, hash_t("root"))) == 100
def test_shell_run() -> None:
"""Test run on a shell command."""
with Fun(MockServer()):
db, store = get_connection()
cmd = shell("cat file1", inp={"file1": b"bla bla"}, out=["bla"])
_ = run_op(db, store, cmd.hash)
with tempfile.TemporaryDirectory() as d:
debug.shell(cmd, d)
n = os.listdir(d)
assert "stdout0" in n
assert "stderr0" in n
assert "input_files" in n
assert "output_files" in n
with open(os.path.join(d, "errors.json"), "r") as f:
assert "MissingOutput" in f.read()
def test_artefact() -> None:
"""Test artefact debug."""
with Fun(MockServer()) as db:
cmd = shell("cat file1", inp={"file1": b"bla bla"}, out=["bla"])
with tempfile.TemporaryDirectory() as d:
debug.artefact(cmd.stdout, d, connection=db)
n = os.listdir(d)
assert "metadata.json" in n
assert "error.json" in n
assert "data" not in n
with tempfile.TemporaryDirectory() as d:
debug.artefact(cmd.inp["file1"], d, connection=db)
n = os.listdir(d)
assert "metadata.json" in n
assert "error.json" not in n
assert "data" in n
def test_dag_build() -> None:
"""Test simple DAG build."""
with Fun(MockServer()) as db:
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell("cat file1 file2", inp=dict(file1=step1, file2=dat))
output = step2.stdout
_dag.build_dag(db, output.hash)
assert len(db.smembers(join(DAG_OPERATIONS, output.hash))) == 2
# test deletion
_dag.delete_all_dags(db)
assert len(db.smembers(join(DAG_OPERATIONS, output.hash))) == 0
# test new _dag
_dag.build_dag(db, step1.hash)
assert len(db.smembers(join(DAG_OPERATIONS, step1.hash))) == 1
assert len(_dag.descendants(db, step1.parent)) == 1
def test_error_tolerant() -> None:
"""Test error tolerant funsie."""
def error_tolerant_fun(inp: Result[bytes]) -> bytes:
if isinstance(inp, Error):
return b"err"
else:
return b""
with Fun(MockServer()):
db, store = get_connection()
s1 = funsies.shell("cp file1 file3",
inp=dict(file1="bla"),
out=["file2"])
s2 = funsies.morph(error_tolerant_fun, s1.out["file2"], strict=False)
with pytest.raises(RuntimeError):
# Test operation not found
run_op(db, store, s2.hash)
run_op(db, store, s1.op)
run_op(db, store, s2.parent)
assert funsies.take(s2) == b"err"
def test_timeout_deadlock() -> None:
"""Test funsies that time out.
Here we explicitly check if dependents are still enqueued or if the whole
thing deadlocks.
"""
def timeout_fun(*inp: str) -> bytes:
time.sleep(3.0)
return b"what"
def cap(inp: bytes) -> bytes:
return inp.capitalize()
with f.ManagedFun(nworkers=2):
# Test when python function times out
s1 = f.reduce(timeout_fun,
"bla bla",
"bla bla",
opt=f.options(timeout=1))
s1b = f.morph(cap, s1)
# Test when shell function times out
s2 = f.shell("sleep 20", "echo 'bla bla'", opt=f.options(timeout=1))
s2b = f.morph(cap, s2.stdouts[1])
f.execute(s1b, s2b)
# Check err for reduce
f.wait_for(s1b, timeout=1.5)
err = f.take(s1b, strict=False)
assert isinstance(err, f.errors.Error)
assert err.kind == f.errors.ErrorKind.JobTimedOut
assert err.source == s1.parent
# Check err for shell
f.wait_for(s2b, timeout=1.5)
err = f.take(s2b, strict=False)
assert isinstance(err, f.errors.Error)
assert err.kind == f.errors.ErrorKind.JobTimedOut
assert err.source == s2.hash
def test_integration(reference: str, nworkers: int) -> None:
"""Test full integration."""
# make a temp file and copy reference database
dir = tempfile.mkdtemp()
if not make_reference:
shutil.copy(os.path.join(ref_dir, reference, "appendonly.aof"), dir)
shutil.copy(os.path.join(ref_dir, "redis.conf"), dir)
# Dictionary for test data
test_data = {}
# Start funsie script
with ManagedFun(nworkers=nworkers,
directory=dir,
redis_args=["redis.conf"]):
dat = put(b"bla bla")
step1 = morph(lambda x: x.decode().upper().encode(), dat)
step2 = shell(
"cat file1 file2; grep 'bla' file2 file1 > file3; date >> file3",
inp=dict(file1=step1, file2=dat),
out=["file2", "file3"],
)
echo = shell("sleep 1", "date")
merge = reduce(
join_bytes,
step2.out["file3"],
echo.stdouts[1],
name="merger",
)
def tolist(x: bytes, y: bytes) -> Dict[int, str]:
return {1: x.decode(), 8: y.decode()}
A = py(tolist, merge, echo.stdouts[1])
test_data["test1"] = A
def raises(inp: bytes) -> bytes:
raise RuntimeError("an error was raised")
def error_count(*inp: Result[bytes]) -> bytes:
out = utils.match_results(inp, lambda x: 0, lambda x: 1)
return str(sum(out)).encode()
err = morph(raises, dat)
count = reduce(error_count,
dat,
dat,
err,
dat,
err,
err,
echo.stdouts[0],
strict=False)
cat = utils.concat(merge,
dat,
err,
count,
echo.stdouts[1],
strict=False)
test_data["test2"] = cat
execute(step1)
wait_for(step1, timeout=10.0)
execute(step2)
wait_for(step2, timeout=10.0)
assert take(step1) == b"BLA BLA"
assert take(step2.stdout) == b"BLA BLAbla bla"
if make_reference:
folder = os.path.join(ref_dir, reference)
os.makedirs(folder, exist_ok=True)
for name, artefact in test_data.items():
with open(os.path.join(folder, name), "wb") as f:
execute(artefact)
wait_for(artefact, 10.0)
out = take(artefact)
data2 = _serdes.encode(artefact.kind, out)
assert isinstance(data2, bytes)
f.write(data2)
shutil.copy(
os.path.join(dir, "appendonly.aof"),
os.path.join(folder, "appendonly.aof"),
)
else:
# Test against reference dbs
for name, artefact in test_data.items():
execute(artefact)
wait_for(artefact, 10.0)
with open(os.path.join(ref_dir, reference, name), "rb") as f:
data = f.read()
out = take(artefact)
data_ref = _serdes.encode(artefact.kind, out)
assert isinstance(data_ref, bytes)
assert data == data_ref
shutil.rmtree(dir)
def test_integration(reference: str, nworkers: int) -> None:
"""Test full integration."""
# make a temp file and copy reference database
dir = tempfile.mkdtemp()
if not make_reference:
shutil.copy(os.path.join(ref_dir, reference, "appendonly.aof"), dir)
shutil.copytree(os.path.join(ref_dir, reference, "data"),
os.path.join(dir, "data"))
shutil.copy(os.path.join(ref_dir, "redis.conf"), dir)
# data url
datadir = f"file://{os.path.join(dir, 'data')}"
# Dictionary for test data
test_data: dict[str, Any] = {}
def update_data(a: dict[int, int], b: list[int]) -> dict[int, int]:
for i in b:
a[i] = a.get(i, 0) + 1
return a
def sum_data(x: dict[int, int]) -> int:
return sum([int(k) * v for k, v in x.items()])
def make_secret(x: int) -> str:
return secrets.token_hex(x)
# Start funsie script
with ManagedFun(
nworkers=nworkers,
directory=dir,
data_url=datadir,
redis_args=["redis.conf"],
) as db:
integers = put([5, 4, 8, 9, 9, 10, 1, 3])
init_data = put({100: 9})
test_data["init_data"] = init_data
nbytes = put(4)
s1 = reduce(update_data, init_data, integers)
num = morph(sum_data, s1)
date = shell("date").stdout
test_data["date"] = date
rand = morph(make_secret, nbytes)
s4 = template(
"date:{{date}}\n" + "some random bytes:{{random}}\n" +
"a number: {{num}}\n" + "a string: {{string}}\n",
{
"date": date,
"random": rand,
"num": num,
"string": "wazza"
},
name="a template",
)
test_data["s4"] = s4
execute(s4)
wait_for(s4, 5)
# check that the db doesn't itself include data
for k in db.keys():
assert b"data" not in k
if make_reference:
folder = os.path.join(ref_dir, reference)
os.makedirs(folder, exist_ok=True)
for name, artefact in test_data.items():
with open(os.path.join(folder, name), "wb") as f:
execute(artefact)
wait_for(artefact, 10.0)
out = take(artefact)
data2 = _serdes.encode(artefact.kind, out)
assert isinstance(data2, bytes)
f.write(data2)
shutil.copy(
os.path.join(dir, "appendonly.aof"),
os.path.join(folder, "appendonly.aof"),
)
shutil.copytree(
os.path.join(dir, "data"),
os.path.join(folder, "data"),
)
else:
# Test against reference dbs
for name, artefact in test_data.items():
execute(artefact)
wait_for(artefact, 10.0)
with open(os.path.join(ref_dir, reference, name), "rb") as f:
data = f.read()
out = take(artefact)
data_ref = _serdes.encode(artefact.kind, out)
assert isinstance(data_ref, bytes)
assert data == data_ref
# delete tempdir
shutil.rmtree(dir)
return out
out = []
for s in structures:
out += [f.morph(to_dict, s, out=Encoding.json)] # elements to dicts
return f.reduce(sort_by_energy, *out) # transform to a sorted list
with f.Fun():
# put smiles in db
smiles = f.put(b"C(O)CCCC(O)")
# Generate 3d conformers with openbabel
gen3d = f.shell(
"obabel input.smi --gen3d --ff mmff94 --minimize -O struct.mol",
inp={"input.smi": smiles},
out=["struct.mol"],
)
# abort if molecule is empty
struct = not_empty(gen3d.out["struct.mol"])
# Generate conformers.
confab = f.shell(
"obabel input.mol -O conformers.xyz --confab --verbose",
inp={"input.mol": struct},
out=["conformers.xyz"],
)
# Optimize conformer ensemble using xtb.
optimized1 = f.dynamic.sac(
# split the xyz file
