def test_artefact_wrong_type() -> None:
"""Test storing non-bytes in implicit artefacts."""
server = MockServer()
db, store = server.new_connection()
art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
_graph.set_data(
db,
store,
art.hash,
_serdes.encode(art.kind, "what"),
_graph.ArtefactStatus.done,
)
out = _graph.get_data(db, store, art)
assert isinstance(out, Error)
assert out.kind == ErrorKind.WrongType
art = _graph.variable_artefact(db, hash_t("2"), "file", Encoding.json)
_graph.set_data(
db,
store,
art.hash,
_serdes.encode(art.kind, ["what", 1]),
_graph.ArtefactStatus.done,
)
out = _graph.get_data(db, store, art)
assert out == ["what", 1]
def test_artefact_load_errors() -> None:
"""Test loading artefact errors."""
db = Redis()
store = RedisStorage(db)
with pytest.raises(RuntimeError):
_ = _graph.Artefact.grab(db, hash_t("bla"))
# TODO check that warnings are logged?
_graph.constant_artefact(db, store, b"bla bla")
_graph.constant_artefact(db, store, b"bla bla")
_graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
_graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
def test_operation_pack() -> None:
"""Test packing and unpacking of operations."""
opt = options()
server = MockServer()
db, store = server.new_connection()
a = _graph.constant_artefact(db, store, b"bla bla")
b = _graph.constant_artefact(db, store, b"bla bla bla")
fun = f.Funsie(
how=f.FunsieHow.shell,
what="cat infile",
inp={"infile": Encoding.blob},
out={"out": Encoding.json},
extra={},
)
op = _graph.make_op(db, fun, {"infile": a}, opt)
op2 = _graph.Operation.grab(db, op.hash)
assert op == op2
with pytest.raises(AttributeError):
op = _graph.make_op(db, fun, {}, opt)
with pytest.raises(AttributeError):
# no inputs
op = _graph.make_op(db, fun, {}, opt)
with pytest.raises(AttributeError):
# too many inputs
op = _graph.make_op(db, fun, {"infile": a, "infile2": b}, opt)
with pytest.raises(RuntimeError):
op = _graph.Operation.grab(db, hash_t("b"))
def test_shorten_hash():
"""Test hash shortening."""
m = hashlib.sha1()
m.update("hi 👋 I am a sha 1 hash".encode())
val = m.hexdigest()
h = hash_t(val)
short_hash = sh.shorten_hash(h)
assert short_hash == h[:6]
def test_artefact_add_implicit() -> None:
"""Test adding implicit artefacts."""
options()
server = MockServer()
db, store = server.new_connection()
art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
out = _graph.get_data(db, store, art)
assert isinstance(out, Error)
assert out.kind == ErrorKind.NotFound
def test_artefact_add_large() -> None:
"""Test adding large artefacts to a Redis store."""
db = Redis()
store = RedisStorage(db, block_size=8)
art = _graph.variable_artefact(db, hash_t("1"), "file", cons.Encoding.blob)
data = b"12345" * 100
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert db.llen(cons.join(cons.ARTEFACTS, art.hash, "data")) == 63
assert data == data2
def test_artefact_disk() -> None:
"""Test saving an artefact to disk and restoring it."""
with tempfile.TemporaryDirectory() as td:
db = Redis()
store = DiskStorage(td)
art = _graph.variable_artefact(db, hash_t("12345678"), "file",
cons.Encoding.blob)
data = b"12345" * 100
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert data == data2
def test_artefact_replace_large() -> None:
"""Test replacing large artefacts."""
db = Redis()
store = RedisStorage(db, block_size=8)
art = _graph.variable_artefact(db, hash_t("1"), "file", cons.Encoding.blob)
data = b"12345" * 100
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
store.block_size = 1000000
_graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert data == data2
def test_artefact_disk_json() -> None:
"""Test saving an artefact to disk and restoring it."""
with tempfile.TemporaryDirectory() as td:
db = Redis()
store = DiskStorage(td)
art = _graph.variable_artefact(db, hash_t("12345678"), "file",
cons.Encoding.json)
dat = [0, 1, 2, 3, "a string"]
bdat = _serdes.encode(_serdes.kind(dat), dat)
_graph.set_data(db, store, art.hash, bdat, _graph.ArtefactStatus.done)
data2 = _graph.get_data(db, store, art)
assert dat == data2
def test_get_short_hash():
"""Test short hash saving and loading."""
m = hashlib.sha1()
m.update("hi 👋 I am a sha 1 hash".encode())
val = m.hexdigest()
m = hashlib.sha1()
m.update("hi 👋 am another sha 1 hash".encode())
val2 = m.hexdigest()
# set up a hash that collides with val
val_collide = val[:6] + "b" * (len(val) - 6)
print(val)
print(val2)
print(val_collide)
db = Redis()
sh.hash_save(db, hash_t(val))
sh.hash_save(db, hash_t(val_collide))
sh.hash_save(db, hash_t(val2))
dat = sh.hash_load(db, "")
assert len(dat) == 3
for i in range(1, 7):
dat = sh.hash_load(db, val[:i])
assert len(dat) == 2
for i in range(7, len(val)):
dat = sh.hash_load(db, val[:i])
assert len(dat) == 1
dat = sh.hash_load(db, val2)
assert len(dat) == 1
with pytest.raises(AttributeError):
sh.hash_load(db, val + "x")
