def test_artefact_wrong_type() -> None: """Test storing non-bytes in implicit artefacts.""" server = MockServer() db, store = server.new_connection() art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob) _graph.set_data( db, store, art.hash, _serdes.encode(art.kind, "what"), _graph.ArtefactStatus.done, ) out = _graph.get_data(db, store, art) assert isinstance(out, Error) assert out.kind == ErrorKind.WrongType art = _graph.variable_artefact(db, hash_t("2"), "file", Encoding.json) _graph.set_data( db, store, art.hash, _serdes.encode(art.kind, ["what", 1]), _graph.ArtefactStatus.done, ) out = _graph.get_data(db, store, art) assert out == ["what", 1]
def test_artefact_load_errors() -> None: """Test loading artefact errors.""" db = Redis() store = RedisStorage(db) with pytest.raises(RuntimeError): _ = _graph.Artefact.grab(db, hash_t("bla")) # TODO check that warnings are logged? _graph.constant_artefact(db, store, b"bla bla") _graph.constant_artefact(db, store, b"bla bla") _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob) _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob)
def test_operation_pack() -> None: """Test packing and unpacking of operations.""" opt = options() server = MockServer() db, store = server.new_connection() a = _graph.constant_artefact(db, store, b"bla bla") b = _graph.constant_artefact(db, store, b"bla bla bla") fun = f.Funsie( how=f.FunsieHow.shell, what="cat infile", inp={"infile": Encoding.blob}, out={"out": Encoding.json}, extra={}, ) op = _graph.make_op(db, fun, {"infile": a}, opt) op2 = _graph.Operation.grab(db, op.hash) assert op == op2 with pytest.raises(AttributeError): op = _graph.make_op(db, fun, {}, opt) with pytest.raises(AttributeError): # no inputs op = _graph.make_op(db, fun, {}, opt) with pytest.raises(AttributeError): # too many inputs op = _graph.make_op(db, fun, {"infile": a, "infile2": b}, opt) with pytest.raises(RuntimeError): op = _graph.Operation.grab(db, hash_t("b"))
def test_shorten_hash(): """Test hash shortening.""" m = hashlib.sha1() m.update("hi 👋 I am a sha 1 hash".encode()) val = m.hexdigest() h = hash_t(val) short_hash = sh.shorten_hash(h) assert short_hash == h[:6]
def test_artefact_add_implicit() -> None: """Test adding implicit artefacts.""" options() server = MockServer() db, store = server.new_connection() art = _graph.variable_artefact(db, hash_t("1"), "file", Encoding.blob) out = _graph.get_data(db, store, art) assert isinstance(out, Error) assert out.kind == ErrorKind.NotFound
def test_artefact_add_large() -> None: """Test adding large artefacts to a Redis store.""" db = Redis() store = RedisStorage(db, block_size=8) art = _graph.variable_artefact(db, hash_t("1"), "file", cons.Encoding.blob) data = b"12345" * 100 _graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done) data2 = _graph.get_data(db, store, art) assert db.llen(cons.join(cons.ARTEFACTS, art.hash, "data")) == 63 assert data == data2
def test_artefact_disk() -> None: """Test saving an artefact to disk and restoring it.""" with tempfile.TemporaryDirectory() as td: db = Redis() store = DiskStorage(td) art = _graph.variable_artefact(db, hash_t("12345678"), "file", cons.Encoding.blob) data = b"12345" * 100 _graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done) data2 = _graph.get_data(db, store, art) assert data == data2
def test_artefact_replace_large() -> None: """Test replacing large artefacts.""" db = Redis() store = RedisStorage(db, block_size=8) art = _graph.variable_artefact(db, hash_t("1"), "file", cons.Encoding.blob) data = b"12345" * 100 _graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done) store.block_size = 1000000 _graph.set_data(db, store, art.hash, data, _graph.ArtefactStatus.done) data2 = _graph.get_data(db, store, art) assert data == data2
def test_artefact_disk_json() -> None: """Test saving an artefact to disk and restoring it.""" with tempfile.TemporaryDirectory() as td: db = Redis() store = DiskStorage(td) art = _graph.variable_artefact(db, hash_t("12345678"), "file", cons.Encoding.json) dat = [0, 1, 2, 3, "a string"] bdat = _serdes.encode(_serdes.kind(dat), dat) _graph.set_data(db, store, art.hash, bdat, _graph.ArtefactStatus.done) data2 = _graph.get_data(db, store, art) assert dat == data2
def test_get_short_hash(): """Test short hash saving and loading.""" m = hashlib.sha1() m.update("hi 👋 I am a sha 1 hash".encode()) val = m.hexdigest() m = hashlib.sha1() m.update("hi 👋 am another sha 1 hash".encode()) val2 = m.hexdigest() # set up a hash that collides with val val_collide = val[:6] + "b" * (len(val) - 6) print(val) print(val2) print(val_collide) db = Redis() sh.hash_save(db, hash_t(val)) sh.hash_save(db, hash_t(val_collide)) sh.hash_save(db, hash_t(val2)) dat = sh.hash_load(db, "") assert len(dat) == 3 for i in range(1, 7): dat = sh.hash_load(db, val[:i]) assert len(dat) == 2 for i in range(7, len(val)): dat = sh.hash_load(db, val[:i]) assert len(dat) == 1 dat = sh.hash_load(db, val2) assert len(dat) == 1 with pytest.raises(AttributeError): sh.hash_load(db, val + "x")