def test_db_set_value(env):
r = env.test_runtime()
r.configure_executor(heartbeat_interval=1)
e1 = r.start_executor()
c = r.register_builder(Builder(None, "col1"))
assert r.db.get_entry_state(c.name, make_key("cfg1")) is None
r.db.announce_entries(e1.id, [c("cfg1")], [])
assert r.db.get_entry_state(c.name, make_key("cfg1")) == "announced"
assert r.try_read_entry(c("cfg1")) is None
assert r.try_read_entry(c("cfg1"), include_announced=True) is not None
e = make_raw_entry(r, c, "cfg1", "value1")
r.db.set_entry_values(e1.id, [e])
assert r.db.get_entry_state(c.name, make_key("cfg1")) == "finished"
assert r.try_read_entry(c("cfg1")) is not None
assert r.try_read_entry(c("cfg1"), include_announced=True) is not None
with pytest.raises(Exception):
r.db.set_entry_values(e1.id, [e])
e2 = make_raw_entry(r, c, "cfg2", "value2")
with pytest.raises(Exception):
r.db.set_entry_values(e1.id, [e2])
r.db.announce_entries(e1.id, [c("cfg2")], [])
r.db.set_entry_values(e1.id, [e2])
with pytest.raises(Exception):
r.db.create_entries([e2])
e3 = make_raw_entry(r, c, "cfg3", "value3")
r.db.create_entries([e3])
def test_db_run_stats(env):
runtime = env.test_runtime()
runtime.configure_executor(heartbeat_interval=1)
e1 = runtime.start_executor()
c = runtime.register_builder(Builder(None, "col1"))
_ = runtime.register_builder(Builder(None, "col2"))
assert runtime.db.announce_entries(
e1.id, [c("a"), c("b"), c("c"), c("d"),
c("e")], [])
assert runtime.db.get_entry_state(c.name, make_key("a")) == "announced"
runtime.db._dump()
entry = make_raw_entry(runtime, c, "a", "value", comp_time=1)
runtime.db.set_entry_values(e1.id, [entry])
entry = make_raw_entry(runtime, c, "b", "value", comp_time=2)
runtime.db.set_entry_values(e1.id, [entry])
entry = make_raw_entry(runtime, c, "c", "value", comp_time=3)
runtime.db.set_entry_values(e1.id, [entry])
entry = make_raw_entry(runtime, c, "d", "value", comp_time=4)
runtime.db.set_entry_values(e1.id, [entry])
r = runtime.db.get_run_stats("col1")
assert pytest.approx(2.5) == r["avg"]
assert pytest.approx(1.29099, 0.00001) == r["stdev"]
assert r["count"] == 4
r = runtime.db.get_run_stats("col2")
assert None is r["avg"]
assert None is r["stdev"]
assert r["count"] == 0
def test_executor(env, n_processes):
def to_dict(lst):
return {x["id"]: x for x in lst}
runtime = env.test_runtime()
r = runtime._executor_summaries()
assert len(r) == 0
executor = env.executor(runtime,
heartbeat_interval=1,
n_processes=n_processes)
executor.start()
executor2 = env.executor(runtime,
heartbeat_interval=1,
n_processes=n_processes)
executor2._debug_do_not_start_heartbeat = True
executor2.start()
executor3 = env.executor(runtime,
heartbeat_interval=1,
n_processes=n_processes)
executor3.start()
c = runtime.register_builder(Builder(None, "abc"))
runtime.db.announce_entries(executor3.id, [c("x")], [])
assert runtime.db.get_entry_state(c.name, make_key("x")) == "announced"
executor3.stop()
assert runtime.db.get_entry_state(c.name, make_key("x")) is None
r = to_dict(runtime._executor_summaries())
assert len(r) == 3
assert r[executor.id]["status"] == "running"
assert r[executor2.id]["status"] == "running"
assert r[executor3.id]["status"] == "stopped"
time.sleep(3)
r = to_dict(runtime._executor_summaries())
assert len(r) == 3
assert r[executor.id]["status"] == "running"
assert r[executor2.id]["status"] == "lost"
assert r[executor3.id]["status"] == "stopped"
def test_xdb_announce_basic(env):
r = env.test_runtime()
c = r.register_builder(Builder(lambda x: x, "col1"))
assert announce(r, [c(x="test1"), c(x="test2")])
assert not announce(r, [c(x="test2"), c(x="test3")])
assert not announce(r, [c(x="test2"), c(x="test3")])
not announce(r, [c(x="test3")])
assert r.db.get_active_state(make_key(
c.name, {"x": "test1"})) == JobState.ANNOUNCED
r.db.drop_unfinished_jobs()
assert r.db.get_active_state(make_key(c.name,
{"x": "test1"})) == JobState.DETACHED
assert announce(r, [c(x="test2")])
assert not announce(r, [c(x="test2"), c(x="test3")])
assert not announce(r, [c(x="test2")])
assert announce(r, [c(x="test3")])
def test_db_announce(env):
r = env.test_runtime()
e1 = env.executor(r, heartbeat_interval=1)
e1._debug_do_not_start_heartbeat = True
e1.start()
e2 = env.executor(r, heartbeat_interval=1)
e2.runtime = r
e2.start()
c = r.register_builder(Builder(None, "col1"))
assert r.db.announce_entries(e1.id, [c("test1"), c("test2")], [])
assert not r.db.announce_entries(e2.id, [c("test2"), c("test3")], [])
assert not r.db.announce_entries(e1.id, [c("test2"), c("test3")], [])
assert r.db.announce_entries(e2.id, [c("test3")], [])
assert r.db.get_entry_state(c.name, make_key("test1")) == "announced"
time.sleep(3)
assert r.db.get_entry_state(c.name, make_key("test1")) is None
assert not r.db.announce_entries(e2.id, [c("test2"), c("test3")], [])
assert r.db.announce_entries(e2.id, [c("test2")], [])
assert not r.db.announce_entries(e2.id, [c("test2")], [])
def test_make_key_ignored_keys():
assert make_key("z", {"x": 10, "__not_here": 20}) == make_key("z", {"x": 10})
assert make_key("z", {"x": 10, "__not_here": 20}) == make_key(
"z", {"x": 10, "__no_here": 30}
)
assert make_key("z", {"x": 10, "__not_here": 20}) == make_key(
"z", {"x": 10, "__no_here2": 40}
)
def test_make_key_invalid():
class X():
pass
with pytest.raises(Exception):
make_key(["10", X()])
with pytest.raises(Exception):
make_key([X()])
with pytest.raises(Exception):
make_key({10: 10})
def test_make_key_invalid():
class X:
pass
with pytest.raises(Exception, match="Invalid item in config"):
make_key("abc", ["10", X()])
with pytest.raises(Exception, match="Invalid item in config"):
make_key("abc", [X()])
with pytest.raises(Exception, match="Invalid key in config"):
make_key("abc", {10: 10})
def make_raw_entry(runtime, c, cfg, value, comp_time=1):
builder = runtime._builders[c.name]
return builder.make_raw_entry(c.name, make_key(cfg), cfg,
pickle.dumps(value), None, comp_time)
def test_make_key_basics():
assert make_key(10) == "10"
assert make_key("Hello!") == "'Hello!'"
assert make_key(3.14) == "3.14"
assert make_key([True, False, 2]) == "[True,False,2,]"
assert make_key({"x": 10, "y": 20}) == make_key({"y": 20, "x": 10})
assert make_key({"x": 10, "y": 20}) != make_key({"y": 10, "x": 20})
assert make_key((10, 20)) == make_key([10, 20])
assert make_key([10, 20]) != make_key([20, 10])
def test_make_key_different_builders():
assert make_key("abc", 10) != make_key("ab", 10)
assert make_key("abc", 10) != make_key("abc", 11)