def test_two_ensembles_memory_usage(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
# Ensemble one should eventually end
joshua.tail_ensemble(ensemble_id, username="******")
# Start ensemble two
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
# Ensemble two should eventually end
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def test_dead_agent(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
# simulate another agent dying after starting a test
assert joshua_model.try_starting_test(ensemble_id, 12345)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
# Ensemble should still eventually end
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def test_two_agents_large_ensemble(monkeypatch, tmp_path, empty_ensemble):
"""
:monkeypatch: https://docs.pytest.org/en/stable/monkeypatch.html
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
# Make downloading an ensemble take an extra second, and increment
# downloads_started at the beginning of downloading
downloads_started = ThreadSafeCounter()
def ensure_state_test_delay():
downloads_started.increment()
time.sleep(1)
monkeypatch.setattr(joshua_agent, "ensure_state_test_delay",
ensure_state_test_delay)
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agents = []
for rank in range(2):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
while True:
# Wait until the first agent has begun downloading before starting the second agent
if downloads_started.get() > 0:
break
time.sleep(0.01)
joshua.tail_ensemble(ensemble_id, username="******")
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert get_started(joshua_model.db) == 1
for agent in agents:
agent.join()
def test_two_agents(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agents = []
for rank in range(2):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
# before starting agent two, wait until agent one has started on this ensemble
while get_started(joshua_model.db) != 1:
time.sleep(0.001)
joshua.tail_ensemble(ensemble_id, username="******")
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
# The second agent won't have started this ensemble (unless somehow > 10
# seconds passed without the first agent completing the ensemble)
assert get_started(joshua_model.db) == 1
for agent in agents:
agent.join()
def test_stop_ensemble(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble(
"joshua", {"max_runs": 1e12}, open(empty_ensemble, "rb")
)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
while len(joshua_model.show_in_progress(ensemble_id)) == 0:
time.sleep(0.001)
joshua.stop_ensemble(ensemble_id, username="******")
assert joshua_model.show_in_progress(ensemble_id) == []
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def test_create_ensemble():
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {}, io.BytesIO())
assert len(joshua_model.list_active_ensembles()) > 0
