def test_two_ensembles_memory_usage(tmp_path, empty_ensemble): """ :tmp_path: https://docs.pytest.org/en/stable/tmpdir.html """ assert len(joshua_model.list_active_ensembles()) == 0 ensemble_id = joshua_model.create_ensemble("joshua", { "max_runs": 1, "timeout": 1 }, open(empty_ensemble, "rb")) agent = threading.Thread( target=joshua_agent.agent, args=(), kwargs={ "work_dir": tmp_path, "agent_idle_timeout": 1, }, ) agent.setDaemon(True) agent.start() # Ensemble one should eventually end joshua.tail_ensemble(ensemble_id, username="******") # Start ensemble two ensemble_id = joshua_model.create_ensemble("joshua", { "max_runs": 1, "timeout": 1 }, open(empty_ensemble, "rb")) # Ensemble two should eventually end joshua.tail_ensemble(ensemble_id, username="******") agent.join()
def test_dead_agent(tmp_path, empty_ensemble): """ :tmp_path: https://docs.pytest.org/en/stable/tmpdir.html """ assert len(joshua_model.list_active_ensembles()) == 0 ensemble_id = joshua_model.create_ensemble("joshua", { "max_runs": 1, "timeout": 1 }, open(empty_ensemble, "rb")) # simulate another agent dying after starting a test assert joshua_model.try_starting_test(ensemble_id, 12345) agent = threading.Thread( target=joshua_agent.agent, args=(), kwargs={ "work_dir": tmp_path, "agent_idle_timeout": 1, }, ) agent.setDaemon(True) agent.start() # Ensemble should still eventually end joshua.tail_ensemble(ensemble_id, username="******") agent.join()
def test_two_agents_large_ensemble(monkeypatch, tmp_path, empty_ensemble): """ :monkeypatch: https://docs.pytest.org/en/stable/monkeypatch.html :tmp_path: https://docs.pytest.org/en/stable/tmpdir.html """ # Make downloading an ensemble take an extra second, and increment # downloads_started at the beginning of downloading downloads_started = ThreadSafeCounter() def ensure_state_test_delay(): downloads_started.increment() time.sleep(1) monkeypatch.setattr(joshua_agent, "ensure_state_test_delay", ensure_state_test_delay) @fdb.transactional def get_started(tr): return joshua_model._get_snap_counter(tr, ensemble_id, "started") assert len(joshua_model.list_active_ensembles()) == 0 ensemble_id = joshua_model.create_ensemble("joshua", { "max_runs": 1, "timeout": 1 }, open(empty_ensemble, "rb")) agents = [] for rank in range(2): agent = threading.Thread( target=joshua_agent.agent, args=(), kwargs={ "work_dir": os.path.join(tmp_path, str(rank)), "agent_idle_timeout": 1, }, ) agent.setDaemon(True) agent.start() agents.append(agent) while True: # Wait until the first agent has begun downloading before starting the second agent if downloads_started.get() > 0: break time.sleep(0.01) joshua.tail_ensemble(ensemble_id, username="******") @fdb.transactional def get_started(tr): return joshua_model._get_snap_counter(tr, ensemble_id, "started") assert get_started(joshua_model.db) == 1 for agent in agents: agent.join()
def test_two_agents(tmp_path, empty_ensemble): """ :tmp_path: https://docs.pytest.org/en/stable/tmpdir.html """ @fdb.transactional def get_started(tr): return joshua_model._get_snap_counter(tr, ensemble_id, "started") assert len(joshua_model.list_active_ensembles()) == 0 ensemble_id = joshua_model.create_ensemble("joshua", { "max_runs": 1, "timeout": 1 }, open(empty_ensemble, "rb")) agents = [] for rank in range(2): agent = threading.Thread( target=joshua_agent.agent, args=(), kwargs={ "work_dir": os.path.join(tmp_path, str(rank)), "agent_idle_timeout": 1, }, ) agent.setDaemon(True) agent.start() agents.append(agent) # before starting agent two, wait until agent one has started on this ensemble while get_started(joshua_model.db) != 1: time.sleep(0.001) joshua.tail_ensemble(ensemble_id, username="******") @fdb.transactional def get_started(tr): return joshua_model._get_snap_counter(tr, ensemble_id, "started") # The second agent won't have started this ensemble (unless somehow > 10 # seconds passed without the first agent completing the ensemble) assert get_started(joshua_model.db) == 1 for agent in agents: agent.join()
def test_stop_ensemble(tmp_path, empty_ensemble): """ :tmp_path: https://docs.pytest.org/en/stable/tmpdir.html """ assert len(joshua_model.list_active_ensembles()) == 0 ensemble_id = joshua_model.create_ensemble( "joshua", {"max_runs": 1e12}, open(empty_ensemble, "rb") ) agent = threading.Thread( target=joshua_agent.agent, args=(), kwargs={ "work_dir": tmp_path, "agent_idle_timeout": 1, }, ) agent.setDaemon(True) agent.start() while len(joshua_model.show_in_progress(ensemble_id)) == 0: time.sleep(0.001) joshua.stop_ensemble(ensemble_id, username="******") assert joshua_model.show_in_progress(ensemble_id) == [] joshua.tail_ensemble(ensemble_id, username="******") agent.join()
def test_create_ensemble(): assert len(joshua_model.list_active_ensembles()) == 0 ensemble_id = joshua_model.create_ensemble("joshua", {}, io.BytesIO()) assert len(joshua_model.list_active_ensembles()) > 0