def test_dead_agent(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
# simulate another agent dying after starting a test
assert joshua_model.try_starting_test(ensemble_id, 12345)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
# Ensemble should still eventually end
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def test_two_ensembles_memory_usage(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
# Ensemble one should eventually end
joshua.tail_ensemble(ensemble_id, username="******")
# Start ensemble two
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
# Ensemble two should eventually end
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def jobtail():
jobid = request.args.get('id')
username = request.args.get('username')
if not jobid and not username:
return redirect(url_for('main.index'))
fileid = ('j-' + jobid) if jobid else ('u-' + username)
raw_arg = request.args.get('raw', default='false').lower()
raw = (raw_arg == 'true') or (raw_arg == '1')
errorsonly_arg = request.args.get('errorsonly', default='true').lower()
errorsonly = (errorsonly_arg == 'true') or (errorsonly_arg == '1')
xml_arg = request.args.get('xml', default='false').lower()
xml = (xml_arg == 'true') or (xml_arg == '1')
simple_arg = request.args.get('simple', default='false').lower()
simple = (simple_arg == 'true') or (simple_arg == '1')
sanity_arg = request.args.get('sanity', default='false').lower()
sanity = (sanity_arg == 'true') or (sanity_arg == '1')
jobfile = os.path.join(app.config['JOSHUA_UPLOAD_FOLDER'],
'tail__' + fileid + '.log')
app.logger.info('jobtail: id: {} errors: {} xml: {} simple: {}'.format(
jobid, errorsonly, xml, simple))
stdout_backup = sys.stdout
sys.stdout = open(jobfile, 'w')
joshua.tail_ensemble(jobid,
raw=raw,
errors_only=errorsonly,
xml=xml,
sanity=sanity,
simple=simple,
stopped=False,
username=username)
sys.stdout = stdout_backup
filehandle = open(jobfile, 'r')
return Response(filehandle.read(), mimetype='text/plain')
def test_two_agents_large_ensemble(monkeypatch, tmp_path, empty_ensemble):
"""
:monkeypatch: https://docs.pytest.org/en/stable/monkeypatch.html
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
# Make downloading an ensemble take an extra second, and increment
# downloads_started at the beginning of downloading
downloads_started = ThreadSafeCounter()
def ensure_state_test_delay():
downloads_started.increment()
time.sleep(1)
monkeypatch.setattr(joshua_agent, "ensure_state_test_delay",
ensure_state_test_delay)
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agents = []
for rank in range(2):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
while True:
# Wait until the first agent has begun downloading before starting the second agent
if downloads_started.get() > 0:
break
time.sleep(0.01)
joshua.tail_ensemble(ensemble_id, username="******")
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert get_started(joshua_model.db) == 1
for agent in agents:
agent.join()
def test_two_agents(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agents = []
for rank in range(2):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
# before starting agent two, wait until agent one has started on this ensemble
while get_started(joshua_model.db) != 1:
time.sleep(0.001)
joshua.tail_ensemble(ensemble_id, username="******")
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
# The second agent won't have started this ensemble (unless somehow > 10
# seconds passed without the first agent completing the ensemble)
assert get_started(joshua_model.db) == 1
for agent in agents:
agent.join()
def test_ensemble_fails(tmp_path, empty_ensemble_fail):
ensemble_id = joshua_model.create_ensemble(
"joshua", {"max_runs": 1, "timeout": 1}, open(empty_ensemble_fail, "rb")
)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
assert get_passes(joshua_model.db, ensemble_id) == 0
assert get_fails(joshua_model.db, ensemble_id) >= 1
def test_stop_ensemble(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble(
"joshua", {"max_runs": 1e12}, open(empty_ensemble, "rb")
)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
while len(joshua_model.show_in_progress(ensemble_id)) == 0:
time.sleep(0.001)
joshua.stop_ensemble(ensemble_id, username="******")
assert joshua_model.show_in_progress(ensemble_id) == []
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()