def test_two_ensembles_memory_usage(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
# Ensemble one should eventually end
joshua.tail_ensemble(ensemble_id, username="******")
# Start ensemble two
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
# Ensemble two should eventually end
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def test_dead_agent(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
# simulate another agent dying after starting a test
assert joshua_model.try_starting_test(ensemble_id, 12345)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
# Ensemble should still eventually end
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def upload():
if request.method == 'POST' and not current_user.is_authenticated:
return redirect(url_for('main.index'))
form = UploadJobForm()
if form.validate_on_submit():
filename = secure_filename(form.file.data.filename)
filepath = os.path.join(app.config['JOSHUA_UPLOAD_FOLDER'],
secure_filename(current_user.username))
if not os.path.exists(filepath):
os.mkdir(filepath, 0o755)
saved_file = os.path.join(filepath, filename)
form.file.data.save(saved_file)
properties = form.get_properties()
flash('Uploaded: user: {} package: {}'.format(
current_user.username, filename))
app.logger.info('Uploaded: user: {} package: {}'.format(
current_user.username, filename))
# convert to non-unicode string for username
properties['username'] = str(current_user.username)
# if not form.allow_multiple.data:
# joshua.stop_ensemble(username=current_user.username, sanity=form.sanity.data)
with open(saved_file, "rb") as tarfile:
tarfile.seek(0, os.SEEK_END)
size = tarfile.tell()
tarfile.seek(0, os.SEEK_SET)
properties['data_size'] = size
ensemble_id = joshua_model.create_ensemble(properties['username'],
properties, tarfile, False)
app.logger.info('Ensemble {} created with properties: {}!'.format(
ensemble_id, properties))
flash('Ensemble {} created!'.format(ensemble_id))
return render_template('upload.html', user=current_user, form=form)
def test_delete_ensemble(tmp_path, empty_ensemble_timeout):
ensemble_id = joshua_model.create_ensemble(
"joshua", {
"max_runs": 10,
"timeout": 1
}, open(empty_ensemble_timeout, "rb"))
agents = []
for rank in range(10):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
time.sleep(0.5) # Give the agents some time to start
joshua_model.delete_ensemble(ensemble_id)
time.sleep(1) # Wait for long enough that agents timeout
assert len(joshua_model.list_all_ensembles()) == 0
for agent in agents:
agent.join()
def test_two_agents_large_ensemble(monkeypatch, tmp_path, empty_ensemble):
"""
:monkeypatch: https://docs.pytest.org/en/stable/monkeypatch.html
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
# Make downloading an ensemble take an extra second, and increment
# downloads_started at the beginning of downloading
downloads_started = ThreadSafeCounter()
def ensure_state_test_delay():
downloads_started.increment()
time.sleep(1)
monkeypatch.setattr(joshua_agent, "ensure_state_test_delay",
ensure_state_test_delay)
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agents = []
for rank in range(2):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
while True:
# Wait until the first agent has begun downloading before starting the second agent
if downloads_started.get() > 0:
break
time.sleep(0.01)
joshua.tail_ensemble(ensemble_id, username="******")
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert get_started(joshua_model.db) == 1
for agent in agents:
agent.join()
def upload_ensemble():
request_data = request.form.to_dict()
request_files = request.files.to_dict()
if not request_data:
app.logger.info('api_upload: No form data')
return {"error": 'api_upload: No form data'}, 400
if not request_files:
app.logger.info('api_upload: Missing uploaded file')
return {
"message":
'api_upload: Missing uploaded file request: {}'.format(
request_data)
}, 400
schema = UploadJobForm()
try:
properties = schema.load(request_data)
except Exception as err:
app.logger.info('api_upload: Validation error: {}'.format(err))
return {
"message":
'api_upload: Post field validation error: {}'.format(err)
}, 422
fileobj = request_files['file']
filename = secure_filename(fileobj.filename)
filepath = os.path.join(app.config['JOSHUA_UPLOAD_FOLDER'],
secure_filename(properties['username']))
if not os.path.exists(filepath):
os.mkdir(filepath, 0o755)
saved_file = os.path.join(filepath, filename)
fileobj.save(saved_file)
if not tarfile.is_tarfile(saved_file):
os.remove(saved_file)
app.logger.info(
'api_upload: not a valid tar file: {}'.format(saved_file))
return {"error": 'api_upload: not a valid tar file'}, 400
# convert to non-unicode string for username
properties['username'] = str(properties['username'])
with open(saved_file, "rb") as file:
file.seek(0, os.SEEK_END)
size = file.tell()
file.seek(0, os.SEEK_SET)
properties['data_size'] = size
ensemble_id = joshua_model.create_ensemble(properties['username'],
properties, file, False)
app.logger.info('Ensemble {} created with properties: {}'.format(
ensemble_id, properties))
# Delete the file
os.remove(saved_file)
return jsonify(ensemble_id), 200
def test_two_agents(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {
"max_runs": 1,
"timeout": 1
}, open(empty_ensemble, "rb"))
agents = []
for rank in range(2):
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": os.path.join(tmp_path, str(rank)),
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
agents.append(agent)
# before starting agent two, wait until agent one has started on this ensemble
while get_started(joshua_model.db) != 1:
time.sleep(0.001)
joshua.tail_ensemble(ensemble_id, username="******")
@fdb.transactional
def get_started(tr):
return joshua_model._get_snap_counter(tr, ensemble_id, "started")
# The second agent won't have started this ensemble (unless somehow > 10
# seconds passed without the first agent completing the ensemble)
assert get_started(joshua_model.db) == 1
for agent in agents:
agent.join()
def test_ensemble_fails(tmp_path, empty_ensemble_fail):
ensemble_id = joshua_model.create_ensemble(
"joshua", {"max_runs": 1, "timeout": 1}, open(empty_ensemble_fail, "rb")
)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
assert get_passes(joshua_model.db, ensemble_id) == 0
assert get_fails(joshua_model.db, ensemble_id) >= 1
def test_stop_ensemble(tmp_path, empty_ensemble):
"""
:tmp_path: https://docs.pytest.org/en/stable/tmpdir.html
"""
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble(
"joshua", {"max_runs": 1e12}, open(empty_ensemble, "rb")
)
agent = threading.Thread(
target=joshua_agent.agent,
args=(),
kwargs={
"work_dir": tmp_path,
"agent_idle_timeout": 1,
},
)
agent.setDaemon(True)
agent.start()
while len(joshua_model.show_in_progress(ensemble_id)) == 0:
time.sleep(0.001)
joshua.stop_ensemble(ensemble_id, username="******")
assert joshua_model.show_in_progress(ensemble_id) == []
joshua.tail_ensemble(ensemble_id, username="******")
agent.join()
def test_create_ensemble():
assert len(joshua_model.list_active_ensembles()) == 0
ensemble_id = joshua_model.create_ensemble("joshua", {}, io.BytesIO())
assert len(joshua_model.list_active_ensembles()) > 0
