def test_app_dev_cant_use_running_train_job(self):
app_dev = make_app_dev()
app = gen()
train_job_id = make_train_job(app_dev, app=app)
# Will throw error as train job is still running
with pytest.raises(Exception, match='InvalidTrainJobError'):
app_dev.create_inference_job(app)
def test_app_dev_cant_use_private_model(self):
app_dev = make_app_dev()
(task, app, _, train_dataset_id, val_dataset_id, budget) = make_train_job_info(app_dev)
model_id = make_private_model(task=task) # Have private model created
# Can't create train job with private model
with pytest.raises(Exception, match='InvalidModelError'):
app_dev.create_train_job(app, task, train_dataset_id, val_dataset_id, budget, models=[model_id])
def test_app_dev_cant_view_others_job(self, app_dev_create_train_job_and_waited):
(app_dev, app, task, *args) = app_dev_create_train_job_and_waited
app_dev: Client
app_dev_user = app_dev.get_current_user()
app_dev_id = app_dev_user['id']
app_dev2 = make_app_dev()
with pytest.raises(Exception):
app_dev2.get_train_jobs_by_user(app_dev_id)
def test_multiple_app_devs_use_same_app(self, app_dev_create_train_job_and_waited):
(app_dev, app, task, *args) = app_dev_create_train_job_and_waited
app_dev2 = make_app_dev()
(_, _, model_id, train_dataset_id, val_dataset_id, budget) = make_train_job_info(app_dev2, task=task) # Get another set of job info
# App dev 2 create another train job with same app
train_job = app_dev2.create_train_job(app, task, train_dataset_id, val_dataset_id, budget, models=[model_id])
assert train_job['app'] == app
assert train_job['app_version'] == 1 # Should not increment
def app_dev_create_inference_job_and_waited(self):
app_dev = make_app_dev()
(task, app, train_job_id) = make_inference_job_info(app_dev)
# Create inference job
inference_job = app_dev.create_inference_job(app)
assert 'id' in inference_job
wait_for_inference_job_status(app_dev, app, InferenceJobStatus.RUNNING)
return (app_dev, app, inference_job['id'], task)
def app_dev_create_dataset(self):
app_dev = make_app_dev()
(name, task, file_path) = make_dataset_info()
# Create dataset
dataset = app_dev.create_dataset(name, task, file_path)
assert 'id' in dataset
dataset_id = dataset['id']
return (app_dev, dataset_id, task)
def app_dev_create_train_job_and_waited(self):
app_dev = make_app_dev()
(task, app, model_id, train_dataset_id, val_dataset_id, budget) = make_train_job_info(app_dev)
# Create train job
train_job = app_dev.create_train_job(app, task, train_dataset_id, val_dataset_id, budget, models=[model_id])
assert 'id' in train_job
wait_for_train_job_status(app_dev, app, TrainJobStatus.STOPPED)
return (app_dev, app, train_job['id'], task)
def test_app_dev_informed_model_error(self):
app_dev = make_app_dev()
(task, app, _, train_dataset_id, val_dataset_id, budget) = make_train_job_info(app_dev)
model_id = make_invalid_model(task=task) # Have invalid model created
# Create train job
app_dev.create_train_job(app, task, train_dataset_id, val_dataset_id, budget, models=[model_id])
# Train job will be errored
with pytest.raises(Exception, match='errored'):
wait_for_train_job_status(app_dev, app, TrainJobStatus.STOPPED)
def test_app_dev_stop_train_job(self):
app_dev = make_app_dev()
(task, app, model_id, train_dataset_id, val_dataset_id, budget) = make_train_job_info(app_dev)
# Create train job
train_job = app_dev.create_train_job(app, task, train_dataset_id, val_dataset_id, budget, models=[model_id])
assert 'id' in train_job
# Stop train job
app_dev.stop_train_job(app)
# Train job should have stopped
train_job = app_dev.get_train_job(app)
assert train_job['status'] == TrainJobStatus.STOPPED
def test_app_dev_cant_manage_users(self):
ban_email = f'{gen()}@rafiki'
email = gen_email()
password = gen()
model_dev = make_model_dev(email=ban_email)
app_dev = make_app_dev()
# Can't create user
with pytest.raises(Exception):
app_dev.create_user(email, password, UserType.ADMIN)
# Can't list users
with pytest.raises(Exception):
app_dev.get_users()
# Can't ban user
with pytest.raises(Exception):
app_dev.ban_user(ban_email)
def test_admin_ban_users(self):
admin = make_admin()
model_dev_email = gen_email()
app_dev_email = gen_email()
password = gen()
model_dev = make_model_dev(email=model_dev_email, password=password)
app_dev = make_app_dev(email=app_dev_email, password=password)
# Ban both users
admin.ban_user(model_dev_email)
admin.ban_user(app_dev_email)
# Both users cannot login again
with pytest.raises(Exception):
model_dev.login(model_dev_email, password)
with pytest.raises(Exception):
app_dev.login(app_dev_email, password)
def test_app_dev_create_train_job_with_gpu(self):
app_dev = make_app_dev()
(task, app, model_id, train_dataset_id, val_dataset_id, budget) = make_train_job_info(app_dev)
budget[BudgetOption.GPU_COUNT] = 1 # With GPU
# Create train job
train_job = app_dev.create_train_job(app, task, train_dataset_id, val_dataset_id, budget, models=[model_id])
assert 'id' in train_job
# Wait until train job stops
wait_for_train_job_status(app_dev, app, TrainJobStatus.STOPPED)
# Train job should have stopped without error
train_job = app_dev.get_train_job(app)
assert train_job['status'] == TrainJobStatus.STOPPED
# Train job should have trials
trials = app_dev.get_trials_of_train_job(app)
assert len(trials) > 0
def test_app_dev_informed_model_error_in_inference(self):
app_dev = make_app_dev()
task = gen()
model_id = make_model(task=task,
model_file_path=MODEL_INVALID_LOAD_PARAMS_PATH)
(task, app, train_job_id) = make_inference_job_info(app_dev,
task=task,
model_id=model_id)
# Create inference job
inference_job = app_dev.create_inference_job(app)
# Will eventuallly throw error as inference job is no longer running
with pytest.raises(Exception, match='InvalidRunningInferenceJobError'):
wait_for_inference_job_status(app_dev, app,
InferenceJobStatus.ERRORED)
# Inference job errored
inference_jobs = app_dev.get_inference_jobs_of_app(app)
inference_job = next(x for x in inference_jobs
if x['id'] == inference_job['id'])
assert inference_job['status'] == InferenceJobStatus.ERRORED
def test_app_dev_view_public_model(self):
app_dev = make_app_dev()
model_dev = make_model_dev()
(name, task, model_file_path, model_class,
dependencies) = make_model_info()
# Model dev creates public model
model = model_dev.create_model(name,
task,
model_file_path,
model_class,
dependencies,
access_right=ModelAccessRight.PUBLIC)
assert 'id' in model
model_id = model['id']
# App dev sees it in available models
models = app_dev.get_available_models()
assert any([x['id'] == model_id for x in models])
# App dev sees it in available models after filtering by task
models = app_dev.get_available_models(task=task)
assert any([x['id'] == model_id for x in models])