def test_filter_jobs_by_status(auth_client, db, user, status, job_ids):
job0 = Job(
id=0,
items_spec="*",
manifest=b"dummy_func",
status=JobStatus.running,
owner=user,
)
db.add(job0)
job1 = Job(
id=1,
items_spec="*",
manifest=b"dummy_func",
status=JobStatus.pending,
owner=user,
)
db.add(job1)
db.commit()
query = "&".join(["status={}".format(s) for s in status])
response = auth_client.get(f"/api/jobs?{query}")
assert response.status_code == 200
response_data = response.json()
assert {job['id'] for job in response_data['result']} == set(job_ids)
assert response_data['pagination']['all_records_count'] == len(job_ids)
def test_refresh_job(auth_client, user, db, package_version, supervisor):
func_serialized = pickle.dumps(dummy_func)
job = Job(
owner_id=user.id,
manifest=func_serialized,
items_spec="*",
status=JobStatus.success,
)
task = Task(job=job, status=TaskStatus.success, package_version=package_version)
db.add(job)
db.add(task)
db.commit()
assert job.status == JobStatus.success
assert len(job.tasks) == 1
response = auth_client.patch(f"/api/jobs/{job.id}", json={"status": "pending"})
assert response.status_code == 200
db.refresh(job)
assert job.status == JobStatus.pending
assert len(job.tasks) == 1
supervisor.run_jobs()
supervisor.check_status()
assert job.status == JobStatus.success
assert len(job.tasks) == 1
response = auth_client.patch(
f"/api/jobs/{job.id}", json={"status": "pending", "force": True}
)
supervisor.run_jobs()
db.refresh(job)
assert job.status == JobStatus.running
assert len(job.tasks) == 2
# forcing one job should not affect the other
other_job = Job(
id=2,
owner_id=user.id,
manifest=func_serialized,
items_spec="*",
status=JobStatus.success,
)
job.status = JobStatus.pending
db.add(other_job)
db.commit()
response = auth_client.patch(
f"/api/jobs/{other_job.id}", json={"status": "pending", "force": True}
)
db.refresh(job)
assert job.status == JobStatus.pending
assert len(job.tasks) == 2
def test_get_user_jobs(auth_client, db, user, package_version, other_user):
job = Job(items_spec="*", owner=user, manifest=b"dummy_func")
db.add(job)
other_job = Job(items_spec="*", owner=other_user, manifest=b"dummy_func")
db.add(other_job)
db.commit()
response = auth_client.get("/api/jobs")
assert response.status_code == 200
data = response.json()['result']
assert len(data) == 1
async def test_running_task(db, user, package_version, manager):
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
processes = run_tasks(db, manager)
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
time.sleep(0.01)
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.running
# wait for job to finish
await processes[0].wait()
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.success
async def test_running_task(db, user, package_version, supervisor):
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
supervisor.run_jobs()
processes = supervisor.run_tasks()
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
# wait for task status to change
for i in range(50):
time.sleep(0.05)
db.refresh(task)
if task.status != TaskStatus.pending:
break
assert task.status == TaskStatus.running
# wait for job to finish
await processes[0].wait()
supervisor.check_status()
db.refresh(task)
assert task.status == TaskStatus.success
async def test_create_job(db, user, package_version, supervisor):
func_serialized = pickle.dumps(func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
supervisor.run_jobs()
new_jobs = supervisor.run_tasks()
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
# wait for job to finish
await new_jobs[0].wait()
supervisor.check_status()
db.refresh(task)
assert task.status == TaskStatus.success
assert os.path.isfile("test-output.txt")
db.refresh(job)
assert job.status == JobStatus.success
def package_version_job(db, user, package_version):
func_serialized = pickle.dumps(dummy_func)
job = Job(owner=user, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
yield job
db.delete(job)
db.commit()
def test_empty_package_spec(db, user, package_version, caplog, items_spec, supervisor):
func_serialized = pickle.dumps(func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec=items_spec)
db.add(job)
db.commit()
supervisor.run_jobs()
db.refresh(job)
task = db.query(Task).one_or_none()
assert job.status == JobStatus.success
assert task is None
def put_harvest(
package_spec: PackageSpec,
db=Depends(get_db),
auth: authorization.Rules = Depends(get_rules),
):
user = auth.get_user()
job = Job(
owner_id=user,
manifest=harvest_serialized,
items_spec=package_spec.package_spec,
)
db.add(job)
db.commit()
async def test_run_tasks_only_on_new_versions(
db, user, package_version, dao, channel_name, package_name, supervisor
):
func_serialized = pickle.dumps(dummy_func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
supervisor.run_jobs()
new_jobs = supervisor.run_tasks()
db.refresh(job)
task = db.query(Task).one()
await new_jobs[0].wait()
supervisor.check_status()
db.refresh(task)
db.refresh(job)
assert task.status == TaskStatus.success
assert job.status == JobStatus.success
job.status = JobStatus.pending
db.commit()
supervisor.run_jobs()
new_jobs = supervisor.run_tasks()
supervisor.check_status()
db.refresh(job)
assert not new_jobs
assert job.status == JobStatus.success
filename = "test-package-0.2-0.tar.bz2"
add_package_version(filename, "0.2", channel_name, user, dao, package_name)
job.status = JobStatus.pending
db.commit()
supervisor.run_jobs()
new_jobs = supervisor.run_tasks()
assert len(new_jobs) == 1
assert job.status == JobStatus.running
assert len(job.tasks) == 2
assert job.tasks[0].status == TaskStatus.success
assert job.tasks[1].status == TaskStatus.pending
# force rerunning
job.status = JobStatus.pending
supervisor.run_jobs(force=True)
db.refresh(job)
new_jobs = supervisor.run_tasks()
assert len(job.tasks) == 4
assert len(new_jobs) == 2
def test_filter_versions(db, user, package_version, spec, n_tasks, supervisor):
func_serialized = pickle.dumps(func)
job = Job(
owner_id=user.id,
manifest=func_serialized,
items_spec=spec,
)
db.add(job)
db.commit()
supervisor.run_jobs()
supervisor.run_tasks()
db.refresh(job)
n_created_tasks = db.query(Task).count()
assert n_created_tasks == n_tasks
def test_get_tasks(auth_client, db, user, package_version, query_params,
expected_task_count):
job = Job(items_spec="*", owner=user, manifest=pickle.dumps(dummy_func))
db.add(job)
task = Task(job=job, package_version=package_version)
db.add(task)
db.commit()
response = auth_client.get(f"/api/jobs/{job.id}/tasks?{query_params}")
assert response.status_code == 200
data = response.json()
assert len(data['result']) == expected_task_count
if expected_task_count > 0:
assert data['result'][0]['id'] == task.id
assert data['result'][0]['job_id'] == job.id
async def test_failed_task(db, user, package_version, supervisor):
func_serialized = pickle.dumps(failed_func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
supervisor.run_jobs()
new_jobs = supervisor.run_tasks()
task = db.query(Task).one()
with pytest.raises(Exception, match="some exception"):
await new_jobs[0].wait()
supervisor.check_status()
db.refresh(task)
assert task.status == TaskStatus.failed
db.refresh(job)
assert job.status == JobStatus.failed
async def test_restart_worker_process(db, user, package_version, manager,
caplog):
# test if we can resume jobs if a worker was killed/restarted
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
run_tasks(db, manager)
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
time.sleep(0.01)
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.running
_process_cache.clear()
check_status(db)
assert task.status == TaskStatus.created
assert "lost" in caplog.text
new_processes = run_tasks(db, manager)
db.refresh(task)
assert len(new_processes) == 1
assert task.status == TaskStatus.pending
more_processes = run_tasks(db, manager)
await new_processes[0].wait()
even_more_processes = run_tasks(db, manager)
check_status(db)
db.refresh(task)
assert not more_processes
assert not even_more_processes
assert task.status == TaskStatus.success
async def test_restart_worker_process(
db, user, package_version, supervisor, caplog, items_spec
):
# test if we can resume jobs if a worker was killed/restarted
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
supervisor.run_jobs()
supervisor.run_tasks()
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
# wait for task status to change
for i in range(50):
time.sleep(0.05)
db.refresh(task)
if task.status != TaskStatus.pending:
break
db.refresh(task)
assert task.status == TaskStatus.running
# simulate restart
supervisor = Supervisor(db, supervisor.manager)
db.refresh(job)
assert job.status == JobStatus.running
assert task.status == TaskStatus.failed
assert "failed" in caplog.text
supervisor.run_once()
db.refresh(job)
assert job.status == JobStatus.failed
def many_jobs(db, user):
n_jobs = 5
jobs = []
for i in range(n_jobs):
job = Job(
id=i,
items_spec="*",
manifest=b"dummy_func",
status=JobStatus.running,
owner=user,
)
db.add(job)
jobs.append(job)
yield jobs
for job in jobs:
db.delete(job)
db.commit()
def test_jobs_pagination(auth_client, db, user, skip, limit):
n_jobs = 5
for i in range(n_jobs):
job = Job(
id=i,
items_spec="*",
manifest=pickle.dumps(dummy_func),
status=JobStatus.running,
owner=user,
)
db.add(job)
response = auth_client.get(f"/api/jobs?skip={skip}&limit={limit}")
assert response.status_code == 200
jobs = response.json()["result"]
assert jobs[0]['id'] == skip
if limit > 0:
assert jobs[-1]['id'] == min(n_jobs - 1, skip + limit - 1)
assert len(jobs) == min(n_jobs - skip, limit)
else:
assert jobs[-1]['id'] == n_jobs - 1
def create_job(
self,
job_manifest,
user_id,
extra_args={},
start_at: Optional[datetime] = None,
repeat_every_seconds: Optional[int] = None,
):
extra_args_json: Optional[str]
if extra_args:
extra_args_json = json.dumps(extra_args)
else:
extra_args_json = None
job = Job(
manifest=job_manifest,
owner_id=user_id,
extra_args=extra_args_json,
status=JobStatus.pending,
start_at=start_at,
repeat_every_seconds=repeat_every_seconds,
)
self.db.add(job)
self.db.commit()
return job
def test_create_task(db, user, package_version):
job = Job(owner_id=user.id, manifest=b"")
task = Task(job=job)
db.add(job)
db.add(task)
db.commit()
