def test_filter_jobs_by_status(auth_client, db, user, status, job_ids): job0 = Job( id=0, items_spec="*", manifest=b"dummy_func", status=JobStatus.running, owner=user, ) db.add(job0) job1 = Job( id=1, items_spec="*", manifest=b"dummy_func", status=JobStatus.pending, owner=user, ) db.add(job1) db.commit() query = "&".join(["status={}".format(s) for s in status]) response = auth_client.get(f"/api/jobs?{query}") assert response.status_code == 200 response_data = response.json() assert {job['id'] for job in response_data['result']} == set(job_ids) assert response_data['pagination']['all_records_count'] == len(job_ids)
def test_refresh_job(auth_client, user, db, package_version, supervisor): func_serialized = pickle.dumps(dummy_func) job = Job( owner_id=user.id, manifest=func_serialized, items_spec="*", status=JobStatus.success, ) task = Task(job=job, status=TaskStatus.success, package_version=package_version) db.add(job) db.add(task) db.commit() assert job.status == JobStatus.success assert len(job.tasks) == 1 response = auth_client.patch(f"/api/jobs/{job.id}", json={"status": "pending"}) assert response.status_code == 200 db.refresh(job) assert job.status == JobStatus.pending assert len(job.tasks) == 1 supervisor.run_jobs() supervisor.check_status() assert job.status == JobStatus.success assert len(job.tasks) == 1 response = auth_client.patch( f"/api/jobs/{job.id}", json={"status": "pending", "force": True} ) supervisor.run_jobs() db.refresh(job) assert job.status == JobStatus.running assert len(job.tasks) == 2 # forcing one job should not affect the other other_job = Job( id=2, owner_id=user.id, manifest=func_serialized, items_spec="*", status=JobStatus.success, ) job.status = JobStatus.pending db.add(other_job) db.commit() response = auth_client.patch( f"/api/jobs/{other_job.id}", json={"status": "pending", "force": True} ) db.refresh(job) assert job.status == JobStatus.pending assert len(job.tasks) == 2
def test_get_user_jobs(auth_client, db, user, package_version, other_user): job = Job(items_spec="*", owner=user, manifest=b"dummy_func") db.add(job) other_job = Job(items_spec="*", owner=other_user, manifest=b"dummy_func") db.add(other_job) db.commit() response = auth_client.get("/api/jobs") assert response.status_code == 200 data = response.json()['result'] assert len(data) == 1
async def test_run_tasks_only_on_new_versions( db, user, package_version, dao, channel_name, package_name, supervisor ): func_serialized = pickle.dumps(dummy_func) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() supervisor.run_jobs() new_jobs = supervisor.run_tasks() db.refresh(job) task = db.query(Task).one() await new_jobs[0].wait() supervisor.check_status() db.refresh(task) db.refresh(job) assert task.status == TaskStatus.success assert job.status == JobStatus.success job.status = JobStatus.pending db.commit() supervisor.run_jobs() new_jobs = supervisor.run_tasks() supervisor.check_status() db.refresh(job) assert not new_jobs assert job.status == JobStatus.success filename = "test-package-0.2-0.tar.bz2" add_package_version(filename, "0.2", channel_name, user, dao, package_name) job.status = JobStatus.pending db.commit() supervisor.run_jobs() new_jobs = supervisor.run_tasks() assert len(new_jobs) == 1 assert job.status == JobStatus.running assert len(job.tasks) == 2 assert job.tasks[0].status == TaskStatus.success assert job.tasks[1].status == TaskStatus.pending # force rerunning job.status = JobStatus.pending supervisor.run_jobs(force=True) db.refresh(job) new_jobs = supervisor.run_tasks() assert len(job.tasks) == 4 assert len(new_jobs) == 2
async def test_create_job(db, user, package_version, supervisor): func_serialized = pickle.dumps(func) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() supervisor.run_jobs() new_jobs = supervisor.run_tasks() db.refresh(job) task = db.query(Task).one() assert job.status == JobStatus.running assert task.status == TaskStatus.pending # wait for job to finish await new_jobs[0].wait() supervisor.check_status() db.refresh(task) assert task.status == TaskStatus.success assert os.path.isfile("test-output.txt") db.refresh(job) assert job.status == JobStatus.success
async def test_running_task(db, user, package_version, manager): func_serialized = pickle.dumps(long_running) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() run_jobs(db) processes = run_tasks(db, manager) db.refresh(job) task = db.query(Task).one() assert job.status == JobStatus.running assert task.status == TaskStatus.pending time.sleep(0.01) check_status(db) db.refresh(task) assert task.status == TaskStatus.running # wait for job to finish await processes[0].wait() check_status(db) db.refresh(task) assert task.status == TaskStatus.success
async def test_running_task(db, user, package_version, supervisor): func_serialized = pickle.dumps(long_running) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() supervisor.run_jobs() processes = supervisor.run_tasks() db.refresh(job) task = db.query(Task).one() assert job.status == JobStatus.running assert task.status == TaskStatus.pending # wait for task status to change for i in range(50): time.sleep(0.05) db.refresh(task) if task.status != TaskStatus.pending: break assert task.status == TaskStatus.running # wait for job to finish await processes[0].wait() supervisor.check_status() db.refresh(task) assert task.status == TaskStatus.success
def package_version_job(db, user, package_version): func_serialized = pickle.dumps(dummy_func) job = Job(owner=user, manifest=func_serialized, items_spec="*") db.add(job) db.commit() yield job db.delete(job) db.commit()
def test_empty_package_spec(db, user, package_version, caplog, items_spec, supervisor): func_serialized = pickle.dumps(func) job = Job(owner_id=user.id, manifest=func_serialized, items_spec=items_spec) db.add(job) db.commit() supervisor.run_jobs() db.refresh(job) task = db.query(Task).one_or_none() assert job.status == JobStatus.success assert task is None
def put_harvest( package_spec: PackageSpec, db=Depends(get_db), auth: authorization.Rules = Depends(get_rules), ): user = auth.get_user() job = Job( owner_id=user, manifest=harvest_serialized, items_spec=package_spec.package_spec, ) db.add(job) db.commit()
def update_job( job_data: JobUpdateModel, db=Depends(get_db), job: job_db_models.Job = Depends(get_job_or_fail), auth: authorization.Rules = Depends(get_rules), ): """refresh job (re-run on new packages)""" auth.assert_jobs() job.status = job_data.status # type: ignore # ignore tasks that have already been run if job_data.force: run_jobs(db, job_id=job.id, force=True) db.commit()
def test_filter_versions(db, user, package_version, spec, n_tasks, supervisor): func_serialized = pickle.dumps(func) job = Job( owner_id=user.id, manifest=func_serialized, items_spec=spec, ) db.add(job) db.commit() supervisor.run_jobs() supervisor.run_tasks() db.refresh(job) n_created_tasks = db.query(Task).count() assert n_created_tasks == n_tasks
def test_get_tasks(auth_client, db, user, package_version, query_params, expected_task_count): job = Job(items_spec="*", owner=user, manifest=pickle.dumps(dummy_func)) db.add(job) task = Task(job=job, package_version=package_version) db.add(task) db.commit() response = auth_client.get(f"/api/jobs/{job.id}/tasks?{query_params}") assert response.status_code == 200 data = response.json() assert len(data['result']) == expected_task_count if expected_task_count > 0: assert data['result'][0]['id'] == task.id assert data['result'][0]['job_id'] == job.id
async def test_restart_worker_process(db, user, package_version, manager, caplog): # test if we can resume jobs if a worker was killed/restarted func_serialized = pickle.dumps(long_running) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() run_jobs(db) run_tasks(db, manager) db.refresh(job) task = db.query(Task).one() assert job.status == JobStatus.running assert task.status == TaskStatus.pending time.sleep(0.01) check_status(db) db.refresh(task) assert task.status == TaskStatus.running _process_cache.clear() check_status(db) assert task.status == TaskStatus.created assert "lost" in caplog.text new_processes = run_tasks(db, manager) db.refresh(task) assert len(new_processes) == 1 assert task.status == TaskStatus.pending more_processes = run_tasks(db, manager) await new_processes[0].wait() even_more_processes = run_tasks(db, manager) check_status(db) db.refresh(task) assert not more_processes assert not even_more_processes assert task.status == TaskStatus.success
async def test_failed_task(db, user, package_version, supervisor): func_serialized = pickle.dumps(failed_func) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() supervisor.run_jobs() new_jobs = supervisor.run_tasks() task = db.query(Task).one() with pytest.raises(Exception, match="some exception"): await new_jobs[0].wait() supervisor.check_status() db.refresh(task) assert task.status == TaskStatus.failed db.refresh(job) assert job.status == JobStatus.failed
def update_job( job_data: JobUpdateModel, db=Depends(get_db), job: job_db_models.Job = Depends(get_job_or_fail), auth: authorization.Rules = Depends(get_rules), ): """refresh job (re-run on new packages)""" auth.assert_jobs(owner_id=job.owner_id) job.status = job_data.status # type: ignore if job_data.force and job.status in [ JobStatus.running, JobStatus.pending, ]: # restart tasks that have already been run for task in job.tasks: task.status = "skipped" db.commit()
async def test_restart_worker_process( db, user, package_version, supervisor, caplog, items_spec ): # test if we can resume jobs if a worker was killed/restarted func_serialized = pickle.dumps(long_running) job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*") db.add(job) db.commit() supervisor.run_jobs() supervisor.run_tasks() db.refresh(job) task = db.query(Task).one() assert job.status == JobStatus.running assert task.status == TaskStatus.pending # wait for task status to change for i in range(50): time.sleep(0.05) db.refresh(task) if task.status != TaskStatus.pending: break db.refresh(task) assert task.status == TaskStatus.running # simulate restart supervisor = Supervisor(db, supervisor.manager) db.refresh(job) assert job.status == JobStatus.running assert task.status == TaskStatus.failed assert "failed" in caplog.text supervisor.run_once() db.refresh(job) assert job.status == JobStatus.failed
def many_jobs(db, user): n_jobs = 5 jobs = [] for i in range(n_jobs): job = Job( id=i, items_spec="*", manifest=b"dummy_func", status=JobStatus.running, owner=user, ) db.add(job) jobs.append(job) yield jobs for job in jobs: db.delete(job) db.commit()
def test_jobs_pagination(auth_client, db, user, skip, limit): n_jobs = 5 for i in range(n_jobs): job = Job( id=i, items_spec="*", manifest=pickle.dumps(dummy_func), status=JobStatus.running, owner=user, ) db.add(job) response = auth_client.get(f"/api/jobs?skip={skip}&limit={limit}") assert response.status_code == 200 jobs = response.json()["result"] assert jobs[0]['id'] == skip if limit > 0: assert jobs[-1]['id'] == min(n_jobs - 1, skip + limit - 1) assert len(jobs) == min(n_jobs - skip, limit) else: assert jobs[-1]['id'] == n_jobs - 1
def create_job( self, job_manifest, user_id, extra_args={}, start_at: Optional[datetime] = None, repeat_every_seconds: Optional[int] = None, ): extra_args_json: Optional[str] if extra_args: extra_args_json = json.dumps(extra_args) else: extra_args_json = None job = Job( manifest=job_manifest, owner_id=user_id, extra_args=extra_args_json, status=JobStatus.pending, start_at=start_at, repeat_every_seconds=repeat_every_seconds, ) self.db.add(job) self.db.commit() return job
def test_create_task(db, user, package_version): job = Job(owner_id=user.id, manifest=b"") task = Task(job=job) db.add(job) db.add(task) db.commit()