async def test_running_task(db, user, package_version, manager):
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
processes = run_tasks(db, manager)
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
time.sleep(0.01)
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.running
# wait for job to finish
await processes[0].wait()
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.success
def test_package_specs(auth_client, db, config, work_manager, package_version,
spec, n_tasks):
response = auth_client.post(
"/api/jobs",
json={
"items_spec": spec,
"manifest": "quetz-transmutation:transmutation"
},
)
assert response.status_code == 201
run_jobs(db)
run_tasks(db, work_manager)
check_status(db)
n_created_task = db.query(Task).count()
assert n_created_task == n_tasks
# cleanup
try:
db.query(PackageVersion).delete()
db.query(Job).delete()
db.query(Task).delete()
finally:
db.commit()
async def test_create_job(db, user, package_version, manager):
func_serialized = pickle.dumps(func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
new_jobs = run_tasks(db, manager)
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
# wait for job to finish
await new_jobs[0].wait()
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.success
assert os.path.isfile("test-output.txt")
db.refresh(job)
assert job.status == JobStatus.success
async def test_harvest_endpoint_and_job(
auth_client, db, config, work_manager, package_version, app, channel_name
):
async with AsyncClient(app=app, base_url="http://test") as ac:
response = await ac.put("/api/harvester", json={"package_spec": "*"})
assert response.status_code == 200
run_jobs(db)
new_jobs = run_tasks(db, work_manager)
await new_jobs[0].wait()
check_status(db)
task = db.query(Task).one()
db.refresh(task)
assert task.status == TaskStatus.success
pkgstore = config.get_package_store()
fileh = pkgstore.serve_path(
channel_name,
Path("metadata")
/ package_version.platform
/ package_version.filename.replace(".tar.bz2", ".json"),
)
ok_ = fileh.read(10)
assert ok_
# cleanup
try:
db.query(PackageVersion).delete()
db.query(Job).delete()
db.query(Task).delete()
finally:
db.commit()
async def test_run_tasks_only_on_new_versions(db, user, package_version,
manager, dao, channel_name,
package_name):
func_serialized = pickle.dumps(dummy_func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
new_jobs = run_tasks(db, manager)
db.refresh(job)
task = db.query(Task).one()
await new_jobs[0].wait()
check_status(db)
db.refresh(task)
db.refresh(job)
assert task.status == TaskStatus.success
assert job.status == JobStatus.success
job.status = JobStatus.pending
db.commit()
run_jobs(db)
new_jobs = run_tasks(db, manager)
check_status(db)
db.refresh(job)
assert not new_jobs
assert job.status == JobStatus.success
filename = "test-package-0.2-0.tar.bz2"
add_package_version(filename, "0.2", channel_name, user, dao, package_name)
job.status = JobStatus.pending
db.commit()
run_jobs(db)
new_jobs = run_tasks(db, manager)
assert len(new_jobs) == 1
assert job.status == JobStatus.running
assert len(job.tasks) == 2
assert job.tasks[0].status == TaskStatus.success
assert job.tasks[1].status == TaskStatus.pending
# force rerunning
job.status = JobStatus.pending
run_jobs(db, force=True)
db.refresh(job)
new_jobs = run_tasks(db, manager)
assert len(job.tasks) == 4
assert len(new_jobs) == 2
async def test_transmutation_endpoint(api_key, db, config, work_manager,
package_version, app, channel_name):
# we need to use asynchronous http client because the test is async
async with AsyncClient(app=app, base_url="http://test") as ac:
response = await ac.post(
"/api/jobs",
json={
"items_spec": "*",
"manifest": "quetz-transmutation:transmutation"
},
headers={"x-api-key": api_key.key},
)
assert response.status_code == 201
run_jobs(db)
new_jobs = run_tasks(db, work_manager)
await new_jobs[0].wait()
check_status(db)
task = db.query(Task).one()
db.refresh(task)
assert task.status == TaskStatus.success
pkgstore = config.get_package_store()
fileh = pkgstore.serve_path(
channel_name,
Path(package_version.platform) /
package_version.filename.replace(".tar.bz2", ".conda"),
)
ok_ = fileh.read(10)
assert ok_
conda_version = (db.query(PackageVersion).filter(
PackageVersion.package_format == 'conda').one_or_none())
assert conda_version
# cleanup
try:
db.query(PackageVersion).delete()
db.query(Job).delete()
db.query(Task).delete()
finally:
db.commit()
async def test_failed_task(db, user, package_version, manager):
func_serialized = pickle.dumps(failed_func)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
new_jobs = run_tasks(db, manager)
task = db.query(Task).one()
with pytest.raises(Exception, match="some exception"):
await new_jobs[0].wait()
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.failed
db.refresh(job)
assert job.status == JobStatus.success
def watch_job_queue(
path: str = typer.Argument(None, help="Path to the plugin folder"),
num_procs: Optional[int] = typer.Option(
None, help="Number of processes to use. Default: number of CPU cores"),
) -> None:
import time
configure_logger(loggers=("quetz", ))
from quetz.jobs.runner import check_status, run_jobs, run_tasks
from quetz.tasks.workers import SubprocessWorker
config = _get_config(path)
manager = SubprocessWorker("", {}, config, {'max_workers': num_procs})
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
try:
while True:
run_jobs(db)
run_tasks(db, manager)
check_status(db)
time.sleep(5)
except KeyboardInterrupt:
db.close()
async def test_restart_worker_process(db, user, package_version, manager,
caplog):
# test if we can resume jobs if a worker was killed/restarted
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
run_jobs(db)
run_tasks(db, manager)
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
time.sleep(0.01)
check_status(db)
db.refresh(task)
assert task.status == TaskStatus.running
_process_cache.clear()
check_status(db)
assert task.status == TaskStatus.created
assert "lost" in caplog.text
new_processes = run_tasks(db, manager)
db.refresh(task)
assert len(new_processes) == 1
assert task.status == TaskStatus.pending
more_processes = run_tasks(db, manager)
await new_processes[0].wait()
even_more_processes = run_tasks(db, manager)
check_status(db)
db.refresh(task)
assert not more_processes
assert not even_more_processes
assert task.status == TaskStatus.success