Ejemplo n.º 1
0
async def test_running_task(db, user, package_version, manager):

    func_serialized = pickle.dumps(long_running)
    job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
    db.add(job)
    db.commit()
    run_jobs(db)
    processes = run_tasks(db, manager)
    db.refresh(job)
    task = db.query(Task).one()

    assert job.status == JobStatus.running

    assert task.status == TaskStatus.pending

    time.sleep(0.01)
    check_status(db)

    db.refresh(task)
    assert task.status == TaskStatus.running

    # wait for job to finish
    await processes[0].wait()

    check_status(db)

    db.refresh(task)
    assert task.status == TaskStatus.success
Ejemplo n.º 2
0
def test_package_specs(auth_client, db, config, work_manager, package_version,
                       spec, n_tasks):

    response = auth_client.post(
        "/api/jobs",
        json={
            "items_spec": spec,
            "manifest": "quetz-transmutation:transmutation"
        },
    )

    assert response.status_code == 201
    run_jobs(db)
    run_tasks(db, work_manager)

    check_status(db)

    n_created_task = db.query(Task).count()

    assert n_created_task == n_tasks

    # cleanup
    try:
        db.query(PackageVersion).delete()
        db.query(Job).delete()
        db.query(Task).delete()
    finally:
        db.commit()
Ejemplo n.º 3
0
async def test_create_job(db, user, package_version, manager):

    func_serialized = pickle.dumps(func)
    job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
    db.add(job)
    db.commit()
    run_jobs(db)
    new_jobs = run_tasks(db, manager)
    db.refresh(job)
    task = db.query(Task).one()

    assert job.status == JobStatus.running

    assert task.status == TaskStatus.pending

    # wait for job to finish
    await new_jobs[0].wait()

    check_status(db)

    db.refresh(task)
    assert task.status == TaskStatus.success
    assert os.path.isfile("test-output.txt")

    db.refresh(job)
    assert job.status == JobStatus.success
Ejemplo n.º 4
0
async def test_harvest_endpoint_and_job(
    auth_client, db, config, work_manager, package_version, app, channel_name
):

    async with AsyncClient(app=app, base_url="http://test") as ac:
        response = await ac.put("/api/harvester", json={"package_spec": "*"})

    assert response.status_code == 200
    run_jobs(db)
    new_jobs = run_tasks(db, work_manager)
    await new_jobs[0].wait()

    check_status(db)

    task = db.query(Task).one()
    db.refresh(task)
    assert task.status == TaskStatus.success

    pkgstore = config.get_package_store()
    fileh = pkgstore.serve_path(
        channel_name,
        Path("metadata")
        / package_version.platform
        / package_version.filename.replace(".tar.bz2", ".json"),
    )
    ok_ = fileh.read(10)
    assert ok_

    # cleanup
    try:
        db.query(PackageVersion).delete()
        db.query(Job).delete()
        db.query(Task).delete()
    finally:
        db.commit()
Ejemplo n.º 5
0
async def test_run_tasks_only_on_new_versions(db, user, package_version,
                                              manager, dao, channel_name,
                                              package_name):

    func_serialized = pickle.dumps(dummy_func)
    job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
    db.add(job)
    db.commit()
    run_jobs(db)
    new_jobs = run_tasks(db, manager)
    db.refresh(job)
    task = db.query(Task).one()

    await new_jobs[0].wait()
    check_status(db)
    db.refresh(task)
    db.refresh(job)
    assert task.status == TaskStatus.success
    assert job.status == JobStatus.success

    job.status = JobStatus.pending
    db.commit()
    run_jobs(db)
    new_jobs = run_tasks(db, manager)
    check_status(db)
    db.refresh(job)
    assert not new_jobs
    assert job.status == JobStatus.success

    filename = "test-package-0.2-0.tar.bz2"
    add_package_version(filename, "0.2", channel_name, user, dao, package_name)

    job.status = JobStatus.pending
    db.commit()
    run_jobs(db)
    new_jobs = run_tasks(db, manager)
    assert len(new_jobs) == 1
    assert job.status == JobStatus.running
    assert len(job.tasks) == 2
    assert job.tasks[0].status == TaskStatus.success
    assert job.tasks[1].status == TaskStatus.pending

    # force rerunning
    job.status = JobStatus.pending
    run_jobs(db, force=True)
    db.refresh(job)
    new_jobs = run_tasks(db, manager)
    assert len(job.tasks) == 4
    assert len(new_jobs) == 2
Ejemplo n.º 6
0
async def test_transmutation_endpoint(api_key, db, config, work_manager,
                                      package_version, app, channel_name):

    # we need to use asynchronous http client because the test is async
    async with AsyncClient(app=app, base_url="http://test") as ac:
        response = await ac.post(
            "/api/jobs",
            json={
                "items_spec": "*",
                "manifest": "quetz-transmutation:transmutation"
            },
            headers={"x-api-key": api_key.key},
        )

    assert response.status_code == 201
    run_jobs(db)
    new_jobs = run_tasks(db, work_manager)
    await new_jobs[0].wait()

    check_status(db)

    task = db.query(Task).one()
    db.refresh(task)
    assert task.status == TaskStatus.success

    pkgstore = config.get_package_store()
    fileh = pkgstore.serve_path(
        channel_name,
        Path(package_version.platform) /
        package_version.filename.replace(".tar.bz2", ".conda"),
    )
    ok_ = fileh.read(10)
    assert ok_

    conda_version = (db.query(PackageVersion).filter(
        PackageVersion.package_format == 'conda').one_or_none())

    assert conda_version

    # cleanup
    try:
        db.query(PackageVersion).delete()
        db.query(Job).delete()
        db.query(Task).delete()
    finally:
        db.commit()
Ejemplo n.º 7
0
async def test_failed_task(db, user, package_version, manager):

    func_serialized = pickle.dumps(failed_func)
    job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
    db.add(job)
    db.commit()
    run_jobs(db)
    new_jobs = run_tasks(db, manager)
    task = db.query(Task).one()
    with pytest.raises(Exception, match="some exception"):
        await new_jobs[0].wait()

    check_status(db)

    db.refresh(task)
    assert task.status == TaskStatus.failed

    db.refresh(job)
    assert job.status == JobStatus.success
Ejemplo n.º 8
0
Archivo: cli.py Proyecto: beenje/quetz
def watch_job_queue(
    path: str = typer.Argument(None, help="Path to the plugin folder"),
    num_procs: Optional[int] = typer.Option(
        None, help="Number of processes to use. Default: number of CPU cores"),
) -> None:
    import time

    configure_logger(loggers=("quetz", ))

    from quetz.jobs.runner import check_status, run_jobs, run_tasks
    from quetz.tasks.workers import SubprocessWorker

    config = _get_config(path)
    manager = SubprocessWorker("", {}, config, {'max_workers': num_procs})
    with working_directory(path):
        db = get_session(config.sqlalchemy_database_url)
        try:
            while True:
                run_jobs(db)
                run_tasks(db, manager)
                check_status(db)
                time.sleep(5)
        except KeyboardInterrupt:
            db.close()
Ejemplo n.º 9
0
async def test_restart_worker_process(db, user, package_version, manager,
                                      caplog):
    # test if we can resume jobs if a worker was killed/restarted
    func_serialized = pickle.dumps(long_running)

    job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
    db.add(job)
    db.commit()
    run_jobs(db)
    run_tasks(db, manager)
    db.refresh(job)
    task = db.query(Task).one()

    assert job.status == JobStatus.running

    assert task.status == TaskStatus.pending

    time.sleep(0.01)
    check_status(db)

    db.refresh(task)
    assert task.status == TaskStatus.running

    _process_cache.clear()

    check_status(db)
    assert task.status == TaskStatus.created
    assert "lost" in caplog.text

    new_processes = run_tasks(db, manager)
    db.refresh(task)

    assert len(new_processes) == 1
    assert task.status == TaskStatus.pending

    more_processes = run_tasks(db, manager)
    await new_processes[0].wait()
    even_more_processes = run_tasks(db, manager)
    check_status(db)
    db.refresh(task)
    assert not more_processes
    assert not even_more_processes
    assert task.status == TaskStatus.success