def start_supervisor_daemon(path, num_procs=None):
from quetz.jobs.runner import Supervisor
from quetz.tasks.workers import get_worker
configure_logger(loggers=("quetz", ))
config = _get_config(path)
manager = get_worker(config, num_procs=num_procs)
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
supervisor = Supervisor(db, manager)
try:
supervisor.run()
except KeyboardInterrupt:
logger.info("stopping supervisor")
finally:
db.close()
async def test_restart_worker_process(
db, user, package_version, supervisor, caplog, items_spec
):
# test if we can resume jobs if a worker was killed/restarted
func_serialized = pickle.dumps(long_running)
job = Job(owner_id=user.id, manifest=func_serialized, items_spec="*")
db.add(job)
db.commit()
supervisor.run_jobs()
supervisor.run_tasks()
db.refresh(job)
task = db.query(Task).one()
assert job.status == JobStatus.running
assert task.status == TaskStatus.pending
# wait for task status to change
for i in range(50):
time.sleep(0.05)
db.refresh(task)
if task.status != TaskStatus.pending:
break
db.refresh(task)
assert task.status == TaskStatus.running
# simulate restart
supervisor = Supervisor(db, supervisor.manager)
db.refresh(job)
assert job.status == JobStatus.running
assert task.status == TaskStatus.failed
assert "failed" in caplog.text
supervisor.run_once()
db.refresh(job)
assert job.status == JobStatus.failed
def sync_supervisor(db, dao, config):
"supervisor with synchronous test worker"
manager = TestWorker(config, db, dao)
supervisor = Supervisor(db, manager)
return supervisor
def supervisor(db, config, manager):
supervisor = Supervisor(db, manager)
return supervisor
def supervisor(config, db, dao):
manager = TestWorker(config, db, dao)
supervisor = Supervisor(db, manager)
return supervisor
def job_supervisor(db, config, dao, dummy_remote_session_object):
manager = TestWorker(config, db, dao, dummy_remote_session_object)
supervisor = Supervisor(db, manager)
return supervisor
def supervisor(config, db):
manager = SubprocessWorker(config)
supervisor = Supervisor(db, manager)
return supervisor