def get_tasks_worker(
background_tasks: BackgroundTasks,
dao: Dao = Depends(get_dao),
auth: authorization.Rules = Depends(get_rules),
session: requests.Session = Depends(get_remote_session),
config: Config = Depends(get_config),
) -> Task:
if config.configured_section("worker"):
worker = config.worker_type
else:
worker = "thread"
if worker == "thread":
worker = ThreadingWorker(background_tasks, dao, auth, session, config)
elif worker == "subprocess":
worker = SubprocessWorker(auth.API_key, auth.session, config)
elif worker == "redis":
if rq_available:
worker = RQManager(
config.worker_redis_ip,
config.worker_redis_port,
config.worker_redis_db,
auth.API_key,
auth.session,
config,
)
else:
raise ValueError("redis and rq not installed on machine")
else:
raise ValueError("wrong configuration in worker.type")
return Task(auth, worker)
def manager(config, db):
manager = SubprocessWorker(config)
yield manager
manager._executor.shutdown()
SubprocessWorker._executor = None
db.query(Job).delete()
db.commit()
def watch_job_queue(
path: str = typer.Argument(None, help="Path to the plugin folder"),
num_procs: Optional[int] = typer.Option(
None, help="Number of processes to use. Default: number of CPU cores"),
) -> None:
import time
configure_logger(loggers=("quetz", ))
from quetz.jobs.runner import check_status, run_jobs, run_tasks
from quetz.tasks.workers import SubprocessWorker
config = _get_config(path)
manager = SubprocessWorker("", {}, config, {'max_workers': num_procs})
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
try:
while True:
run_jobs(db)
run_tasks(db, manager)
check_status(db)
time.sleep(5)
except KeyboardInterrupt:
db.close()
def subprocess_worker(api_key, browser_session, db, config):
SubprocessWorker._executor = None
worker = SubprocessWorker(config)
return worker
def manager(config):
manager = SubprocessWorker("", {}, config)
return manager
def supervisor(config, db):
manager = SubprocessWorker(config)
supervisor = Supervisor(db, manager)
return supervisor