def runner(ctx, period): """Starts a swh-scheduler runner service. This process is responsible for checking for ready-to-run tasks and schedule them.""" from swh.scheduler.celery_backend.config import build_app from swh.scheduler.celery_backend.runner import run_ready_tasks app = build_app(ctx.obj["config"].get("celery")) app.set_current() logger = logging.getLogger(__name__ + ".runner") scheduler = ctx.obj["scheduler"] logger.debug("Scheduler %s" % scheduler) try: while True: logger.debug("Run ready tasks") try: ntasks = len(run_ready_tasks(scheduler, app)) if ntasks: logger.info("Scheduled %s tasks", ntasks) except Exception: logger.exception("Unexpected error in run_ready_tasks()") if not period: break time.sleep(period) except KeyboardInterrupt: ctx.exit(0)
def test_task_exception(swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_scheduler): task_type = swh_scheduler.get_task_type("swh-test-error") assert task_type assert task_type["backend_name"] == TASK_ERROR swh_scheduler.create_tasks([create_task_dict("swh-test-error", "oneshot")]) backend_tasks = run_ready_tasks(swh_scheduler, swh_scheduler_celery_app) assert len(backend_tasks) == 1 task = backend_tasks[0] result = AsyncResult(id=task["backend_id"]) with pytest.raises(NotImplementedError): result.get()
def test_scheduler_fixture(swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_scheduler): "Test that the scheduler fixture works properly" task_type = swh_scheduler.get_task_type("swh-test-ping") assert task_type assert task_type["backend_name"] == TASK_PING swh_scheduler.create_tasks([create_task_dict("swh-test-ping", "oneshot")]) backend_tasks = run_ready_tasks(swh_scheduler, swh_scheduler_celery_app) assert backend_tasks for task in backend_tasks: # Make sure the task completed AsyncResult(id=task["backend_id"]).get()
def test_run_ready_task_with_priority(swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_scheduler): """Ensure scheduler runner schedules priority tasks ready for scheduling""" task_type_name, backend_name = "swh-test-add", TASK_ADD task_type = swh_scheduler.get_task_type(task_type_name) assert task_type assert task_type["backend_name"] == backend_name task_inputs = [ ("oneshot", (10, 22), "low"), ("oneshot", (20, 10), "normal"), ("recurring", (30, 10), "high"), ] tasks = swh_scheduler.create_tasks( create_task_dict(task_type_name, policy, *args, priority=priority) for (policy, args, priority) in task_inputs) assert len(tasks) == len(task_inputs) task_ids = set() for task in tasks: assert task["status"] == "next_run_not_scheduled" assert task["priority"] is not None task_ids.add(task["id"]) backend_tasks = run_ready_tasks(swh_scheduler, swh_scheduler_celery_app, task_types=[], with_priority=True) assert len(backend_tasks) == len(tasks) scheduled_tasks = swh_scheduler.search_tasks(task_type=task_type_name) assert len(scheduled_tasks) == len(tasks) for task in scheduled_tasks: assert task["status"] == "next_run_scheduled" assert task["id"] in task_ids # Ensure each priority task is indeed scheduled to the queue backend for i, (_, args, _) in enumerate(task_inputs): task = backend_tasks[i] value = AsyncResult(id=task["backend_id"]).get() assert value == sum(args)
def runner(ctx, period, task_type_names, with_priority): """Starts a swh-scheduler runner service. This process is responsible for checking for ready-to-run tasks and schedule them.""" from swh.scheduler.celery_backend.config import build_app from swh.scheduler.celery_backend.runner import run_ready_tasks config = ctx.obj["config"] app = build_app(config.get("celery")) app.set_current() logger = logging.getLogger(__name__ + ".runner") scheduler = ctx.obj["scheduler"] logger.debug("Scheduler %s", scheduler) task_types = [] for task_type_name in task_type_names: task_type = scheduler.get_task_type(task_type_name) if not task_type: raise ValueError(f"Unknown {task_type_name}") task_types.append(task_type) try: while True: logger.debug("Run ready tasks") try: ntasks = len( run_ready_tasks(scheduler, app, task_types, with_priority)) if ntasks: logger.info("Scheduled %s tasks", ntasks) except Exception: logger.exception("Unexpected error in run_ready_tasks()") sentry_sdk.capture_exception() if not period: break time.sleep(period) except KeyboardInterrupt: ctx.exit(0)