def _schedule_run_bundles(self):
"""
This method implements a state machine. The states are:
STAGED, no worker_run DB entry:
Ready to send run message to available worker.
STARTING, has worker_run DB entry:
Run message sent, waiting for the run to start.
RUNNING, has worker_run DB entry:
Worker reported that the run has started.
READY / FAILED, no worker_run DB entry:
Finished.
"""
workers = WorkerInfoAccessor(self._model, self._worker_model,
self._worker_timeout_seconds - 5)
# Handle some exceptional cases.
self._cleanup_dead_workers(workers)
self._restage_stuck_starting_bundles(workers)
self._bring_offline_stuck_running_bundles(workers)
self._acknowledge_recently_finished_bundles(workers)
# A dictionary structured as {user id : user information} to track those visited user information
user_info_cache = {}
staged_bundles_to_run = self._get_staged_bundles_to_run(
workers, user_info_cache)
# Schedule, preferring user-owned workers.
self._schedule_run_bundles_on_workers(workers, staged_bundles_to_run,
user_info_cache)
def _schedule_run_bundles(self):
"""
This method implements a state machine. The states are:
STAGED, no worker_run DB entry:
Ready to send run message to available worker.
STARTING, has worker_run DB entry:
Run message sent, waiting for the run to start.
RUNNING, has worker_run DB entry:
Worker reported that the run has started.
READY / FAILED, no worker_run DB entry:
Finished.
"""
workers = WorkerInfoAccessor(self._worker_model.get_workers())
# Handle some exceptional cases.
self._cleanup_dead_workers(workers)
self._restage_stuck_starting_bundles(workers)
self._bring_offline_stuck_running_bundles(workers)
self._fail_on_too_many_resources()
self._acknowledge_recently_finished_bundles(workers)
# Schedule, preferring user-owned workers.
self._schedule_run_bundles_on_workers(workers, user_owned=True)
self._schedule_run_bundles_on_workers(workers, user_owned=False)