def __init__(self, args):
for name, val in args._get_kwargs():
setattr(self, name, val)
self.run_id = os.environ["RUN_ID"]
self.logged_waiting = False
self.waiting = set()
self.lock = locklib.Lock(locklib.RUN_STATUS, timeout=RUN_STATUS_LOCK_TIMEOUT)
def _run_trials(batch_run, trials):
trial_runs = _init_trial_runs(batch_run, trials)
run_status_lock = locklib.Lock(locklib.RUN_STATUS,
timeout=RUN_STATUS_LOCK_TIMEOUT)
for trial_run in trial_runs:
if __batch_exiting.is_set():
break
_try_run_pending_trial(trial_run, batch_run, run_status_lock)
def __init__(
self,
start_run_cb,
is_queue_cb,
name="queue",
init_cb=None,
can_start_cb=None,
wait_for_running_cb=None,
sync_state_cb=None,
cleanup_cb=None,
max_startable_runs=None,
run_once=False,
wait_for_running=False,
gpus=None,
poll_interval=DEFAULT_POLL_INTERVAL,
run_status_lock_timeout=DEFAULT_RUN_STATUS_LOCK_TIMEOUT,
):
self.start_run_cb = start_run_cb
self.is_queue_cb = is_queue_cb
self.name = name
self.init_cb = init_cb
self.can_start_cb = can_start_cb
self.wait_for_running_cb = wait_for_running_cb
self.sync_state_cb = sync_state_cb
self.cleanup_cb = cleanup_cb
self.max_startable_runs = max_startable_runs
self.run_once = run_once
self.wait_for_running = wait_for_running
self.gpus = gpus
self.poll_interval = poll_interval
self.run_id = os.environ["RUN_ID"]
self._logged_gpu_mismatch = set()
self.waiting = set()
self.logged_waiting = False
self.run_status_lock = locklib.Lock(locklib.RUN_STATUS,
timeout=run_status_lock_timeout)
self.started = 0
def _run_trials(batch_run, trials):
trial_runs = _init_trial_runs(batch_run, trials)
run_status_lock = locklib.Lock(locklib.RUN_STATUS,
timeout=RUN_STATUS_LOCK_TIMEOUT)
for trial_run in trial_runs:
_try_run_staged_trial(trial_run, batch_run, run_status_lock)