def wait_kill_tasks(self, context, scheduler, job_key, instances=None):
monitor = JobMonitor(scheduler, job_key)
if not monitor.wait_until(
JobMonitor.terminal, instances=instances, with_timeout=True):
context.print_err("Tasks were not killed in time.")
return EXIT_TIMEOUT
return EXIT_OK
def execute(self, context):
config = context.get_job_config(context.options.jobspec, context.options.config_file)
if config.raw().has_cron_schedule():
raise context.CommandError(
EXIT_COMMAND_FAILURE,
"Cron jobs may only be scheduled with \"aurora cron schedule\" command")
api = context.get_api(config.cluster())
resp = api.create_job(config)
context.log_response_and_raise(resp, err_code=EXIT_COMMAND_FAILURE,
err_msg="Job creation failed due to error:")
if context.options.open_browser:
webbrowser.open_new_tab(get_job_page(api, context.options.jobspec))
if context.options.wait_until == "RUNNING":
JobMonitor(api.scheduler_proxy, config.job_key()).wait_until(JobMonitor.running_or_finished)
elif context.options.wait_until == "FINISHED":
JobMonitor(api.scheduler_proxy, config.job_key()).wait_until(JobMonitor.terminal)
# Check to make sure the job was created successfully.
status_response = api.check_status(config.job_key())
if (status_response.responseCode is not ResponseCode.OK or
status_response.result.scheduleStatusResult.tasks is None or
status_response.result.scheduleStatusResult.tasks == []):
context.print_err("Error occurred while creating job %s" % context.options.jobspec)
return EXIT_COMMAND_FAILURE
else:
context.print_out("Job create succeeded: job url=%s" %
get_job_page(api, context.options.jobspec))
return EXIT_OK
def test_terminate(self):
mock_event = mock.Mock()
monitor = JobMonitor(self._scheduler,
self._job_key,
terminating_event=mock_event)
monitor.terminate()
mock_event.set.assert_called_once_with()
def execute(self, context):
config = context.get_job_config(context.options.jobspec,
context.options.config_file)
api = context.get_api(config.cluster())
resp = api.create_job(config)
context.log_response(resp)
if resp.responseCode == ResponseCode.INVALID_REQUEST:
context.print_err("job create failed because job not found")
raise context.CommandError(EXIT_INVALID_PARAMETER, "Job not found")
elif resp.responseCode == ResponseCode.ERROR:
context.print_err("job create failed because of scheduler error")
raise context.CommandError(
EXIT_COMMAND_FAILURE,
"Error reported by scheduler; see log for details")
if context.options.open_browser:
context.open_job_page(api, config)
if context.options.wait_until == "RUNNING":
JobMonitor(api.scheduler_proxy, config.job_key()).wait_until(
JobMonitor.running_or_finished)
elif context.options.wait_until == "FINISHED":
JobMonitor(api.scheduler_proxy,
config.job_key()).wait_until(JobMonitor.terminal)
context.print_out("job create succeeded: job url=%s" %
context.get_job_page(api, context.options.jobspec))
return EXIT_OK
def test_wait_with_instances(self):
self.mock_get_tasks([
self.create_task(ScheduleStatus.FAILED, '2'),
])
monitor = JobMonitor(self._scheduler, self._job_key)
assert monitor.wait_until(monitor.terminal, instances=[2])
self.expect_task_status(once=True, instances=[2])
def test_wait_with_instances(self):
self.mock_get_tasks([
self.create_task(ScheduleStatus.FAILED, '2'),
])
monitor = JobMonitor(self._scheduler, self._job_key)
assert monitor.wait_until(monitor.terminal, instances=[2])
self.expect_task_status(once=True, instances=[2])
def test_wait_until_state(self):
self.mock_get_tasks([
self.create_task(ScheduleStatus.RUNNING, '1'),
self.create_task(ScheduleStatus.RUNNING, '2'),
self.create_task(ScheduleStatus.FAILED, '3'),
])
monitor = JobMonitor(self._scheduler, self._job_key)
assert monitor.wait_until(monitor.running_or_finished)
self.expect_task_status(once=True)
def test_wait_until_timeout(self):
self.mock_get_tasks([
self.create_task(ScheduleStatus.RUNNING, '1'),
self.create_task(ScheduleStatus.RUNNING, '2'),
self.create_task(ScheduleStatus.RUNNING, '3'),
])
monitor = JobMonitor(self._scheduler, self._job_key, terminating_event=self._event)
assert not monitor.wait_until(monitor.terminal, with_timeout=True)
self.expect_task_status()
def test_wait_until_state(self):
self.mock_get_tasks([
self.create_task(ScheduleStatus.RUNNING, '1'),
self.create_task(ScheduleStatus.RUNNING, '2'),
self.create_task(ScheduleStatus.FAILED, '3'),
])
monitor = JobMonitor(self._scheduler, self._job_key)
assert monitor.wait_until(monitor.running_or_finished)
self.expect_task_status(once=True)
def test_wait_until_timeout(self):
self.mock_get_tasks([
self.create_task(ScheduleStatus.RUNNING, '1'),
self.create_task(ScheduleStatus.RUNNING, '2'),
self.create_task(ScheduleStatus.RUNNING, '3'),
])
monitor = JobMonitor(self._scheduler,
self._job_key,
terminating_event=self._event)
assert not monitor.wait_until(monitor.terminal, with_timeout=True)
self.expect_task_status()
def execute(self, context):
config = context.get_job_config(context.options.jobspec, context.options.config_file)
api = context.get_api(config.cluster())
monitor = JobMonitor(api, config.role(), config.environment(), config.name())
resp = api.create_job(config)
if resp.responseCode == ResponseCode.INVALID_REQUEST:
raise context.CommandError(EXIT_INVALID_PARAMETER, 'Job not found')
elif resp.responseCode == ResponseCode.ERROR:
raise context.CommandError(EXIT_COMMAND_FAILURE, resp.message)
if context.options.open_browser:
context.open_job_page(api, config)
if context.options.wait_until == 'RUNNING':
monitor.wait_until(monitor.running_or_finished)
elif context.options.wait_until == 'FINISHED':
monitor.wait_until(monitor.terminal)
return EXIT_OK
def really_create(job_spec, config_file, options):
try:
config = get_job_config(job_spec, config_file, options)
except ValueError as v:
print("Error: %s" % v)
sys.exit(1)
api = make_client(config.cluster())
resp = api.create_job(config)
check_and_log_response(resp)
handle_open(api.scheduler_proxy.scheduler_client().url, config.role(),
config.environment(), config.name())
if options.wait_until == 'RUNNING':
JobMonitor(api.scheduler_proxy,
config.job_key()).wait_until(JobMonitor.running_or_finished)
elif options.wait_until == 'FINISHED':
JobMonitor(api.scheduler_proxy,
config.job_key()).wait_until(JobMonitor.terminal)
def wait_until(wait_until_option, job_key, api, instances=None):
"""Waits for job instances to reach specified status.
:param wait_until_option: Expected instance status
:type wait_until_option: ADD_INSTANCE_WAIT_OPTION
:param job_key: Job key to wait on
:type job_key: AuroraJobKey
:param api: Aurora scheduler API
:type api: AuroraClientAPI
:param instances: Specific instances to wait on
:type instances: set of int
"""
if wait_until_option == "RUNNING":
JobMonitor(api.scheduler_proxy, job_key).wait_until(
JobMonitor.running_or_finished,
instances=instances)
elif wait_until_option == "FINISHED":
JobMonitor(api.scheduler_proxy, job_key).wait_until(JobMonitor.terminal, instances=instances)
def create(job_spec, config_file):
"""usage: create cluster/role/env/job config
Creates a job based on a configuration file.
"""
options = app.get_options()
try:
config = get_job_config(job_spec, config_file, options)
except ValueError as v:
print("Error: %s" % v)
sys.exit(1)
api = make_client(config.cluster())
monitor = JobMonitor(api, config.role(), config.environment(), config.name())
resp = api.create_job(config)
check_and_log_response(resp)
handle_open(api.scheduler.scheduler().url, config.role(), config.environment(), config.name())
if options.wait_until == 'RUNNING':
monitor.wait_until(monitor.running_or_finished)
elif options.wait_until == 'FINISHED':
monitor.wait_until(monitor.terminal)
def test_empty_job_succeeds(self):
self.mock_get_tasks([])
monitor = JobMonitor(self._scheduler, self._job_key)
assert monitor.wait_until(monitor.running_or_finished)
self.expect_task_status(once=True)
def test_terminated_exits_immediately(self):
self._event.set()
monitor = JobMonitor(self._scheduler,
self._job_key,
terminating_event=self._event)
assert monitor.wait_until(monitor.terminal)
def wait_kill_tasks(self, context, scheduler, job_key, instances=None):
monitor = JobMonitor(scheduler, job_key)
if not monitor.wait_until(JobMonitor.terminal, instances=instances, with_timeout=True):
context.print_err("Instances %s were not killed in time" % instances)
return EXIT_TIMEOUT
return EXIT_OK
def wait_kill_tasks(scheduler, job_key, instances=None):
monitor = JobMonitor(scheduler, job_key)
if not monitor.wait_until(monitor.terminal, instances=instances, with_timeout=True):
die('Tasks were not killed in time.')
def test_empty_job_succeeds(self):
self.mock_get_tasks([])
monitor = JobMonitor(self._scheduler, self._job_key)
assert monitor.wait_until(monitor.running_or_finished)
self.expect_task_status(once=True)
def wait_kill_tasks(scheduler, job_key, instances=None):
monitor = JobMonitor(scheduler, job_key)
if not monitor.wait_until(
monitor.terminal, instances=instances, with_timeout=True):
die('Tasks were not killed in time.')
def test_terminated_exits_immediately(self): self._event.set() monitor = JobMonitor(self._scheduler, self._job_key, terminating_event=self._event) assert monitor.wait_until(monitor.terminal)
def test_terminate(self): mock_event = mock.Mock() monitor = JobMonitor(self._scheduler, self._job_key, terminating_event=mock_event) monitor.terminate() mock_event.set.assert_called_once_with()
