def test_cleanup_on_failure(self):
config = BASIC_CONFIG + dedent(
"""
jobs:
- name: "failjob"
node: local
schedule: "constant"
actions:
- name: "failaction"
command: "failplz"
"""
) + TOUCH_CLEANUP_FMT
self.start_with_config(config)
action_run_url = self.client.get_url('MASTER.failjob.0.failaction')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.FAILED,
)
action_run_url = self.client.get_url('MASTER.failjob.1.cleanup')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.SUCCEEDED,
)
job_runs = self.client.job(self.client.get_url('MASTER.failjob'),
)['runs']
assert_gt(len(job_runs), 1)
def test_cleanup_on_failure(self):
config = BASIC_CONFIG + dedent("""
jobs:
- name: "failjob"
node: local
schedule: "constant"
actions:
- name: "failaction"
command: "failplz"
""") + TOUCH_CLEANUP_FMT
self.start_with_config(config)
action_run_url = self.client.get_url('MASTER.failjob.0.failaction')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.FAILED,
)
action_run_url = self.client.get_url('MASTER.failjob.1.cleanup')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.SUCCEEDED,
)
job_runs = self.client.job(
self.client.get_url('MASTER.failjob'), )['runs']
assert_gt(len(job_runs), 1)
def test_tronctl_with_job(self):
self.start_with_config(SINGLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT)
job_name = 'MASTER.echo_job'
job_url = self.client.get_url(job_name)
self.sandbox.tronctl('start', job_name)
cleanup_url = self.client.get_url('MASTER.echo_job.1.cleanup')
sandbox.wait_on_state(
self.client.action_runs,
cleanup_url,
actionrun.ActionRun.SUCCEEDED,
)
action_run_url = self.client.get_url('MASTER.echo_job.1.echo_action')
assert_equal(
self.client.action_runs(action_run_url)['state'],
actionrun.ActionRun.SUCCEEDED,
)
job_run_url = self.client.get_url('MASTER.echo_job.1')
assert_equal(
self.client.job_runs(job_run_url)['state'],
actionrun.ActionRun.SUCCEEDED,
)
assert_equal(self.client.job(job_url)['status'], 'enabled')
self.sandbox.tronctl('disable', job_name)
sandbox.wait_on_state(self.client.job, job_url, 'disabled', 'status')
def test_job_queueing_false_with_overlap(self):
"""Test that a job that has queueing false properly cancels an
overlapping job run.
"""
config = BASIC_CONFIG + dedent("""
jobs:
- name: "cancel_overlap"
schedule: "interval 1s"
queueing: False
node: local
actions:
- name: "do_something"
command: "sleep 3s"
- name: "do_other"
command: "sleep 3s"
cleanup_action:
command: "echo done"
""")
self.start_with_config(config)
job_url = self.client.get_url('MASTER.cancel_overlap')
job_run_url = self.client.get_url('MASTER.cancel_overlap.1')
def wait_on_job_schedule():
return len(self.client.job(job_url)['runs']) == 2
sandbox.wait_on_sandbox(wait_on_job_schedule)
sandbox.wait_on_state(self.client.job, job_run_url,
actionrun.ActionRun.STATE_CANCELLED.name)
action_run_states = [action_run['state'] for action_run in
self.client.job_runs(job_run_url)['runs']]
expected = [actionrun.ActionRun.STATE_CANCELLED.name
for _ in xrange(len(action_run_states))]
assert_equal(action_run_states, expected)
def test_trond_restart_job_running_with_dependencies(self):
config = BASIC_CONFIG + textwrap.dedent("""
jobs:
- name: complex_job
node: local
schedule: interval 10min
actions:
- name: first_act
command: sleep 20 && echo "I'm waiting"
- name: following_act
command: echo "thing"
requires: ['first_act']
- name: last_act
command: echo foo
requires: ['following_act']
""")
self.start_with_config(config)
job_name = 'MASTER.complex_job'
self.sandbox.tronctl('start', job_name)
action_run_url = self.client.get_url('MASTER.complex_job.1.first_act')
sandbox.wait_on_state(self.client.action_runs, action_run_url,
actionrun.ActionRun.STATE_RUNNING.name)
self.restart_trond()
assert_equal(self.client.job_runs(action_run_url)['state'],
actionrun.ActionRun.STATE_UNKNOWN.name)
for followup_action_run in ('following_act', 'last_act'):
url = self.client.get_url('%s.1.%s' % (job_name, followup_action_run))
assert_equal(self.client.action_runs(url)['state'],
actionrun.ActionRun.STATE_QUEUED.name)
def test_skip_failed_actions(self):
config = BASIC_CONFIG + dedent("""
jobs:
- name: "multi_step_job"
node: local
schedule: "constant"
actions:
- name: "broken"
command: "failingcommand"
- name: "works"
command: "echo ok"
requires: [broken]
""")
self.start_with_config(config)
action_run_url = self.client.get_url('MASTER.multi_step_job.0.broken')
waiter = sandbox.build_waiter_func(self.client.action_runs, action_run_url)
waiter(actionrun.ActionRun.STATE_FAILED.name)
self.sandbox.tronctl('skip', 'MASTER.multi_step_job.0.broken')
waiter(actionrun.ActionRun.STATE_SKIPPED.name)
action_run_url = self.client.get_url('MASTER.multi_step_job.0.works')
sandbox.wait_on_state(self.client.action_runs, action_run_url,
actionrun.ActionRun.STATE_SUCCEEDED.name)
job_run_url = self.client.get_url('MASTER.multi_step_job.0')
sandbox.wait_on_state(self.client.job_runs, job_run_url,
actionrun.ActionRun.STATE_SUCCEEDED.name)
def test_trond_restart_job_with_run_history(self):
config = BASIC_CONFIG + textwrap.dedent("""
jobs:
- name: fast_job
node: local
schedule: constant
actions:
- name: single_act
command: "sleep 20 && echo good"
""")
self.start_with_config(config)
action_run_url = self.client.get_url('MASTER.fast_job.0.single_act')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.RUNNING,
)
self.restart_trond()
assert_equal(
self.client.job_runs(action_run_url)['state'],
actionrun.ActionRun.UNKNOWN,
)
next_run_url = self.client.get_url('MASTER.fast_job.-1.single_act')
sandbox.wait_on_state(
self.client.action_runs,
next_run_url,
actionrun.ActionRun.RUNNING,
)
def test_skip_failed_actions(self):
config = BASIC_CONFIG + dedent("""
jobs:
- name: "multi_step_job"
node: local
schedule: "constant"
actions:
- name: "broken"
command: "failingcommand"
- name: "works"
command: "echo ok"
requires: [broken]
""")
self.start_with_config(config)
action_run_url = self.client.get_url('MASTER.multi_step_job.0.broken')
waiter = sandbox.build_waiter_func(self.client.action_runs,
action_run_url)
waiter(actionrun.ActionRun.STATE_FAILED.name)
self.sandbox.tronctl('skip', 'MASTER.multi_step_job.0.broken')
waiter(actionrun.ActionRun.STATE_SKIPPED.name)
action_run_url = self.client.get_url('MASTER.multi_step_job.0.works')
sandbox.wait_on_state(self.client.action_runs, action_run_url,
actionrun.ActionRun.STATE_SUCCEEDED.name)
job_run_url = self.client.get_url('MASTER.multi_step_job.0')
sandbox.wait_on_state(self.client.job_runs, job_run_url,
actionrun.ActionRun.STATE_SUCCEEDED.name)
def test_trond_restart_job_with_run_history(self):
config = BASIC_CONFIG + textwrap.dedent(
"""
jobs:
- name: fast_job
node: local
schedule: constant
actions:
- name: single_act
command: "sleep 20 && echo good"
"""
)
self.start_with_config(config)
action_run_url = self.client.get_url('MASTER.fast_job.0.single_act')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.RUNNING,
)
self.restart_trond()
assert_equal(
self.client.job_runs(action_run_url)['state'],
actionrun.ActionRun.UNKNOWN,
)
next_run_url = self.client.get_url('MASTER.fast_job.-1.single_act')
sandbox.wait_on_state(
self.client.action_runs,
next_run_url,
actionrun.ActionRun.RUNNING,
)
def test_tronctl_with_job(self):
self.start_with_config(SINGLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT)
job_name = 'MASTER.echo_job'
job_url = self.client.get_url(job_name)
self.sandbox.tronctl('start', job_name)
cleanup_url = self.client.get_url('MASTER.echo_job.1.cleanup')
sandbox.wait_on_state(
self.client.action_runs,
cleanup_url,
actionrun.ActionRun.SUCCEEDED,
)
action_run_url = self.client.get_url('MASTER.echo_job.1.echo_action')
assert_equal(
self.client.action_runs(action_run_url)['state'],
actionrun.ActionRun.SUCCEEDED,
)
job_run_url = self.client.get_url('MASTER.echo_job.1')
assert_equal(
self.client.job_runs(job_run_url)['state'],
actionrun.ActionRun.SUCCEEDED,
)
assert_equal(self.client.job(job_url)['status'], 'enabled')
self.sandbox.tronctl('disable', job_name)
sandbox.wait_on_state(self.client.job, job_url, 'disabled', 'status')
def test_tronctl_basic(self):
self.start_with_config(SINGLE_ECHO_CONFIG + TOUCH_CLEANUP_FMT)
self.sandbox.tronctl_start('MASTER.echo_job')
cleanup_url = self.client.get_url('MASTER.echo_job.1.cleanup')
sandbox.wait_on_state(self.client.action, cleanup_url,
actionrun.ActionRun.STATE_SUCCEEDED.short_name)
action_run_url = self.client.get_url('MASTER.echo_job.1.echo_action')
assert_equal(self.client.action(action_run_url)['state'],
actionrun.ActionRun.STATE_SUCCEEDED.short_name)
job_run_url = self.client.get_url('MASTER.echo_job.1')
assert_equal(self.client.job_runs(job_run_url)['state'],
actionrun.ActionRun.STATE_SUCCEEDED.short_name)
def test_node_reconfig(self):
job_config = dedent(
"""
jobs:
- name: a_job
node: local
schedule: "interval 1s"
actions:
- name: first_action
command: "echo something"
"""
)
second_config = dedent(
"""
ssh_options:
agent: true
nodes:
- name: local
hostname: '127.0.0.1'
state_persistence:
name: "state_data.shelve"
store_type: shelve
"""
) + job_config
self.start_with_config(BASIC_CONFIG + job_config)
job_url = self.client.get_url('MASTER.a_job.0')
sandbox.wait_on_state(
self.client.job_runs,
job_url,
actionrun.ActionRun.SUCCEEDED,
)
self.sandbox.tronfig(second_config)
job_url = self.client.get_url('MASTER.a_job')
def wait_on_next_run():
last_run = self.client.job(job_url)['runs'][0]
return last_run['node']['hostname'] == '127.0.0.1'
sandbox.wait_on_sandbox(wait_on_next_run)
def test_trond_restart_job_running_with_dependencies(self):
config = BASIC_CONFIG + textwrap.dedent("""
jobs:
- name: complex_job
node: local
schedule: cron * * * * *
actions:
- name: first_act
command: sleep 20 && echo "I'm waiting"
- name: following_act
command: echo "thing"
requires: ['first_act']
- name: last_act
command: echo foo
requires: ['following_act']
""")
self.start_with_config(config)
job_name = 'MASTER.complex_job'
self.sandbox.tronctl('start', job_name)
action_run_url = self.client.get_url('MASTER.complex_job.1.first_act')
sandbox.wait_on_state(
self.client.action_runs,
action_run_url,
actionrun.ActionRun.RUNNING,
)
self.restart_trond()
assert_equal(
self.client.job_runs(action_run_url)['state'],
actionrun.ActionRun.UNKNOWN,
)
for followup_action_run in ('following_act', 'last_act'):
url = self.client.get_url('%s.1.%s' % (
job_name,
followup_action_run,
))
assert_equal(
self.client.action_runs(url)['state'],
actionrun.ActionRun.QUEUED,
)
def test_job_queueing_false_with_overlap(self):
"""Test that a job that has queueing false properly cancels an
overlapping job run.
"""
config = BASIC_CONFIG + dedent("""
jobs:
- name: "cancel_overlap"
schedule: "cron * * * * *"
queueing: False
node: local
actions:
- name: "do_something"
command: "sleep 3s"
- name: "do_other"
command: "sleep 3s"
cleanup_action:
command: "echo done"
""")
self.start_with_config(config)
job_url = self.client.get_url('MASTER.cancel_overlap')
job_run_url = self.client.get_url('MASTER.cancel_overlap.1')
def wait_on_job_schedule():
return len(self.client.job(job_url)['runs']) == 2
sandbox.wait_on_sandbox(wait_on_job_schedule)
sandbox.wait_on_state(
self.client.job,
job_run_url,
actionrun.ActionRun.CANCELLED,
)
action_run_states = [
action_run['state']
for action_run in self.client.job_runs(job_run_url)['runs']
]
expected = [
actionrun.ActionRun.CANCELLED
for _ in range(len(action_run_states))
]
assert_equal(action_run_states, expected)
def test_node_reconfig(self):
job_config = dedent("""
jobs:
- name: a_job
node: local
schedule: "cron * * * * *"
actions:
- name: first_action
command: "echo something"
""")
second_config = dedent("""
ssh_options:
agent: true
nodes:
- name: local
hostname: '127.0.0.1'
state_persistence:
name: "state_data.shelve"
store_type: shelve
""") + job_config
self.start_with_config(BASIC_CONFIG + job_config)
job_url = self.client.get_url('MASTER.a_job.0')
sandbox.wait_on_state(
self.client.job_runs,
job_url,
actionrun.ActionRun.SUCCEEDED,
)
self.sandbox.tronfig(second_config)
job_url = self.client.get_url('MASTER.a_job')
def wait_on_next_run():
last_run = self.client.job(job_url)['runs'][0]
return last_run['node']['hostname'] == '127.0.0.1'
sandbox.wait_on_sandbox(wait_on_next_run)
def test_node_reconfig(self):
job_service_config = dedent("""
jobs:
- name: a_job
node: local
schedule: "interval 1s"
actions:
- name: first_action
command: "echo something"
services:
- name: a_service
node: local
pid_file: /tmp/does_not_exist
command: "echo service start"
monitor_interval: 1
""")
second_config = dedent("""
ssh_options:
agent: true
nodes:
- name: local
hostname: '127.0.0.1'
state_persistence:
name: "state_data.shelve"
store_type: shelve
""") + job_service_config
self.start_with_config(BASIC_CONFIG + job_service_config)
service_name = 'MASTER.a_service'
service_url = self.client.get_url(service_name)
self.sandbox.tronctl('start', service_name)
sandbox.wait_on_state(self.client.service, service_url,
service.ServiceState.FAILED)
job_url = self.client.get_url('MASTER.a_job.0')
sandbox.wait_on_state(self.client.job_runs, job_url,
actionrun.ActionRun.STATE_SUCCEEDED.name)
self.sandbox.tronfig(second_config)
sandbox.wait_on_state(self.client.service, service_url,
service.ServiceState.DISABLED)
job_url = self.client.get_url('MASTER.a_job')
def wait_on_next_run():
last_run = self.client.job(job_url)['runs'][0]
return last_run['node']['hostname'] == '127.0.0.1'
sandbox.wait_on_sandbox(wait_on_next_run)
def test_node_reconfig(self):
job_service_config = dedent("""
jobs:
- name: a_job
node: local
schedule: "interval 1s"
actions:
- name: first_action
command: "echo something"
services:
- name: a_service
node: local
pid_file: /tmp/does_not_exist
command: "echo service start"
monitor_interval: 1
""")
second_config = dedent("""
ssh_options:
agent: true
nodes:
- name: local
hostname: '127.0.0.1'
state_persistence:
name: "state_data.shelve"
store_type: shelve
""") + job_service_config
self.start_with_config(BASIC_CONFIG + job_service_config)
service_name = 'MASTER.a_service'
service_url = self.client.get_url(service_name)
self.sandbox.tronctl('start', service_name)
sandbox.wait_on_state(self.client.service, service_url,
service.ServiceState.FAILED)
job_url = self.client.get_url('MASTER.a_job.0')
sandbox.wait_on_state(self.client.job_runs, job_url,
actionrun.ActionRun.STATE_SUCCEEDED.name)
self.sandbox.tronfig(second_config)
sandbox.wait_on_state(self.client.service, service_url,
service.ServiceState.DISABLED)
job_url = self.client.get_url('MASTER.a_job')
def wait_on_next_run():
last_run = self.client.job(job_url)['runs'][0]
return last_run['node']['hostname'] == '127.0.0.1'
sandbox.wait_on_sandbox(wait_on_next_run)