def test_from_state_node_no_longer_exists(self):
run = jobrun.JobRun.from_state(self.state_data, self.action_graph,
self.output_path, self.context, self.node_pool)
assert_length(run.action_runs.run_map, 1)
assert_equal(run.job_name, 'thejobname')
assert_equal(run.run_time, self.run_time)
assert_equal(run.node, self.node_pool)
def test_create_missing_none(self):
self.collection.config.count = 2
self.collection.instances = [
create_mock_instance(instance_number=i) for i in range(2)
]
created = self.collection.create_missing()
assert_length(created, 0)
def test_cancel_schedules_a_new_run(self):
config = BASIC_CONFIG + dedent("""
jobs:
- name: "a_job"
node: local
schedule: "daily 05:00:00"
actions:
- name: "first_action"
command: "echo OK"
""")
client = self.sandbox.client
self.sandbox.save_config(config)
self.sandbox.trond()
job_url = client.get_url('MASTER.a_job')
self.sandbox.tronctl(['cancel', 'MASTER.a_job.0'])
def wait_on_cancel():
return len(client.job(job_url)['runs']) == 2
sandbox.wait_on_sandbox(wait_on_cancel)
job_runs = client.job(job_url)['runs']
assert_length(job_runs, 2)
run_states = [run['state'] for run in job_runs]
assert_equal(run_states, ['SCHE', 'CANC'])
def test_start_no_startable_action_runs(self):
self.job_run._do_start = Turtle()
self.job_run.action_runs.has_startable_action_runs = False
assert not self.job_run.start()
assert_call(self.job_run.notify, 0, self.job_run.EVENT_START)
assert_length(self.job_run.notify.calls, 1)
def test_start(self):
self.job_run._do_start = Turtle()
assert self.job_run.start()
assert_call(self.job_run.notify, 0, self.job_run.EVENT_START)
assert_call(self.job_run._do_start, 0)
assert_length(self.job_run.notify.calls, 1)
def test_manual_start_with_run_time(self):
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
manual_runs = self.job_scheduler.manual_start(run_time)
self.job.build_new_runs.assert_called_with(run_time, manual=True)
assert_length(manual_runs, 1)
self.manual_run.start.assert_called_once_with()
def test_action_run_collection_from_state(self):
state_data = [self.action_state_data]
cleanup_action_state_data = {
'job_run_id': 'job_run_id',
'action_name': 'cleanup',
'state': 'succeeded',
'run_time': self.run_time,
'start_time': None,
'end_time': None,
'command': 'do cleanup',
'node_name': 'anode',
'action_runner': {
'status_path': '/tmp/foo',
'exec_path': '/bin/foo'
}
}
collection = ActionRunFactory.action_run_collection_from_state(
self.job_run,
state_data,
cleanup_action_state_data,
)
assert_equal(collection.action_graph, self.action_graph)
assert_length(collection.run_map, 2)
assert_equal(collection.run_map['act1'].action_name, 'act1')
assert_equal(collection.run_map['cleanup'].action_name, 'cleanup')
def test_handler_finished_with_cleanup(self):
self.job_run.action_runs.cleanup_action_run = Turtle(is_done=False)
self.job_run.finalize = Turtle()
self.job_run.handler(None, actionrun.ActionRun.STATE_SUCCEEDED)
assert_length(self.job_run.finalize.calls, 0)
assert_call(self.job_run.action_runs.cleanup_action_run.start, 0)
def test_build_action_run_collection(self):
collection = ActionRunFactory.build_action_run_collection(self.job_run)
assert_equal(collection.action_graph, self.action_graph)
assert_in('act1', collection.run_map)
assert_in('act2', collection.run_map)
assert_length(collection.run_map, 2)
assert_equal(collection.run_map['act1'].action_name, 'act1')
def test_manual_start_with_run_time(self):
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
manual_runs = self.job_scheduler.manual_start(run_time)
self.job.build_new_runs.assert_called_with(run_time, manual=True)
assert_length(manual_runs, 1)
self.manual_run.start.assert_called_once_with()
def test_restore_state(self):
run_collection = jobrun.JobRunCollection(20)
state_data = [
dict(
run_num=i,
job_name="thename",
run_time="sometime",
start_time="start_time",
end_time="sometime",
cleanup_run=None,
runs=[],
) for i in range(3, -1, -1)
]
action_graph = mock.create_autospec(actiongraph.ActionGraph)
output_path = mock.create_autospec(filehandler.OutputPath)
context = mock.Mock()
node_pool = mock.create_autospec(node.NodePool)
restored_runs = run_collection.restore_state(
state_data,
action_graph,
output_path,
context,
node_pool,
)
assert_equal(run_collection.runs[0].run_num, 3)
assert_equal(run_collection.runs[3].run_num, 0)
assert_length(restored_runs, 4)
def test_create_missing(self):
self.collection.config.count = 5
self.config.pid_file = "%s"
self.config.monitor_interval = 5
created = self.collection.create_missing()
assert_length(created, 5)
assert_equal(set(created), set(self.collection.instances))
def test_get_runs_to_schedule_guess(self):
job_runs = list(self.job_scheduler.get_runs_to_schedule(None))
assert self.job.scheduler.next_run_time.call_args_list == [mock.call(None)]
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
job_runs[0].attach.assert_any_call(True, self.job)
def test_handler_not_done(self):
self.job_run.action_runs.is_active = True
self.job_run._start_action_runs = lambda: []
self.job_run.finalize = Turtle()
self.job_run.handler(None, actionrun.ActionRun.STATE_SUCCEEDED)
assert_length(self.job_run.finalize.calls, 0)
def test_remove_old_runs(self):
self.run_collection.run_limit = 1
self.run_collection.remove_old_runs()
assert_length(self.run_collection.runs, 1)
assert_call(self.job_runs[-1].cleanup, 0)
for job_run in self.run_collection.runs:
assert_length(job_run.cancel.calls, 0)
def test_pending(self):
run_num = self.run_collection.next_run_num()
scheduled_run = self._mock_run(
run_num=run_num, state=actionrun.ActionRun.STATE_SCHEDULED)
self.run_collection.runs.appendleft(scheduled_run)
pending = list(self.run_collection.get_pending())
assert_length(pending, 2)
assert_equal(pending, [scheduled_run, self.job_runs[0]])
def test_handler_no_queued(self):
self.job_scheduler.run_job = Turtle()
def get_queued(state):
if state == ActionRun.STATE_QUEUED:
return []
self.job.runs.get_runs_by_state = get_queued
self.job_scheduler.handler(self.job, job.Job.NOTIFY_RUN_DONE)
assert_length(self.job_scheduler.run_job.calls, 0)
def test_get_runs_to_schedule_no_last_run(self):
self.job.runs.get_newest = lambda **kwargs: None
job_runs = list(self.job_scheduler.get_runs_to_schedule())
assert_length(self.job.scheduler.next_run_time.calls, 1)
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
assert_call(job_runs[0].attach, 0, True, self.job)
def test_build_action_run_collection(self):
collection = ActionRunFactory.build_action_run_collection(
self.job_run, self.action_runner)
assert_equal(collection.action_graph, self.action_graph)
assert_in('act1', collection.run_map)
assert_in('act2', collection.run_map)
assert_length(collection.run_map, 2)
assert_equal(collection.run_map['act1'].action_name, 'act1')
def test_get_runs_to_schedule_no_last_run(self):
self.job.runs.get_newest.return_value = None
job_runs = list(self.job_scheduler.get_runs_to_schedule(False))
self.job.scheduler.next_run_time.assert_called_once_with(None)
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
job_runs[0].attach.assert_any_call(True, self.job)
def test_get_runs_to_schedule_given(self):
now = datetime.datetime.now()
job_runs = list(self.job_scheduler.get_runs_to_schedule(now))
assert self.job.scheduler.next_run_time.call_count == 0
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
job_runs[0].attach.assert_any_call(True, self.job)
def test_get_runs_to_schedule_no_last_run(self):
self.job.runs.get_newest.return_value = None
job_runs = list(self.job_scheduler.get_runs_to_schedule(False))
self.job.scheduler.next_run_time.assert_called_once_with(None)
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
job_runs[0].attach.assert_any_call(True, self.job)
def test_run_job_schedule_on_complete(self):
self.job_scheduler.schedule = Turtle()
self.scheduler.schedule_on_complete = True
self.job.runs.get_active = lambda s: []
job_run = Turtle(is_cancelled=False)
self.job_scheduler.run_job(job_run)
assert_length(job_run.start.calls, 1)
assert_length(self.job_scheduler.schedule.calls, 0)
def test_run_job_schedule_on_complete(self):
self.job_scheduler.schedule = MagicMock()
self.job.scheduler.schedule_on_complete = True
self.job.runs.get_active = lambda s: []
job_run = MagicMock(is_cancelled=False)
self.job_scheduler.run_job(job_run)
assert_length(job_run.start.mock_calls, 1)
assert_length(self.job_scheduler.schedule.mock_calls, 0)
def test_get_runs_to_schedule_no_pending(self):
job_runs = list(self.job_scheduler.get_runs_to_schedule())
assert_call(self.job.runs.get_newest, 0, include_manual=False)
assert_length(self.job.scheduler.next_run_time.calls, 1)
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
assert_call(job_runs[0].attach, 0, True, self.job)
def test_get_active(self):
starting_run = self._mock_run(
run_num=self.run_collection.next_run_num(),
state=actionrun.ActionRun.STATE_STARTING)
self.run_collection.runs.appendleft(starting_run)
active = list(self.run_collection.get_active())
assert_length(active, 2)
assert_equal(active, [starting_run, self.job_runs[1]])
def test_run_job(self):
self.job_scheduler.schedule = Turtle()
self.scheduler.schedule_on_complete = False
self.job.runs.get_active = lambda n: []
job_run = Turtle(is_cancelled=False)
self.job_scheduler.run_job(job_run)
assert_length(job_run.start.calls, 1)
assert_length(self.job_scheduler.schedule.calls, 1)
def test_remove_old_runs(self):
self.run_collection.run_limit = 1
self.run_collection.remove_old_runs()
assert_length(self.run_collection.runs, 1)
assert_call(self.job_runs[-1].cleanup, 0)
for job_run in self.run_collection.runs:
assert_length(job_run.cancel.calls, 0)
def test_get_active_with_node(self):
starting_run = self._mock_run(
run_num=self.run_collection.next_run_num(),
state=actionrun.ActionRun.STATE_STARTING)
starting_run.node = 'differentnode'
self.run_collection.runs.appendleft(starting_run)
active = list(self.run_collection.get_active('anode'))
assert_length(active, 1)
assert_equal(active, [self.job_runs[1]])
def test_restore_state_no_state(self):
def restore(jobs, services):
return {}, {}
self.mcp.state_watcher = Turtle(restore=restore)
self.mcp.restore_state()
for job in self.mcp.jobs.values():
assert_length(job.restore_job_state.calls, 0)
for service in self.mcp.services.values():
assert_length(service.restore_service_state.calls, 0)
def test_build_and_sort(self):
autospec_method(self.collection.sort)
count = 4
builder, seq = mock.Mock(), range(count)
instances = self.collection._build_and_sort(builder, seq)
self.collection.sort.assert_called_with()
assert_equal(builder.mock_calls, [mock.call(i) for i in seq])
assert_length(instances, count)
assert_equal(instances, self.collection.instances)
def test_pending(self):
run_num = self.run_collection.next_run_num()
scheduled_run = self._mock_run(
run_num=run_num,
state=actionrun.ActionRun.STATE_SCHEDULED)
self.run_collection.runs.appendleft(scheduled_run)
pending = list(self.run_collection.get_pending())
assert_length(pending, 2)
assert_equal(pending, [scheduled_run, self.job_runs[0]])
def test_get_runs_to_schedule_queue_with_pending(self):
job_runs = list(self.job_scheduler.get_runs_to_schedule(False))
self.job.runs.get_newest.assert_called_with(include_manual=False)
self.job.scheduler.next_run_time.assert_called_once_with(
self.job.runs.get_newest.return_value.run_time)
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
job_runs[0].attach.assert_any_call(True, self.job)
def test_restore_job_state(self):
run_collection = mocks.MockJobRunCollection(get_scheduled=lambda: ['a'])
self.job_scheduler.job = Turtle(runs=run_collection)
self.job_scheduler._set_callback = Turtle()
state_data = 'state_data_token'
self.job_scheduler.restore_state(state_data)
assert_call(self.job_scheduler.job.restore_state, 0, state_data)
assert_length(self.job_scheduler._set_callback.calls, 1)
assert_call(self.job_scheduler._set_callback, 0, 'a')
def test_get_runs_to_schedule_queue_with_pending(self):
job_runs = list(self.job_scheduler.get_runs_to_schedule(False))
self.job.runs.get_newest.assert_called_with(include_manual=False)
self.job.scheduler.next_run_time.assert_called_once_with(
self.job.runs.get_newest.return_value.run_time)
assert_length(job_runs, 1)
# This should return a JobRun which has the job attached as an observer
job_runs[0].attach.assert_any_call(True, self.job)
def test_build_and_sort(self):
autospec_method(self.collection.sort)
count = 4
builder, seq = mock.Mock(), range(count)
instances = self.collection._build_and_sort(builder, seq)
self.collection.sort.assert_called_with()
assert_equal(builder.mock_calls, [mock.call(i) for i in seq])
assert_length(instances, count)
assert_equal(instances, self.collection.instances)
def test_restore_job_state(self):
run_collection = mocks.MockJobRunCollection(
get_scheduled=lambda: ['a'])
self.job_scheduler.job = Turtle(runs=run_collection)
self.job_scheduler._set_callback = Turtle()
state_data = 'state_data_token'
self.job_scheduler.restore_state(state_data)
assert_call(self.job_scheduler.job.restore_state, 0, state_data)
assert_length(self.job_scheduler._set_callback.calls, 1)
assert_call(self.job_scheduler._set_callback, 0, 'a')
def test_from_state(self):
run = jobrun.JobRun.from_state(self.state_data, self.action_graph,
self.output_path, self.context, self.node_pool)
assert_length(run.action_runs.run_map, 1)
assert_equal(run.job_name, self.state_data['job_name'])
assert_equal(run.run_time, self.run_time)
assert run.manual
assert_equal(run.output_path, self.output_path)
assert run.context.next
assert run.action_graph
def test_build_new_runs(self):
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
runs = list(self.job.build_new_runs(run_time))
assert_call(self.job.node_pool.next, 0)
node = self.job.node_pool.next.returns[0]
assert_call(self.job.runs.build_new_run,
0, self.job, run_time, node, manual=False)
assert_length(runs, 1)
assert_call(self.job.watch, 0, runs[0])
def test_restore_state(self):
count = 3
state_data = [dict(instance_number=i * 3, node="node") for i in xrange(count)]
autospec_method(self.collection._build_instance)
created = self.collection.restore_state(state_data)
assert_length(created, count)
assert_equal(set(created), set(self.collection.instances))
expected = [mock.call(self.node_pool.get_by_hostname.return_value, d["instance_number"]) for d in state_data]
for expected_call in expected:
assert_in(expected_call, self.collection._build_instance.mock_calls)
def test_handler_with_startable(self):
self.job_run.action_runs.get_startable_action_runs = lambda: True
startable_run = Turtle()
self.job_run.action_runs.get_startable_action_runs = lambda: [startable_run]
self.job_run.finalize = Turtle()
self.job_run.handler(None, actionrun.ActionRun.STATE_SUCCEEDED)
assert_call(self.job_run.notify, 0, self.job_run.NOTIFY_STATE_CHANGED)
assert_call(startable_run.start, 0)
assert_length(self.job_run.finalize.calls, 0)
def test_restore_state_partial(self):
def restore(jobs, services):
return {'1': 'thing'}, {'2': 'thing'}
self.mcp.state_watcher = Turtle(restore=restore)
self.mcp.restore_state()
assert_call(self.mcp.jobs['1'].restore_job_state, 0, 'thing')
assert_length(self.mcp.jobs['2'].restore_job_state.calls, 0)
assert_length(self.mcp.services['1'].restore_service_state.calls, 0)
assert_call(self.mcp.services['2'].restore_service_state, 0, 'thing')
def test_restore_state(self):
count = 3
state_data = [
dict(instance_number=i * 3, node='node') for i in range(count)
]
self.config.pid_file = "%s"
self.config.monitor_interval = 5
created = self.collection.restore_state(state_data)
assert_length(created, count)
assert_equal(set(created), set(self.collection.instances))
def test_set_action_runs(self):
self.job_run._action_runs = None
action_runs = [Turtle(), Turtle()]
run_collection = Turtle(action_runs_with_cleanup=action_runs)
self.job_run._set_action_runs(run_collection)
assert_length(self.job_run.watch.calls, 2)
for i in xrange(2):
assert_call(self.job_run.watch, i, action_runs[i])
assert_equal(self.job_run.action_runs, run_collection)
assert self.job_run.action_runs_proxy
def test_build_new_runs_manual(self):
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
runs = list(self.job.build_new_runs(run_time, manual=True))
self.job.node_pool.next.assert_called_with()
node = self.job.node_pool.next.return_value
assert_length(runs, 1)
assert_call(self.job.runs.build_new_run,
0, self.job, run_time, node, manual=True)
self.job.watch.assert_called_with(runs[0])
def test_restore_partial(self):
keys = [
sqlalchemystore.SQLStateKey(self.store.job_table, 'stars'),
sqlalchemystore.SQLStateKey(self.store.service_table, 'foo'),
]
item = {'docs': 'builder', 'a': 'b'}
self.store.save([(keys[0], item)])
docs = self.store.restore(keys)
assert_length(docs, 1)
assert_equal(docs[keys[0]], item)
def test_do_start(self):
startable_runs = [Turtle(), Turtle(), Turtle()]
self.job_run.action_runs.get_startable_action_runs = lambda: startable_runs
assert self.job_run._do_start()
assert_call(self.job_run.action_runs.ready, 0)
for i, startable_run in enumerate(startable_runs):
assert_call(startable_run.start, 0)
assert_length(self.job_run.notify.calls, 1)
assert_call(self.job_run.notify, 0, self.job_run.EVENT_STARTED)
def test_build_new_runs_all_nodes(self):
self.job.all_nodes = True
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
runs = list(self.job.build_new_runs(run_time))
assert_length(runs, 2)
for i in xrange(len(runs)):
node = self.job.node_pool.nodes[i]
assert_call(self.job.runs.build_new_run,
i, self.job, run_time, node, manual=False)
assert_call(self.job.watch, 1, runs[1])
def test_schedule(self):
self.job_scheduler.schedule()
assert_length(self.eventloop.call_later.mock_calls, 1)
# Args passed to callLater
call_args = self.eventloop.call_later.mock_calls[0][1]
assert_equal(call_args[1], self.job_scheduler.run_job)
secs = call_args[0]
run = call_args[2]
run.seconds_until_run_time.assert_called_with()
# Assert that we use the seconds we get from the run to schedule
assert_equal(run.seconds_until_run_time.return_value, secs)
def test_build_new_runs_manual(self):
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
runs = list(self.job.build_new_runs(run_time, manual=True))
self.job.node_pool.next.assert_called_with()
node = self.job.node_pool.next.return_value
assert_length(runs, 1)
assert_call(self.job.runs.build_new_run,
0,
self.job,
run_time,
node,
manual=True)
self.job.watch.assert_called_with(runs[0])
def test_manual_start_default_with_timezone(self):
self.job.time_zone = mock.Mock()
with mock.patch(
'tron.core.job_scheduler.timeutils.current_time',
autospec=True,
) as mock_current:
manual_runs = self.job_scheduler.manual_start()
mock_current.assert_called_with(tz=self.job.time_zone)
self.job.build_new_runs.assert_called_with(
mock_current.return_value,
manual=True,
)
assert_length(manual_runs, 1)
self.manual_run.start.assert_called_once_with()
def test_restore_state(self):
count = 3
state_data = [
dict(instance_number=i * 3, node='node') for i in xrange(count)
]
autospec_method(self.collection._build_instance)
created = self.collection.restore_state(state_data)
assert_length(created, count)
assert_equal(set(created), set(self.collection.instances))
expected = [
mock.call(self.node_pool.get_by_hostname.return_value,
d['instance_number']) for d in state_data
]
for expected_call in expected:
assert_in(expected_call,
self.collection._build_instance.mock_calls)
def test_remove_observer(self):
observer = lambda: 2
self.obs.attach('a', observer)
self.obs.attach('c', observer)
self.obs.remove_observer(observer)
assert_length(self.obs._observers['a'], 2)
assert_length(self.obs._observers['b'], 2)
assert_length(self.obs._observers[True], 1)
assert_length(self.obs._observers['c'], 0)
def test_load_from_config(self):
autospec_method(self.collection.jobs.filter_by_name)
autospec_method(self.collection.add)
factory = mock.create_autospec(JobSchedulerFactory)
job_configs = {'a': mock.Mock(), 'b': mock.Mock()}
result = self.collection.load_from_config(job_configs, factory, True)
result = list(result)
self.collection.jobs.filter_by_name.assert_called_with(job_configs)
expected_calls = [mock.call(v) for v in job_configs.values()]
assert_mock_calls(expected_calls, factory.build.mock_calls)
assert_length(self.collection.add.mock_calls, len(job_configs) * 2)
assert_length(result, len(job_configs))
job_schedulers = [
call[1][0] for call in self.collection.add.mock_calls[::2]
]
for job_scheduler in job_schedulers:
job_scheduler.schedule.assert_called_with()
job_scheduler.get_job.assert_called_with()
def test_build_new_runs_all_nodes(self):
self.job.all_nodes = True
run_time = datetime.datetime(2012, 3, 14, 15, 9, 26)
node_count = 2
self.job.node_pool.nodes = [mock.Mock()] * node_count
runs = list(self.job.build_new_runs(run_time))
assert_length(runs, node_count)
for i in xrange(len(runs)):
node = self.job.node_pool.nodes[i]
assert_call(self.job.runs.build_new_run,
i,
self.job,
run_time,
node,
manual=False)
self.job.watch.assert_has_calls([mock.call(run) for run in runs])
def test_remove_observer(self):
def observer():
return 2
self.obs.attach('a', observer)
self.obs.attach('c', observer)
self.obs.remove_observer(observer)
assert_length(self.obs._observers['a'], 2)
assert_length(self.obs._observers['b'], 2)
assert_length(self.obs._observers[True], 1)
assert_length(self.obs._observers['c'], 0)
