def test_status_wildcard(self):
"""Test status using a wildcard. It should first call api.get_jobs, and then do a
getTasksStatus on each job."""
mock_context = FakeAuroraCommandContext()
mock_api = mock_context.get_api('west')
mock_api.check_status.return_value = self.create_status_response()
mock_api.get_jobs.return_value = self.create_getjobs_response()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.cli.context.CLUSTERS', new=self.TEST_CLUSTERS)):
cmd = AuroraCommandLine()
cmd.execute(['job', 'status', '*'])
# Wildcard should have expanded to two jobs, so there should be two calls
# to check_status.
assert mock_api.check_status.call_count == 2
assert mock_api.check_status.call_args_list[0][0][0].cluster == 'west'
assert mock_api.check_status.call_args_list[0][0][0].role == 'RoleA'
assert mock_api.check_status.call_args_list[0][0][0].env == 'test'
assert mock_api.check_status.call_args_list[0][0][0].name == 'hithere'
assert mock_api.check_status.call_args_list[1][0][0].cluster == 'west'
assert mock_api.check_status.call_args_list[1][0][0].role == 'bozo'
assert mock_api.check_status.call_args_list[1][0][0].env == 'test'
assert mock_api.check_status.call_args_list[1][0][0].name == 'hello'
def test_restart_simple(self):
# Test the client-side restart logic in its simplest case: everything succeeds
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
self.setup_mock_scheduler_for_simple_restart(mock_api)
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.instance_watcher.InstanceWatcherHealthCheck',
return_value=mock_health_check),
patch('time.time', side_effect=functools.partial(self.fake_time, self)),
patch('time.sleep', return_value=None)
):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'restart', '--batch_size=5', 'west/bozo/test/hello', fp.name])
# Like the update test, the exact number of calls here doesn't matter.
# what matters is that it must have been called once before batching, plus
# at least once per batch, and there are 4 batches.
assert mock_scheduler_proxy.getTasksStatus.call_count >= 4
# called once per batch
assert mock_scheduler_proxy.restartShards.call_count == 4
# parameters for all calls are generated by the same code, so we just check one
mock_scheduler_proxy.restartShards.assert_called_with(JobKey(environment=self.TEST_ENV,
role=self.TEST_ROLE, name=self.TEST_JOB), [15, 16, 17, 18, 19], None)
def test_diff_server_error(self):
"""Test the diff command if the user passes a config with an error in it."""
mock_options = self.setup_mock_options()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.getTasksStatus.return_value = self.create_failed_status_response()
self.setup_populate_job_config(mock_scheduler_proxy)
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('subprocess.call', return_value=0),
patch('json.loads', return_value=Mock())) as (
mock_scheduler_proxy_class,
mock_clusters,
options,
subprocess_patch,
json_patch):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name])
assert result == EXIT_INVALID_PARAMETER
# In this error case, we should have called the server getTasksStatus;
# but since it fails, we shouldn't call populateJobConfig or subprocess.
mock_scheduler_proxy.getTasksStatus.assert_called_with(
TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo'),
statuses=ACTIVE_STATES))
assert mock_scheduler_proxy.populateJobConfig.call_count == 0
assert subprocess_patch.call_count == 0
def test_create_job_failed(self):
"""Run a test of the "create" command against a mocked-out API:
this time, make the monitor check status several times before successful completion.
"""
mock_context = FakeAuroraCommandContext()
with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context):
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.INIT))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_failed_createjob_response()
# This is the real test: invoke create as if it had been called by the command line.
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'create', '--wait_until=RUNNING',
'west/mchucarroll/test/hello', fp.name])
assert result == EXIT_NETWORK_ERROR
# Now check that the right API calls got made.
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
# getTasksStatus was called once, before the create_job
assert api.scheduler.getTasksStatus.call_count == 1
def test_diff_invalid_config(self):
"""Test the diff command if the user passes a config with an error in it."""
mock_options = self.setup_mock_options()
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response()
self.setup_populate_job_config(mock_scheduler_proxy)
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('twitter.common.app.get_options', return_value=mock_options),
patch('subprocess.call', return_value=0),
patch('json.loads', return_value=Mock())) as (
mock_scheduler_proxy_class,
mock_clusters,
options,
subprocess_patch,
json_patch):
with temporary_file() as fp:
fp.write(self.get_invalid_config('stupid="me"',))
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name])
assert result == EXIT_INVALID_CONFIGURATION
assert mock_scheduler_proxy.getTasksStatus.call_count == 0
assert mock_scheduler_proxy.populateJobConfig.call_count == 0
assert subprocess_patch.call_count == 0
def test_successful_diff(self):
"""Test the diff command."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('subprocess.call', return_value=0),
patch('json.loads', return_value=Mock())) as (_, _, subprocess_patch, _):
mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response()
self.setup_populate_job_config(mock_scheduler_proxy)
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name])
# Diff should get the task status, populate a config, and run diff.
mock_scheduler_proxy.getTasksStatus.assert_called_with(
TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo'),
statuses=ACTIVE_STATES))
assert mock_scheduler_proxy.populateJobConfig.call_count == 1
assert isinstance(mock_scheduler_proxy.populateJobConfig.call_args[0][0], JobConfiguration)
assert (mock_scheduler_proxy.populateJobConfig.call_args[0][0].key ==
JobKey(environment=u'test', role=u'bozo', name=u'hello'))
# Subprocess should have been used to invoke diff with two parameters.
assert subprocess_patch.call_count == 1
assert len(subprocess_patch.call_args[0][0]) == 3
assert subprocess_patch.call_args[0][0][0] == 'diff'
def test_simple_successful_create_job(self):
"""Run a test of the "create" command against a mocked-out API:
Verifies that the creation command sends the right API RPCs, and performs the correct
tests on the result."""
# We'll patch out create_context, which will give us a fake context
# object, and everything can be stubbed through that.
mock_context = FakeAuroraCommandContext()
with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context):
# After making the client, create sets up a job monitor.
# The monitor uses TaskQuery to get the tasks. It's called at least twice:once before
# the job is created, and once after. So we need to set up mocks for the query results.
mock_query = self.create_mock_query()
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.INIT))
mock_context.add_expected_status_query_result(
self.create_mock_status_query_result(ScheduleStatus.RUNNING))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
# This is the real test: invoke create as if it had been called by the command line.
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'create', '--wait_until=RUNNING', 'west/mchucarroll/test/hello',
fp.name])
# Now check that the right API calls got made.
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 2)
def test_simple_successful_cancel_update(self):
"""Run a test of the "kill" command against a mocked-out API:
Verifies that the kill command sends the right API RPCs, and performs the correct
tests on the result."""
mock_context = FakeAuroraCommandContext()
mock_api = mock_context.get_api('west')
mock_api.cancel_update.return_value = self.create_simple_success_response()
with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context):
cmd = AuroraCommandLine()
cmd.execute(['job', 'cancel_update', 'west/bozo/test/hello'])
self.assert_cancel_update_called(mock_api)
def test_successful_status_shallow(self):
"""Test the status command at the shallowest level: calling status should end up invoking
the local APIs get_status method."""
mock_context = FakeAuroraCommandContext()
mock_api = mock_context.get_api('west')
mock_api.check_status.return_value = self.create_status_response()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
cmd = AuroraCommandLine()
cmd.execute(['job', 'status', 'west/bozo/test/hello'])
mock_api.check_status.assert_called_with(AuroraJobKey('west', 'bozo', 'test', 'hello'))
def test_successful_status_deep(self):
"""Test the status command more deeply: in a request with a fully specified
job, it should end up doing a query using getTasksStatus."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.query.return_value = self.create_status_response()
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
cmd = AuroraCommandLine()
cmd.execute(['job', 'status', 'west/bozo/test/hello'])
mock_scheduler_proxy.getTasksStatus.assert_called_with(TaskQuery(jobName='hello',
environment='test', owner=Identity(role='bozo')))
def test_unsuccessful_status_shallow(self):
"""Test the status command at the shallowest level: calling status should end up invoking
the local APIs get_status method."""
# Calls api.check_status, which calls scheduler_proxy.getJobs
mock_context = FakeAuroraCommandContext()
mock_api = mock_context.get_api('west')
mock_api.check_status.return_value = self.create_failed_status_response()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'status', 'west/bozo/test/hello'])
assert result == EXIT_INVALID_PARAMETER
def test_update_invalid_config(self):
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
mock_api = mock_context.get_api('west')
with temporary_file() as fp:
fp.write(self.get_invalid_config('invalid_field=False,'))
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'update', '--force', self.TEST_JOBSPEC, fp.name])
assert result == EXIT_INVALID_CONFIGURATION
assert mock_api.update_job.call_count == 0
def test_status_wildcard_two(self):
"""Test status using a wildcard. It should first call api.get_jobs, and then do a
getTasksStatus on each job. This time, use a pattern that doesn't match all of the jobs."""
mock_context = FakeAuroraCommandContext()
mock_api = mock_context.get_api('west')
mock_api.check_status.return_value = self.create_status_response()
mock_api.get_jobs.return_value = self.create_getjobs_response()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
cmd = AuroraCommandLine()
cmd.execute(['job', 'status', 'example/*/*/hello'])
# Wildcard should have expanded to two jobs, but only matched one,
# so there should be one call to check_status.
assert mock_api.check_status.call_count == 1
mock_api.check_status.assert_called_with(
AuroraJobKey('example', 'bozo', 'test', 'hello'))
def test_update_command_line_succeeds(self):
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
mock_api = mock_context.get_api('west')
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'update', '--force', self.TEST_JOBSPEC, fp.name])
assert mock_api.update_job.call_count == 1
args, kwargs = mock_api.update_job.call_args
assert isinstance(args[0], AuroraConfig)
assert args[1] == 3
assert args[2] is None
def test_kill_job_with_instances_deep_api(self):
"""Test kill client-side API logic."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6',
'west/bozo/test/hello'])
# Now check that the right API calls got made.
assert mock_scheduler_proxy.killTasks.call_count == 1
mock_scheduler_proxy.killTasks.assert_called_with(
TaskQuery(jobName='hello', environment='test', instanceIds=frozenset([0, 2, 4, 5, 6]),
owner=Identity(role='bozo')), None)
def test_cancel_update_api_level(self):
"""Test kill client-side API logic."""
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_scheduler_proxy.releaseLock.return_value = self.get_release_lock_response()
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'cancel_update', 'west/mchucarroll/test/hello'])
# All that cancel_update really does is release the update lock.
# So that's all we really need to check.
assert mock_scheduler_proxy.releaseLock.call_count == 1
assert mock_scheduler_proxy.releaseLock.call_args[0][0].key.job == JobKey(environment='test',
role='mchucarroll', name='hello')
def test_create_job_failed_invalid_config(self):
"""Run a test of the "create" command against a mocked-out API, with a configuration
containing a syntax error"""
mock_context = FakeAuroraCommandContext()
with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context):
with temporary_file() as fp:
fp.write(self.get_invalid_config('invalid_clause=oops'))
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'create', '--wait_until=RUNNING',
'west/mchucarroll/test/hello', fp.name])
assert result == EXIT_INVALID_CONFIGURATION
# Now check that the right API calls got made.
# Check that create_job was not called.
api = mock_context.get_api('west')
assert api.create_job.call_count == 0
assert api.scheduler.getTasksStatus.call_count == 0
def test_kill_job_with_instances(self):
"""Test kill client-side API logic."""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)):
api = mock_context.get_api('west')
api.kill_job.return_value = self.get_kill_job_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6',
'west/mchucarroll/test/hello'])
# Now check that the right API calls got made.
assert api.kill_job.call_count == 1
api.kill_job.assert_called_with(AuroraJobKey.from_path('west/mchucarroll/test/hello'),
[0, 2, 4, 5, 6])
def test_restart_failed_status(self):
# Test the client-side updater logic in its simplest case: everything succeeds, and no rolling
# updates.
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
self.setup_mock_scheduler_for_simple_restart(mock_api)
mock_scheduler_proxy.getTasksStatus.return_value = self.create_error_response()
with contextlib.nested(
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.instance_watcher.InstanceWatcherHealthCheck',
return_value=mock_health_check),
patch('time.time', side_effect=functools.partial(self.fake_time, self)),
patch('time.sleep', return_value=None)):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
result = cmd.execute(['job', 'restart', '--batch_size=5', 'west/bozo/test/hello', fp.name])
assert mock_scheduler_proxy.getTasksStatus.call_count == 1
assert mock_scheduler_proxy.restartShards.call_count == 0
assert result == EXIT_API_ERROR
def test_updater_simple(self):
# Test the client-side updater logic in its simplest case: everything succeeds,
# and no rolling updates. (Rolling updates are covered by the updated tests.)
(mock_api, mock_scheduler_proxy) = self.create_mock_api()
mock_health_check = self.setup_health_checks(mock_api)
mock_quota_check = self.setup_quota_check()
self.setup_mock_scheduler_for_simple_update(mock_api)
# This doesn't work, because:
# - The mock_context stubs out the API.
# - the test relies on using live code in the API.
with contextlib.nested(
patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS),
patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
patch('apache.aurora.client.api.instance_watcher.InstanceWatcherHealthCheck',
return_value=mock_health_check),
patch('apache.aurora.client.api.quota_check.QuotaCheck', return_value=mock_quota_check),
patch('time.time', side_effect=functools.partial(self.fake_time, self)),
patch('time.sleep', return_value=None)):
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'update', 'west/bozo/test/hello', fp.name])
# We don't check all calls. The updater should be able to change. What's important
# is that we verify the key parts of the update process, to have some confidence
# that update did the right things.
# Every update should:
# check its options, acquire an update lock, check status,
# kill the old tasks, start new ones, wait for the new ones to healthcheck,
# and finally release the lock.
# The kill/start should happen in rolling batches.
mock_scheduler_proxy = mock_api.scheduler_proxy
assert mock_scheduler_proxy.acquireLock.call_count == 1
self.assert_correct_killtask_calls(mock_scheduler_proxy)
self.assert_correct_addinstance_calls(mock_scheduler_proxy)
self.assert_correct_status_calls(mock_scheduler_proxy)
assert mock_scheduler_proxy.releaseLock.call_count == 1
def test_create_job_delayed(self):
"""Run a test of the "create" command against a mocked-out API:
this time, make the monitor check status several times before successful completion.
"""
mock_context = FakeAuroraCommandContext()
with contextlib.nested(
patch('time.sleep'),
patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)):
mock_query = self.create_mock_query()
for result in [ScheduleStatus.INIT, ScheduleStatus.PENDING, ScheduleStatus.PENDING,
ScheduleStatus.RUNNING, ScheduleStatus.FINISHED]:
mock_context.add_expected_status_query_result(self.create_mock_status_query_result(result))
api = mock_context.get_api('west')
api.create_job.return_value = self.get_createjob_response()
with temporary_file() as fp:
fp.write(self.get_valid_config())
fp.flush()
cmd = AuroraCommandLine()
cmd.execute(['job', 'create', '--wait_until=RUNNING', 'west/mchucarroll/test/hello',
fp.name])
# Now check that the right API calls got made.
# Check that create_job was called exactly once, with an AuroraConfig parameter.
self.assert_create_job_called(api)
self.assert_scheduler_called(api, mock_query, 4)
