def test_status_wildcard(self): """Test status using a wildcard. It should first call api.get_jobs, and then do a getTasksStatus on each job.""" mock_context = FakeAuroraCommandContext() mock_api = mock_context.get_api('west') mock_api.check_status.return_value = self.create_status_response() mock_api.get_jobs.return_value = self.create_getjobs_response() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context), patch('apache.aurora.client.cli.context.CLUSTERS', new=self.TEST_CLUSTERS)): cmd = AuroraCommandLine() cmd.execute(['job', 'status', '*']) # Wildcard should have expanded to two jobs, so there should be two calls # to check_status. assert mock_api.check_status.call_count == 2 assert mock_api.check_status.call_args_list[0][0][0].cluster == 'west' assert mock_api.check_status.call_args_list[0][0][0].role == 'RoleA' assert mock_api.check_status.call_args_list[0][0][0].env == 'test' assert mock_api.check_status.call_args_list[0][0][0].name == 'hithere' assert mock_api.check_status.call_args_list[1][0][0].cluster == 'west' assert mock_api.check_status.call_args_list[1][0][0].role == 'bozo' assert mock_api.check_status.call_args_list[1][0][0].env == 'test' assert mock_api.check_status.call_args_list[1][0][0].name == 'hello'
def test_restart_simple(self): # Test the client-side restart logic in its simplest case: everything succeeds (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_health_check = self.setup_health_checks(mock_api) self.setup_mock_scheduler_for_simple_restart(mock_api) with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS), patch('apache.aurora.client.api.instance_watcher.InstanceWatcherHealthCheck', return_value=mock_health_check), patch('time.time', side_effect=functools.partial(self.fake_time, self)), patch('time.sleep', return_value=None) ): with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'restart', '--batch_size=5', 'west/bozo/test/hello', fp.name]) # Like the update test, the exact number of calls here doesn't matter. # what matters is that it must have been called once before batching, plus # at least once per batch, and there are 4 batches. assert mock_scheduler_proxy.getTasksStatus.call_count >= 4 # called once per batch assert mock_scheduler_proxy.restartShards.call_count == 4 # parameters for all calls are generated by the same code, so we just check one mock_scheduler_proxy.restartShards.assert_called_with(JobKey(environment=self.TEST_ENV, role=self.TEST_ROLE, name=self.TEST_JOB), [15, 16, 17, 18, 19], None)
def test_successful_diff(self): """Test the diff command.""" (mock_api, mock_scheduler_proxy) = self.create_mock_api() with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS), patch('subprocess.call', return_value=0), patch('json.loads', return_value=Mock())) as (_, _, subprocess_patch, _): mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response() self.setup_populate_job_config(mock_scheduler_proxy) with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name]) # Diff should get the task status, populate a config, and run diff. mock_scheduler_proxy.getTasksStatus.assert_called_with( TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo'), statuses=ACTIVE_STATES)) assert mock_scheduler_proxy.populateJobConfig.call_count == 1 assert isinstance(mock_scheduler_proxy.populateJobConfig.call_args[0][0], JobConfiguration) assert (mock_scheduler_proxy.populateJobConfig.call_args[0][0].key == JobKey(environment=u'test', role=u'bozo', name=u'hello')) # Subprocess should have been used to invoke diff with two parameters. assert subprocess_patch.call_count == 1 assert len(subprocess_patch.call_args[0][0]) == 3 assert subprocess_patch.call_args[0][0][0] == 'diff'
def test_simple_successful_create_job(self): """Run a test of the "create" command against a mocked-out API: Verifies that the creation command sends the right API RPCs, and performs the correct tests on the result.""" # We'll patch out create_context, which will give us a fake context # object, and everything can be stubbed through that. mock_context = FakeAuroraCommandContext() with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context): # After making the client, create sets up a job monitor. # The monitor uses TaskQuery to get the tasks. It's called at least twice:once before # the job is created, and once after. So we need to set up mocks for the query results. mock_query = self.create_mock_query() mock_context.add_expected_status_query_result( self.create_mock_status_query_result(ScheduleStatus.INIT)) mock_context.add_expected_status_query_result( self.create_mock_status_query_result(ScheduleStatus.RUNNING)) api = mock_context.get_api('west') api.create_job.return_value = self.get_createjob_response() # This is the real test: invoke create as if it had been called by the command line. with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'create', '--wait_until=RUNNING', 'west/mchucarroll/test/hello', fp.name]) # Now check that the right API calls got made. # Check that create_job was called exactly once, with an AuroraConfig parameter. self.assert_create_job_called(api) self.assert_scheduler_called(api, mock_query, 2)
def test_simple_successful_cancel_update(self): """Run a test of the "kill" command against a mocked-out API: Verifies that the kill command sends the right API RPCs, and performs the correct tests on the result.""" mock_context = FakeAuroraCommandContext() mock_api = mock_context.get_api('west') mock_api.cancel_update.return_value = self.create_simple_success_response() with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context): cmd = AuroraCommandLine() cmd.execute(['job', 'cancel_update', 'west/bozo/test/hello']) self.assert_cancel_update_called(mock_api)
def test_successful_status_shallow(self): """Test the status command at the shallowest level: calling status should end up invoking the local APIs get_status method.""" mock_context = FakeAuroraCommandContext() mock_api = mock_context.get_api('west') mock_api.check_status.return_value = self.create_status_response() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)): cmd = AuroraCommandLine() cmd.execute(['job', 'status', 'west/bozo/test/hello']) mock_api.check_status.assert_called_with(AuroraJobKey('west', 'bozo', 'test', 'hello'))
def test_successful_status_deep(self): """Test the status command more deeply: in a request with a fully specified job, it should end up doing a query using getTasksStatus.""" (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_scheduler_proxy.query.return_value = self.create_status_response() with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): cmd = AuroraCommandLine() cmd.execute(['job', 'status', 'west/bozo/test/hello']) mock_scheduler_proxy.getTasksStatus.assert_called_with(TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo')))
def test_diff_server_error(self): """Test the diff command if the user passes a config with an error in it.""" mock_options = self.setup_mock_options() (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_scheduler_proxy.getTasksStatus.return_value = self.create_failed_status_response() self.setup_populate_job_config(mock_scheduler_proxy) with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS), patch('twitter.common.app.get_options', return_value=mock_options), patch('subprocess.call', return_value=0), patch('json.loads', return_value=Mock())) as ( mock_scheduler_proxy_class, mock_clusters, options, subprocess_patch, json_patch): with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() result = cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name]) assert result == EXIT_INVALID_PARAMETER # In this error case, we should have called the server getTasksStatus; # but since it fails, we shouldn't call populateJobConfig or subprocess. mock_scheduler_proxy.getTasksStatus.assert_called_with( TaskQuery(jobName='hello', environment='test', owner=Identity(role='bozo'), statuses=ACTIVE_STATES)) assert mock_scheduler_proxy.populateJobConfig.call_count == 0 assert subprocess_patch.call_count == 0
def test_diff_invalid_config(self): """Test the diff command if the user passes a config with an error in it.""" mock_options = self.setup_mock_options() (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_scheduler_proxy.getTasksStatus.return_value = self.create_status_response() self.setup_populate_job_config(mock_scheduler_proxy) with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS), patch('twitter.common.app.get_options', return_value=mock_options), patch('subprocess.call', return_value=0), patch('json.loads', return_value=Mock())) as ( mock_scheduler_proxy_class, mock_clusters, options, subprocess_patch, json_patch): with temporary_file() as fp: fp.write(self.get_invalid_config('stupid="me"',)) fp.flush() cmd = AuroraCommandLine() result = cmd.execute(['job', 'diff', 'west/bozo/test/hello', fp.name]) assert result == EXIT_INVALID_CONFIGURATION assert mock_scheduler_proxy.getTasksStatus.call_count == 0 assert mock_scheduler_proxy.populateJobConfig.call_count == 0 assert subprocess_patch.call_count == 0
def test_create_job_failed(self): """Run a test of the "create" command against a mocked-out API: this time, make the monitor check status several times before successful completion. """ mock_context = FakeAuroraCommandContext() with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context): mock_context.add_expected_status_query_result( self.create_mock_status_query_result(ScheduleStatus.INIT)) api = mock_context.get_api('west') api.create_job.return_value = self.get_failed_createjob_response() # This is the real test: invoke create as if it had been called by the command line. with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() result = cmd.execute(['job', 'create', '--wait_until=RUNNING', 'west/mchucarroll/test/hello', fp.name]) assert result == EXIT_NETWORK_ERROR # Now check that the right API calls got made. # Check that create_job was called exactly once, with an AuroraConfig parameter. self.assert_create_job_called(api) # getTasksStatus was called once, before the create_job assert api.scheduler.getTasksStatus.call_count == 1
def test_update_command_line_succeeds(self): mock_context = FakeAuroraCommandContext() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): mock_api = mock_context.get_api('west') with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'update', '--force', self.TEST_JOBSPEC, fp.name]) assert mock_api.update_job.call_count == 1 args, kwargs = mock_api.update_job.call_args assert isinstance(args[0], AuroraConfig) assert args[1] == 3 assert args[2] is None
def test_status_wildcard_two(self): """Test status using a wildcard. It should first call api.get_jobs, and then do a getTasksStatus on each job. This time, use a pattern that doesn't match all of the jobs.""" mock_context = FakeAuroraCommandContext() mock_api = mock_context.get_api('west') mock_api.check_status.return_value = self.create_status_response() mock_api.get_jobs.return_value = self.create_getjobs_response() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)): cmd = AuroraCommandLine() cmd.execute(['job', 'status', 'example/*/*/hello']) # Wildcard should have expanded to two jobs, but only matched one, # so there should be one call to check_status. assert mock_api.check_status.call_count == 1 mock_api.check_status.assert_called_with( AuroraJobKey('example', 'bozo', 'test', 'hello'))
def test_cancel_update_api_level(self): """Test kill client-side API logic.""" (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_scheduler_proxy.releaseLock.return_value = self.get_release_lock_response() with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'cancel_update', 'west/mchucarroll/test/hello']) # All that cancel_update really does is release the update lock. # So that's all we really need to check. assert mock_scheduler_proxy.releaseLock.call_count == 1 assert mock_scheduler_proxy.releaseLock.call_args[0][0].key.job == JobKey(environment='test', role='mchucarroll', name='hello')
def test_kill_job_with_instances_deep_api(self): """Test kill client-side API logic.""" (mock_api, mock_scheduler_proxy) = self.create_mock_api() with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): mock_scheduler_proxy.killTasks.return_value = self.get_kill_job_response() with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6', 'west/bozo/test/hello']) # Now check that the right API calls got made. assert mock_scheduler_proxy.killTasks.call_count == 1 mock_scheduler_proxy.killTasks.assert_called_with( TaskQuery(jobName='hello', environment='test', instanceIds=frozenset([0, 2, 4, 5, 6]), owner=Identity(role='bozo')), None)
def test_kill_job_with_instances(self): """Test kill client-side API logic.""" mock_context = FakeAuroraCommandContext() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): api = mock_context.get_api('west') api.kill_job.return_value = self.get_kill_job_response() with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'kill', '--config=%s' % fp.name, '--instances=0,2,4-6', 'west/mchucarroll/test/hello']) # Now check that the right API calls got made. assert api.kill_job.call_count == 1 api.kill_job.assert_called_with(AuroraJobKey.from_path('west/mchucarroll/test/hello'), [0, 2, 4, 5, 6])
def test_unsuccessful_status_shallow(self): """Test the status command at the shallowest level: calling status should end up invoking the local APIs get_status method.""" # Calls api.check_status, which calls scheduler_proxy.getJobs mock_context = FakeAuroraCommandContext() mock_api = mock_context.get_api('west') mock_api.check_status.return_value = self.create_failed_status_response() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)): cmd = AuroraCommandLine() result = cmd.execute(['job', 'status', 'west/bozo/test/hello']) assert result == EXIT_INVALID_PARAMETER
def test_update_invalid_config(self): mock_context = FakeAuroraCommandContext() with contextlib.nested( patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS)): mock_api = mock_context.get_api('west') with temporary_file() as fp: fp.write(self.get_invalid_config('invalid_field=False,')) fp.flush() cmd = AuroraCommandLine() result = cmd.execute(['job', 'update', '--force', self.TEST_JOBSPEC, fp.name]) assert result == EXIT_INVALID_CONFIGURATION assert mock_api.update_job.call_count == 0
def test_updater_simple(self): # Test the client-side updater logic in its simplest case: everything succeeds, # and no rolling updates. (Rolling updates are covered by the updated tests.) (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_health_check = self.setup_health_checks(mock_api) mock_quota_check = self.setup_quota_check() self.setup_mock_scheduler_for_simple_update(mock_api) # This doesn't work, because: # - The mock_context stubs out the API. # - the test relies on using live code in the API. with contextlib.nested( patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS), patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.api.instance_watcher.InstanceWatcherHealthCheck', return_value=mock_health_check), patch('apache.aurora.client.api.quota_check.QuotaCheck', return_value=mock_quota_check), patch('time.time', side_effect=functools.partial(self.fake_time, self)), patch('time.sleep', return_value=None)): with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'update', 'west/bozo/test/hello', fp.name]) # We don't check all calls. The updater should be able to change. What's important # is that we verify the key parts of the update process, to have some confidence # that update did the right things. # Every update should: # check its options, acquire an update lock, check status, # kill the old tasks, start new ones, wait for the new ones to healthcheck, # and finally release the lock. # The kill/start should happen in rolling batches. mock_scheduler_proxy = mock_api.scheduler_proxy assert mock_scheduler_proxy.acquireLock.call_count == 1 self.assert_correct_killtask_calls(mock_scheduler_proxy) self.assert_correct_addinstance_calls(mock_scheduler_proxy) self.assert_correct_status_calls(mock_scheduler_proxy) assert mock_scheduler_proxy.releaseLock.call_count == 1
def test_create_job_delayed(self): """Run a test of the "create" command against a mocked-out API: this time, make the monitor check status several times before successful completion. """ mock_context = FakeAuroraCommandContext() with contextlib.nested( patch('time.sleep'), patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context)): mock_query = self.create_mock_query() for result in [ScheduleStatus.INIT, ScheduleStatus.PENDING, ScheduleStatus.PENDING, ScheduleStatus.RUNNING, ScheduleStatus.FINISHED]: mock_context.add_expected_status_query_result(self.create_mock_status_query_result(result)) api = mock_context.get_api('west') api.create_job.return_value = self.get_createjob_response() with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() cmd.execute(['job', 'create', '--wait_until=RUNNING', 'west/mchucarroll/test/hello', fp.name]) # Now check that the right API calls got made. # Check that create_job was called exactly once, with an AuroraConfig parameter. self.assert_create_job_called(api) self.assert_scheduler_called(api, mock_query, 4)
def test_create_job_failed_invalid_config(self): """Run a test of the "create" command against a mocked-out API, with a configuration containing a syntax error""" mock_context = FakeAuroraCommandContext() with patch('apache.aurora.client.cli.jobs.Job.create_context', return_value=mock_context): with temporary_file() as fp: fp.write(self.get_invalid_config('invalid_clause=oops')) fp.flush() cmd = AuroraCommandLine() result = cmd.execute(['job', 'create', '--wait_until=RUNNING', 'west/mchucarroll/test/hello', fp.name]) assert result == EXIT_INVALID_CONFIGURATION # Now check that the right API calls got made. # Check that create_job was not called. api = mock_context.get_api('west') assert api.create_job.call_count == 0 assert api.scheduler.getTasksStatus.call_count == 0
def test_restart_failed_status(self): # Test the client-side updater logic in its simplest case: everything succeeds, and no rolling # updates. (mock_api, mock_scheduler_proxy) = self.create_mock_api() mock_health_check = self.setup_health_checks(mock_api) self.setup_mock_scheduler_for_simple_restart(mock_api) mock_scheduler_proxy.getTasksStatus.return_value = self.create_error_response() with contextlib.nested( patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy), patch('apache.aurora.client.factory.CLUSTERS', new=self.TEST_CLUSTERS), patch('apache.aurora.client.api.instance_watcher.InstanceWatcherHealthCheck', return_value=mock_health_check), patch('time.time', side_effect=functools.partial(self.fake_time, self)), patch('time.sleep', return_value=None)): with temporary_file() as fp: fp.write(self.get_valid_config()) fp.flush() cmd = AuroraCommandLine() result = cmd.execute(['job', 'restart', '--batch_size=5', 'west/bozo/test/hello', fp.name]) assert mock_scheduler_proxy.getTasksStatus.call_count == 1 assert mock_scheduler_proxy.restartShards.call_count == 0 assert result == EXIT_API_ERROR