def setup_marathon_client(): system_paasta_config = setup_system_paasta_config() marathon_config = marathon_tools.MarathonConfig( system_paasta_config.get_marathon_config()) client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password()) return (client, marathon_config, system_paasta_config)
def test_instances_status_marathon( mock_get_actual_deployments, mock_validate_service_instance, mock_load_marathon_service_config, mock_get_matching_appids, mock_marathon_job_status, ): settings.cluster = 'fake_cluster' mock_get_actual_deployments.return_value = { 'fake_cluster.fake_instance': 'GIT_SHA', 'fake_cluster.fake_instance2': 'GIT_SHA', 'fake_cluster2.fake_instance': 'GIT_SHA', 'fake_cluster2.fake_instance2': 'GIT_SHA', } mock_validate_service_instance.return_value = 'marathon' mock_marathon_config = marathon_tools.MarathonConfig( { 'url': 'fake_url', 'user': '******', 'password': '******' }, ) settings.marathon_client = marathon_tools.get_marathon_client( mock_marathon_config.get_url(), mock_marathon_config.get_username(), mock_marathon_config.get_password(), ) mock_get_matching_appids.return_value = ['a', 'b'] mock_service_config = marathon_tools.MarathonServiceConfig( service='fake_service', cluster='fake_cluster', instance='fake_instance', config_dict={'bounce_method': 'fake_bounce'}, branch_dict={}, ) mock_load_marathon_service_config.return_value = mock_service_config mock_marathon_job_status.return_value = 'fake_marathon_status' request = testing.DummyRequest() request.swagger_data = { 'service': 'fake_service', 'instance': 'fake_instance' } response = instance.instance_status(request) assert response['marathon']['bounce_method'] == 'fake_bounce' assert response['marathon']['desired_state'] == 'start'
def setup_marathon_client(): marathon_connection_string = _get_marathon_connection_string() zk_connection_string = _get_zookeeper_connection_string('mesos-testcluster') marathon_config = marathon_tools.MarathonConfig({ 'url': marathon_connection_string, 'user': None, 'password': None, }, '/some_fake_path_to_marathon.json') client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password()) system_paasta_config = utils.SystemPaastaConfig({ 'cluster': 'testcluster', 'docker_volumes': [], 'docker_registry': u'docker-dev.yelpcorp.com', 'zookeeper': zk_connection_string }, '/some_fake_path_to_config_dir/') return (client, marathon_config, system_paasta_config)
class TestSetupMarathonJob: fake_docker_image = 'test_docker:1.0' fake_cluster = 'fake_test_cluster' fake_marathon_service_config = marathon_tools.MarathonServiceConfig( service='servicename', cluster='clustername', instance='instancename', config_dict={ 'instances': 3, 'cpus': 1, 'mem': 100, 'docker_image': fake_docker_image, 'nerve_ns': 'aaaaugh', 'bounce_method': 'brutal' }, branch_dict={}, ) fake_docker_registry = 'remote_registry.com' fake_marathon_config = marathon_tools.MarathonConfig({ 'url': 'http://test_url', 'user': '******', 'password': '******', }, '/fake/fake_file.json') fake_args = mock.MagicMock( service_instance='what_is_love.bby_dont_hurt_me', soa_dir='no_more', verbose=False, ) fake_service_namespace_config = marathon_tools.ServiceNamespaceConfig({ 'mode': 'http' }) def test_main_success(self): fake_client = mock.MagicMock() with contextlib.nested( mock.patch( 'paasta_tools.setup_marathon_job.parse_args', return_value=self.fake_args, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.get_main_marathon_config', return_value=self.fake_marathon_config, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.get_marathon_client', return_value=fake_client, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.load_marathon_service_config', return_value=self.fake_marathon_service_config, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.setup_service', return_value=(0, 'it_is_finished'), autospec=True, ), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), mock.patch('paasta_tools.setup_marathon_job.send_event', autospec=True), mock.patch('sys.exit', autospec=True), ) as ( parse_args_patch, get_main_conf_patch, get_client_patch, read_service_conf_patch, setup_service_patch, load_system_paasta_config_patch, sensu_patch, sys_exit_patch, ): load_system_paasta_config_patch.return_value.get_cluster = mock.Mock(return_value=self.fake_cluster) setup_marathon_job.main() parse_args_patch.assert_called_once_with() get_main_conf_patch.assert_called_once_with() get_client_patch.assert_called_once_with( self.fake_marathon_config.get_url(), self.fake_marathon_config.get_username(), self.fake_marathon_config.get_password(), ) read_service_conf_patch.assert_called_once_with( decompose_job_id(self.fake_args.service_instance)[0], decompose_job_id(self.fake_args.service_instance)[1], self.fake_cluster, soa_dir=self.fake_args.soa_dir, ) setup_service_patch.assert_called_once_with( decompose_job_id(self.fake_args.service_instance)[0], decompose_job_id(self.fake_args.service_instance)[1], fake_client, self.fake_marathon_config, self.fake_marathon_service_config, 'no_more', ) sys_exit_patch.assert_called_once_with(0) def test_main_failure(self): fake_client = mock.MagicMock() with contextlib.nested( mock.patch( 'paasta_tools.setup_marathon_job.parse_args', return_value=self.fake_args, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.get_main_marathon_config', return_value=self.fake_marathon_config, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.get_marathon_client', return_value=fake_client, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.load_marathon_service_config', return_value=self.fake_marathon_service_config, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.setup_service', return_value=(1, 'NEVER'), autospec=True, ), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), mock.patch('paasta_tools.setup_marathon_job.send_event', autospec=True), mock.patch('sys.exit', autospec=True), ) as ( parse_args_patch, get_main_conf_patch, get_client_patch, read_service_conf_patch, setup_service_patch, load_system_paasta_config_patch, sensu_patch, sys_exit_patch, ): load_system_paasta_config_patch.return_value.get_cluster = mock.Mock(return_value=self.fake_cluster) setup_marathon_job.main() parse_args_patch.assert_called_once_with() get_main_conf_patch.assert_called_once_with() get_client_patch.assert_called_once_with( self.fake_marathon_config.get_url(), self.fake_marathon_config.get_username(), self.fake_marathon_config.get_password()) read_service_conf_patch.assert_called_once_with( decompose_job_id(self.fake_args.service_instance)[0], decompose_job_id(self.fake_args.service_instance)[1], self.fake_cluster, soa_dir=self.fake_args.soa_dir) setup_service_patch.assert_called_once_with( decompose_job_id(self.fake_args.service_instance)[0], decompose_job_id(self.fake_args.service_instance)[1], fake_client, self.fake_marathon_config, self.fake_marathon_service_config, 'no_more', ) sys_exit_patch.assert_called_once_with(0) def test_main_sends_event_if_no_deployments(self): fake_client = mock.MagicMock() with contextlib.nested( mock.patch( 'paasta_tools.setup_marathon_job.parse_args', return_value=self.fake_args, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.get_main_marathon_config', return_value=self.fake_marathon_config, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.get_marathon_client', return_value=fake_client, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.load_marathon_service_config', side_effect=NoDeploymentsAvailable(), autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.setup_service', return_value=(1, 'NEVER'), autospec=True, ), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), mock.patch('paasta_tools.setup_marathon_job.send_event', autospec=True), ) as ( parse_args_patch, get_main_conf_patch, get_client_patch, read_service_conf_patch, setup_service_patch, load_system_paasta_config_patch, sensu_patch, ): load_system_paasta_config_patch.return_value.get_cluster = mock.Mock(return_value=self.fake_cluster) with raises(SystemExit) as exc_info: setup_marathon_job.main() parse_args_patch.assert_called_once_with() get_main_conf_patch.assert_called_once_with() get_client_patch.assert_called_once_with( self.fake_marathon_config.get_url(), self.fake_marathon_config.get_username(), self.fake_marathon_config.get_password()) read_service_conf_patch.assert_called_once_with( decompose_job_id(self.fake_args.service_instance)[0], decompose_job_id(self.fake_args.service_instance)[1], self.fake_cluster, soa_dir=self.fake_args.soa_dir) expected_string = 'No deployments found for %s in cluster %s' % ( self.fake_args.service_instance, self.fake_cluster) sensu_patch.assert_called_once_with( decompose_job_id(self.fake_args.service_instance)[0], decompose_job_id(self.fake_args.service_instance)[1], self.fake_args.soa_dir, Status.CRITICAL, expected_string ) assert exc_info.value.code == 0 def test_send_event(self): fake_service = 'fake_service' fake_instance = 'fake_instance' fake_status = '42' fake_output = 'The http port is not open' fake_soa_dir = '' expected_check_name = 'setup_marathon_job.%s' % compose_job_id(fake_service, fake_instance) with contextlib.nested( mock.patch("paasta_tools.monitoring_tools.send_event", autospec=True), mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True), mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True), ) as ( send_event_patch, load_marathon_service_config_patch, load_system_paasta_config_patch, ): load_system_paasta_config_patch.return_value.get_cluster = mock.Mock(return_value='fake_cluster') load_marathon_service_config_patch.return_value.get_monitoring.return_value = {} setup_marathon_job.send_event( fake_service, fake_instance, fake_soa_dir, fake_status, fake_output ) send_event_patch.assert_called_once_with( fake_service, expected_check_name, {'alert_after': '10m', 'check_every': '10s'}, fake_status, fake_output, fake_soa_dir ) load_marathon_service_config_patch.assert_called_once_with( fake_service, fake_instance, load_system_paasta_config_patch.return_value.get_cluster.return_value, load_deployments=False, ) def test_send_bounce_keepalive(self): fake_service = 'fake_service' fake_instance = 'fake_instance' fake_cluster = 'fake_cluster' fake_soa_dir = '' expected_check_name = 'paasta_bounce_progress.%s' % compose_job_id(fake_service, fake_instance) with contextlib.nested( mock.patch("paasta_tools.monitoring_tools.send_event", autospec=True), mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True), ) as ( send_event_patch, load_marathon_service_config_patch, ): load_marathon_service_config_patch.return_value.get_monitoring.return_value = {} setup_marathon_job.send_sensu_bounce_keepalive( service=fake_service, instance=fake_instance, cluster=fake_cluster, soa_dir=fake_soa_dir, ) send_event_patch.assert_called_once_with( service=fake_service, check_name=expected_check_name, overrides=mock.ANY, status=0, output=mock.ANY, soa_dir=fake_soa_dir, ttl='1h', ) load_marathon_service_config_patch.assert_called_once_with( service=fake_service, instance=fake_instance, cluster=fake_cluster, load_deployments=False, ) def test_do_bounce_when_create_app_and_new_app_not_running(self): fake_bounce_func_return = { 'create_app': True, 'tasks_to_drain': [mock.Mock(app_id='fake_task_to_kill_1')], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 5} fake_new_app_running = False fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {} fake_old_app_draining_tasks = {} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False) fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) expected_drain_task_count = len(fake_bounce_func_return['tasks_to_drain']) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 3 first_logged_line = mock_log.mock_calls[0][2]["line"] assert '%s new tasks' % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert 'creating new app with app_id %s' % fake_marathon_jobid in second_logged_line third_logged_line = mock_log.mock_calls[2][2]["line"] assert 'draining %s old tasks' % expected_drain_task_count in third_logged_line assert mock_create_marathon_app.call_count == 1 assert fake_client.kill_task.call_count == 0 assert fake_drain_method.drain.call_count == len(fake_bounce_func_return["tasks_to_drain"]) assert mock_kill_old_ids.call_count == 0 def test_do_bounce_when_create_app_and_new_app_running(self): fake_task_to_drain = mock.Mock(app_id='fake_app_to_kill_1') fake_bounce_func_return = { 'create_app': True, 'tasks_to_drain': [fake_task_to_drain], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 5} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {'fake_app_to_kill_1': set([fake_task_to_drain])} fake_old_app_draining_tasks = {'fake_app_to_kill_1': set()} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False) fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) expected_drain_task_count = len(fake_bounce_func_return['tasks_to_drain']) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) first_logged_line = mock_log.mock_calls[0][2]["line"] assert '%s new tasks' % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert 'draining %s old tasks' % expected_drain_task_count in second_logged_line assert mock_log.call_count == 2 assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == 0 assert mock_kill_old_ids.call_count == 0 assert fake_drain_method.drain.call_count == len(fake_bounce_func_return["tasks_to_drain"]) def test_do_bounce_when_tasks_to_drain(self): fake_task_to_drain = mock.Mock(app_id='fake_app_to_kill_1') fake_bounce_func_return = { 'create_app': False, 'tasks_to_drain': [fake_task_to_drain], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 5} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {'fake_app_to_kill_1': set([fake_task_to_drain])} fake_old_app_draining_tasks = {'fake_app_to_kill_1': set([])} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False) fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) expected_drain_task_count = len(fake_bounce_func_return['tasks_to_drain']) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) # assert mock_log.call_count == 3 first_logged_line = mock_log.mock_calls[0][2]["line"] assert '%s new tasks' % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert 'draining %s old tasks with app_id %s' % (expected_drain_task_count, 'fake_app_to_kill_1') \ in second_logged_line assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == 0 assert mock_kill_old_ids.call_count == 0 assert fake_drain_method.drain.call_count == expected_drain_task_count def test_do_bounce_when_apps_to_kill(self): fake_bounce_func_return = { 'create_app': False, 'tasks_to_drain': [], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 5} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {'fake_app_to_kill_1': set()} fake_old_app_draining_tasks = {'fake_app_to_kill_1': set()} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock() fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 3 first_logged_line = mock_log.mock_calls[0][2]["line"] assert '%s new tasks' % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert 'removing old unused apps with app_ids: %s' % 'fake_app_to_kill_1' in second_logged_line assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == len(fake_bounce_func_return["tasks_to_drain"]) assert mock_kill_old_ids.call_count == 1 third_logged_line = mock_log.mock_calls[2][2]["line"] assert '%s bounce on %s finish' % (fake_bounce_method, fake_serviceinstance) in third_logged_line assert 'Now running %s' % fake_marathon_jobid in third_logged_line def test_do_bounce_when_nothing_to_do(self): fake_bounce_func_return = { 'create_app': False, 'tasks_to_drain': [], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 3} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {} fake_old_app_draining_tasks = {} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock() fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), mock.patch('paasta_tools.setup_marathon_job.send_sensu_bounce_keepalive', autospec=True), ) as ( mock_log, mock_create_marathon_app, mock_kill_old_ids, mock_send_sensu_bounce_keepalive, ): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 0 assert mock_create_marathon_app.call_count == 0 assert fake_drain_method.drain.call_count == 0 assert mock_kill_old_ids.call_count == 0 # When doing nothing, we need to send the keepalive heartbeat to Sensu mock_send_sensu_bounce_keepalive.assert_called_once_with( service=fake_service, instance=fake_instance, cluster=self.fake_cluster, soa_dir='fake_soa_dir', ) def test_setup_service_srv_already_exists(self): fake_name = 'if_trees_could_talk' fake_instance = 'would_they_scream' fake_client = mock.MagicMock(get_app=mock.Mock(return_value=True)) full_id = marathon_tools.format_job_id(fake_name, fake_instance) fake_complete = { 'seven': 'full', 'eight': 'frightened', 'nine': 'eaten', 'id': full_id, } with contextlib.nested( mock.patch( 'paasta_tools.marathon_tools.create_complete_config', return_value=fake_complete, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.load_marathon_config', return_value=self.fake_marathon_config, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.deploy_service', autospec=True, ), ) as ( create_config_patch, get_config_patch, deploy_service_patch, ): setup_marathon_job.setup_service( service=fake_name, instance=fake_instance, client=fake_client, marathon_config=self.fake_marathon_config, service_marathon_config=self.fake_marathon_service_config, soa_dir=None, ) create_config_patch.assert_called_once_with( fake_name, fake_instance, self.fake_marathon_config, ) assert deploy_service_patch.call_count == 1 def test_setup_service_srv_does_not_exist(self): fake_name = 'if_talk_was_cheap' fake_instance = 'psychatrists_would_be_broke' fake_response = mock.Mock( json=mock.Mock(return_value={'message': 'test'})) fake_client = mock.MagicMock(get_app=mock.Mock( side_effect=marathon.exceptions.NotFoundError(fake_response))) full_id = marathon_tools.format_job_id(fake_name, fake_instance, 'oogabooga', 'bananafanafofooga') fake_complete = { 'do': 'you', 'even': 'dota', 'id': full_id, 'docker_image': 'fake_docker_registry/fake_docker_image', } fake_bounce = 'trampoline' fake_drain_method = 'noop' fake_drain_method_params = {} with contextlib.nested( mock.patch( 'paasta_tools.marathon_tools.create_complete_config', return_value=fake_complete, autospec=True, ), mock.patch( 'paasta_tools.setup_marathon_job.deploy_service', return_value=(111, 'Never'), autospec=True, ), mock.patch.object( self.fake_marathon_service_config, 'get_bounce_method', return_value=fake_bounce, autospec=True, ), mock.patch.object( self.fake_marathon_service_config, 'get_drain_method', return_value=fake_drain_method, autospec=True, ), mock.patch.object( self.fake_marathon_service_config, 'get_drain_method_params', return_value=fake_drain_method_params, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.load_marathon_service_config', return_value=self.fake_marathon_service_config, autospec=True, ), mock.patch( 'paasta_tools.marathon_tools.load_service_namespace_config', return_value=self.fake_service_namespace_config, autospec=True, ), ) as ( create_config_patch, deploy_service_patch, get_bounce_patch, get_drain_method_patch, get_drain_method_params_patch, read_service_conf_patch, read_namespace_conf_patch, ): status, output = setup_marathon_job.setup_service( service=fake_name, instance=fake_instance, client=fake_client, marathon_config=self.fake_marathon_config, service_marathon_config=self.fake_marathon_service_config, soa_dir=None, ) assert status == 111 assert output == 'Never' create_config_patch.assert_called_once_with( fake_name, fake_instance, self.fake_marathon_config ) get_bounce_patch.assert_called_once_with() get_drain_method_patch.assert_called_once_with(read_namespace_conf_patch.return_value) deploy_service_patch.assert_called_once_with( service=fake_name, instance=fake_instance, marathon_jobid=full_id, config=fake_complete, client=fake_client, bounce_method=fake_bounce, drain_method_name=fake_drain_method, drain_method_params=fake_drain_method_params, nerve_ns=self.fake_marathon_service_config.get_nerve_namespace(), bounce_health_params=self.fake_marathon_service_config.get_bounce_health_params( read_namespace_conf_patch.return_value), soa_dir=None, ) def test_setup_service_srv_complete_config_raises(self): fake_name = 'test_service' fake_instance = 'test_instance' with mock.patch( 'paasta_tools.setup_marathon_job.marathon_tools.create_complete_config', side_effect=NoDockerImageError, ): status, output = setup_marathon_job.setup_service( service=fake_name, instance=fake_instance, client=None, marathon_config=None, service_marathon_config=None, soa_dir=None, ) assert status == 1 expected = 'Docker image for test_service.test_instance not in' assert expected in output def test_deploy_service_unknown_drain_method(self): fake_bounce = 'exists' fake_drain_method = 'doesntexist' fake_name = 'whoa' fake_instance = 'the_earth_is_tiny' fake_id = marathon_tools.format_job_id(fake_name, fake_instance) fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[])] fake_client = mock.MagicMock( list_apps=mock.Mock(return_value=fake_apps)) fake_config = {'id': fake_id, 'instances': 2} errormsg = 'ERROR: drain_method not recognized: doesntexist. Must be one of (exists1, exists2)' expected = (1, errormsg) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), mock.patch( 'paasta_tools.drain_lib._drain_methods', new={'exists1': mock.Mock(), 'exists2': mock.Mock()}, ) ) as (mock_log, mock_load_system_paasta_config, mock_drain_methods): mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster') actual = setup_marathon_job.deploy_service( service=fake_name, instance=fake_instance, marathon_jobid=fake_id, config=fake_config, client=fake_client, bounce_method=fake_bounce, drain_method_name=fake_drain_method, drain_method_params={}, nerve_ns=fake_instance, bounce_health_params={}, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 1 assert expected == actual def test_deploy_service_unknown_bounce(self): fake_bounce = 'WHEEEEEEEEEEEEEEEE' fake_drain_method = 'noop' fake_name = 'whoa' fake_instance = 'the_earth_is_tiny' fake_id = marathon_tools.format_job_id(fake_name, fake_instance) fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[])] fake_client = mock.MagicMock( list_apps=mock.Mock(return_value=fake_apps)) fake_config = {'id': fake_id, 'instances': 2} errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \ (fake_bounce, ', '.join(list_bounce_methods())) expected = (1, errormsg) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), ) as (mock_log, mock_load_system_paasta_config): mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster') actual = setup_marathon_job.deploy_service( service=fake_name, instance=fake_instance, marathon_jobid=fake_id, config=fake_config, client=fake_client, bounce_method=fake_bounce, drain_method_name=fake_drain_method, drain_method_params={}, nerve_ns=fake_instance, bounce_health_params={}, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 1 assert expected == actual fake_client.list_apps.assert_called_once_with(embed_failures=True) assert fake_client.create_app.call_count == 0 def test_deploy_service_known_bounce(self): fake_bounce = 'areallygoodbouncestrategy' fake_drain_method_name = 'noop' fake_name = 'how_many_strings' fake_instance = 'will_i_need_to_think_of' fake_id = marathon_tools.format_job_id(fake_name, fake_instance, 'git11111111', 'config11111111') fake_config = {'id': fake_id, 'instances': 2} old_app_id = marathon_tools.format_job_id(fake_name, fake_instance, 'git22222222', 'config22222222') old_task_to_drain = mock.Mock(id="old_task_to_drain", app_id=old_app_id) old_task_is_draining = mock.Mock(id="old_task_is_draining", app_id=old_app_id) old_task_dont_drain = mock.Mock(id="old_task_dont_drain", app_id=old_app_id) old_app = mock.Mock(id="/%s" % old_app_id, tasks=[old_task_to_drain, old_task_is_draining, old_task_dont_drain]) fake_client = mock.MagicMock( list_apps=mock.Mock(return_value=[old_app]), kill_task=mock.Mock(spec=lambda app_id, id, scale=False: None), ) fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value={ "create_app": True, "tasks_to_drain": [old_task_to_drain], } ) fake_drain_method = mock.Mock(is_draining=lambda t: t is old_task_is_draining, is_safe_to_kill=lambda t: True) with contextlib.nested( mock.patch( 'paasta_tools.bounce_lib.get_bounce_method_func', return_value=fake_bounce_func, autospec=True, ), mock.patch( 'paasta_tools.bounce_lib.bounce_lock_zookeeper', autospec=True ), mock.patch( 'paasta_tools.bounce_lib.get_happy_tasks', autospec=True, side_effect=lambda x, _, __, **kwargs: x, ), mock.patch('paasta_tools.bounce_lib.kill_old_ids', autospec=True), mock.patch('paasta_tools.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), mock.patch('paasta_tools.drain_lib.get_drain_method', return_value=fake_drain_method), ) as (_, _, _, kill_old_ids_patch, create_marathon_app_patch, mock_log, mock_load_system_paasta_config, _): mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster') result = setup_marathon_job.deploy_service( service=fake_name, instance=fake_instance, marathon_jobid=fake_id, config=fake_config, client=fake_client, bounce_method=fake_bounce, drain_method_name=fake_drain_method_name, drain_method_params={}, nerve_ns=fake_instance, bounce_health_params={}, soa_dir='fake_soa_dir', ) assert result[0] == 0, "Expected successful result; got (%d, %s)" % result fake_client.list_apps.assert_called_once_with(embed_failures=True) assert fake_client.create_app.call_count == 0 fake_bounce_func.assert_called_once_with( new_config=fake_config, new_app_running=False, happy_new_tasks=[], old_app_live_tasks={old_app.id: set([old_task_to_drain, old_task_dont_drain])}, ) assert fake_drain_method.drain.call_count == 2 fake_drain_method.drain.assert_any_call(old_task_is_draining) fake_drain_method.drain.assert_any_call(old_task_to_drain) assert fake_client.kill_task.call_count == 2 fake_client.kill_task.assert_any_call(old_app_id, old_task_is_draining.id, scale=True) fake_client.kill_task.assert_any_call(old_app_id, old_task_to_drain.id, scale=True) create_marathon_app_patch.assert_called_once_with(fake_config['id'], fake_config, fake_client) assert kill_old_ids_patch.call_count == 0 # We should call _log 5 times: # 1. bounce starts # 2. create new app # 3. draining old tasks # 4. remove old apps # 5. bounce finishes assert mock_log.call_count == 5 def test_deploy_service_already_bouncing(self): fake_bounce = 'areallygoodbouncestrategy' fake_drain_method = 'noop' fake_name = 'how_many_strings' fake_instance = 'will_i_need_to_think_of' fake_id = marathon_tools.format_job_id(fake_name, fake_instance, 'gityourmom', 'configyourdad') fake_config = {'id': fake_id, 'instances': 2} old_app_id = ('%s2' % fake_id) old_task = mock.Mock(id="old_task_id", app_id=old_app_id) old_app = mock.Mock(id=old_app_id, tasks=[old_task]) fake_client = mock.MagicMock( list_apps=mock.Mock(return_value=[old_app]), kill_task=mock.Mock(spec=lambda app_id, id, scale=False: None), ) fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value={ "create_app": True, "tasks_to_drain": [old_task], } ) fake_short_id = marathon_tools.format_job_id(fake_name, fake_instance) with contextlib.nested( mock.patch( 'paasta_tools.bounce_lib.get_bounce_method_func', return_value=fake_bounce_func, autospec=True, ), mock.patch( 'paasta_tools.bounce_lib.bounce_lock_zookeeper', side_effect=bounce_lib.LockHeldException, autospec=True ), mock.patch( 'paasta_tools.bounce_lib.get_happy_tasks', autospec=True, side_effect=lambda x, _, __, **kwargs: x, ), mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), ) as (_, _, _, _, mock_load_system_paasta_config): mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster') result = setup_marathon_job.deploy_service( service=fake_name, instance=fake_instance, marathon_jobid=fake_id, config=fake_config, client=fake_client, bounce_method=fake_bounce, drain_method_name=fake_drain_method, drain_method_params={}, nerve_ns=fake_instance, bounce_health_params={}, soa_dir='fake_soa_dir', ) assert result == (1, "Instance %s is already being bounced." % fake_short_id) def test_deploy_service_logs_exceptions(self): fake_bounce = 'WHEEEEEEEEEEEEEEEE' fake_drain_method = 'noop' fake_name = 'whoa' fake_instance = 'the_earth_is_tiny' fake_id = marathon_tools.format_job_id(fake_name, fake_instance) fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[])] fake_client = mock.MagicMock( list_apps=mock.Mock(return_value=fake_apps)) fake_config = {'id': fake_id, 'instances': 2} with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.get_bounce_method_func', side_effect=IOError('foo')), mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True), ) as (mock_log, mock_bounce, mock_load_system_paasta_config): mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster') with raises(IOError): setup_marathon_job.deploy_service( service=fake_name, instance=fake_instance, marathon_jobid=fake_id, config=fake_config, client=fake_client, bounce_method=fake_bounce, drain_method_name=fake_drain_method, drain_method_params={}, nerve_ns=fake_instance, bounce_health_params={}, soa_dir='fake_soa_dir', ) assert fake_name in mock_log.mock_calls[0][2]["line"] assert 'Traceback' in mock_log.mock_calls[1][2]["line"] def test_get_marathon_config(self): fake_conf = {'oh_no': 'im_a_ghost'} with mock.patch( 'paasta_tools.marathon_tools.load_marathon_config', return_value=fake_conf, autospec=True ) as get_conf_patch: assert setup_marathon_job.get_main_marathon_config() == fake_conf get_conf_patch.assert_called_once_with() def test_get_old_live_draining_tasks_empty(self): fake_name = 'whoa' fake_instance = 'the_earth_is_tiny' fake_id = marathon_tools.format_job_id(fake_name, fake_instance) fake_apps = [ mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[]) ] expected_live_tasks = { fake_apps[0].id: set(), fake_apps[1].id: set(), } expected_draining_tasks = { fake_apps[0].id: set(), fake_apps[1].id: set(), } fake_drain_method = mock.Mock(is_draining=lambda _: True) actual = setup_marathon_job.get_old_live_draining_tasks(fake_apps, fake_drain_method) actual_live_tasks, actual_draining_tasks = actual assert actual_live_tasks == expected_live_tasks assert actual_draining_tasks == expected_draining_tasks def test_get_old_live_draining_tasks_not_empty(self): fake_name = 'whoa' fake_instance = 'the_earth_is_tiny' fake_id = marathon_tools.format_job_id(fake_name, fake_instance) def fake_task(state): return mock.Mock(_drain_state=state) fake_apps = [ mock.Mock(id=fake_id, tasks=[fake_task('up'), fake_task('down')]), mock.Mock(id=('%s2' % fake_id), tasks=[fake_task('up'), fake_task('down')]) ] expected_live_tasks = { fake_apps[0].id: set([fake_apps[0].tasks[0]]), fake_apps[1].id: set([fake_apps[1].tasks[0]]), } expected_draining_tasks = { fake_apps[0].id: set([fake_apps[0].tasks[1]]), fake_apps[1].id: set([fake_apps[1].tasks[1]]), } fake_drain_method = mock.Mock(is_draining=lambda t: t._drain_state == 'down') actual = setup_marathon_job.get_old_live_draining_tasks(fake_apps, fake_drain_method) actual_live_tasks, actual_draining_tasks = actual assert actual_live_tasks == expected_live_tasks assert actual_draining_tasks == expected_draining_tasks
class TestCleanupMarathonJobs: cleanup_marathon_jobs.log = mock.Mock() fake_docker_registry = 'http://del.icio.us/' fake_cluster = 'fake_test_cluster' fake_marathon_config = marathon_tools.MarathonConfig( { 'url': 'http://mess_url', 'user': '******', 'password': '******', }, '/some/fake/path/fake_file.json') fake_marathon_client = mock.Mock() def test_main(self): soa_dir = 'paasta_maaaachine' fake_args = mock.Mock(verbose=False, soa_dir=soa_dir) with contextlib.nested( mock.patch('paasta_tools.cleanup_marathon_jobs.parse_args', return_value=fake_args), mock.patch('paasta_tools.cleanup_marathon_jobs.cleanup_apps') ) as (args_patch, cleanup_patch): cleanup_marathon_jobs.main() args_patch.assert_called_once_with() cleanup_patch.assert_called_once_with(soa_dir) def test_cleanup_apps(self): soa_dir = 'not_really_a_dir' expected_apps = [('present', 'away'), ('on-app', 'off')] fake_app_ids = [ mock.Mock(id='present.away.gone.wtf'), mock.Mock(id='on-app.off.stop.jrm'), mock.Mock(id='not-here.oh.no.weirdo') ] self.fake_marathon_client.list_apps = mock.Mock( return_value=fake_app_ids) with contextlib.nested( mock.patch( 'paasta_tools.cleanup_marathon_jobs.get_services_for_cluster', return_value=expected_apps, autospec=True), mock.patch('paasta_tools.marathon_tools.load_marathon_config', autospec=True, return_value=self.fake_marathon_config), mock.patch('paasta_tools.marathon_tools.get_marathon_client', autospec=True, return_value=self.fake_marathon_client), mock.patch('paasta_tools.cleanup_marathon_jobs.delete_app', autospec=True), ) as ( get_services_for_cluster_patch, config_patch, client_patch, delete_patch, ): cleanup_marathon_jobs.cleanup_apps(soa_dir) config_patch.assert_called_once_with() get_services_for_cluster_patch.assert_called_once_with( instance_type='marathon', soa_dir=soa_dir) client_patch.assert_called_once_with( self.fake_marathon_config.get_url(), self.fake_marathon_config.get_username(), self.fake_marathon_config.get_password()) delete_patch.assert_called_once_with( app_id='not-here.oh.no.weirdo', client=self.fake_marathon_client, soa_dir=soa_dir, ) def test_cleanup_apps_doesnt_delete_unknown_apps(self): soa_dir = 'not_really_a_dir' expected_apps = [('present', 'away'), ('on-app', 'off')] fake_app_ids = [mock.Mock(id='non_conforming_app')] self.fake_marathon_client.list_apps = mock.Mock( return_value=fake_app_ids) with contextlib.nested( mock.patch( 'paasta_tools.cleanup_marathon_jobs.get_services_for_cluster', return_value=expected_apps, autospec=True), mock.patch('paasta_tools.marathon_tools.load_marathon_config', autospec=True, return_value=self.fake_marathon_config), mock.patch('paasta_tools.marathon_tools.get_marathon_client', autospec=True, return_value=self.fake_marathon_client), mock.patch('paasta_tools.cleanup_marathon_jobs.delete_app', autospec=True), ) as ( get_services_for_cluster_patch, config_patch, client_patch, delete_patch, ): cleanup_marathon_jobs.cleanup_apps(soa_dir) assert delete_patch.call_count == 0 def test_delete_app(self): app_id = 'example--service.main.git93340779.configddb38a65' client = self.fake_marathon_client with contextlib.nested( mock.patch( 'paasta_tools.cleanup_marathon_jobs.load_system_paasta_config', autospec=True), mock.patch('paasta_tools.bounce_lib.bounce_lock_zookeeper', autospec=True), mock.patch('paasta_tools.bounce_lib.delete_marathon_app', autospec=True), mock.patch('paasta_tools.cleanup_marathon_jobs._log', autospec=True), mock.patch('paasta_tools.cleanup_marathon_jobs.send_event', autospec=True)) as ( mock_load_system_paasta_config, mock_bounce_lock_zookeeper, mock_delete_marathon_app, mock_log, mock_send_sensu_event, ): mock_load_system_paasta_config.return_value.get_cluster = mock.Mock( return_value='fake_cluster') cleanup_marathon_jobs.delete_app(app_id, client, 'fake_soa_dir') mock_delete_marathon_app.assert_called_once_with(app_id, client) mock_load_system_paasta_config.return_value.get_cluster.assert_called_once_with( ) expected_log_line = ( 'Deleted stale marathon job that looks lost: ' + app_id) mock_log.assert_called_once_with( instance='main', service='example_service', level='event', component='deploy', cluster='fake_cluster', line=expected_log_line, ) def test_delete_app_throws_exception(self): app_id = 'example--service.main.git93340779.configddb38a65' client = self.fake_marathon_client with contextlib.nested( mock.patch( 'paasta_tools.cleanup_marathon_jobs.load_system_paasta_config', autospec=True), mock.patch('paasta_tools.bounce_lib.bounce_lock_zookeeper', autospec=True), mock.patch('paasta_tools.bounce_lib.delete_marathon_app', side_effect=ValueError('foo')), mock.patch('paasta_tools.cleanup_marathon_jobs._log', autospec=True), ) as ( mock_load_system_paasta_config, mock_bounce_lock_zookeeper, mock_delete_marathon_app, mock_log, ): with raises(ValueError): cleanup_marathon_jobs.delete_app(app_id, client, 'fake_soa_dir') assert 'example_service' in mock_log.mock_calls[0][2]["line"] assert 'Traceback' in mock_log.mock_calls[1][2]["line"]