def test_do_bounce_when_create_app_and_new_app_running(self): fake_task_to_drain = mock.Mock(app_id='fake_app_to_kill_1') fake_bounce_func_return = { 'create_app': True, 'tasks_to_drain': [fake_task_to_drain], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 5} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {'fake_app_to_kill_1': set([fake_task_to_drain])} fake_old_app_draining_tasks = {'fake_app_to_kill_1': set()} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False) fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) expected_drain_task_count = len(fake_bounce_func_return['tasks_to_drain']) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) first_logged_line = mock_log.mock_calls[0][2]["line"] assert '%s new tasks' % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert 'draining %s old tasks' % expected_drain_task_count in second_logged_line assert mock_log.call_count == 2 assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == 0 assert mock_kill_old_ids.call_count == 0 assert fake_drain_method.drain.call_count == len(fake_bounce_func_return["tasks_to_drain"])
def kill_marathon_app(full_appid, cluster, client, soa_dir): service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid)) service_instance_config = marathon_tools.load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, ) complete_config = service_instance_config.format_marathon_app_dict() nerve_ns = service_instance_config.get_nerve_namespace() service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns) drain_method = drain_lib.get_drain_method( service_instance_config.get_drain_method(service_namespace_config), service=service, instance=instance, nerve_ns=nerve_ns, drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config), ) bounce_func = bounce_lib.get_bounce_method_func('down') while marathon_tools.is_app_id_running(app_id=full_appid, client=client): app_to_kill = client.get_app(full_appid) ( old_app_live_happy_tasks, old_app_live_unhappy_tasks, old_app_draining_tasks, old_app_at_risk_tasks, ) = get_tasks_by_state( other_apps=[app_to_kill], drain_method=drain_method, service=service, nerve_ns=nerve_ns, bounce_health_params=service_instance_config.get_bounce_health_params(service_namespace_config), ) do_bounce( bounce_func=bounce_func, drain_method=drain_method, config=complete_config, new_app_running='', happy_new_tasks=[], old_app_live_happy_tasks=old_app_live_happy_tasks, old_app_live_unhappy_tasks=old_app_live_unhappy_tasks, old_app_draining_tasks=old_app_draining_tasks, old_app_at_risk_tasks=old_app_at_risk_tasks, serviceinstance="{}.{}".format(service, instance), bounce_method='down', service=service, cluster=cluster, instance=instance, marathon_jobid=full_appid, client=client, soa_dir=soa_dir, ) paasta_print("Sleeping for 10 seconds to give the tasks time to drain") time.sleep(10) paasta_print("Sucessfully killed {}".format(full_appid))
def kill_marathon_app(full_appid, cluster, client, soa_dir): service, instance, _, __ = (s.replace("--", "_") for s in decompose_job_id(full_appid)) service_instance_config = marathon_tools.load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir) complete_config = service_instance_config.format_marathon_app_dict() registrations = service_instance_config.get_registrations() service_namespace_config = marathon_tools.load_service_namespace_config( service=service, namespace=registrations[0]) drain_method = drain_lib.get_drain_method( service_instance_config.get_drain_method(service_namespace_config), service=service, instance=instance, registrations=registrations, drain_method_params=service_instance_config.get_drain_method_params( service_namespace_config), ) bounce_func = bounce_lib.get_bounce_method_func("down") while marathon_tools.is_app_id_running(app_id=full_appid, client=client): app_to_kill = client.get_app(full_appid) ( old_app_live_happy_tasks, old_app_live_unhappy_tasks, old_app_draining_tasks, old_app_at_risk_tasks, ) = get_tasks_by_state( other_apps=[app_to_kill], drain_method=drain_method, service=service, nerve_ns=registrations[0], bounce_health_params=service_instance_config. get_bounce_health_params(service_namespace_config), ) do_bounce( bounce_func=bounce_func, drain_method=drain_method, config=complete_config, new_app_running="", happy_new_tasks=[], old_app_live_happy_tasks=old_app_live_happy_tasks, old_app_live_unhappy_tasks=old_app_live_unhappy_tasks, old_app_draining_tasks=old_app_draining_tasks, old_app_at_risk_tasks=old_app_at_risk_tasks, serviceinstance=f"{service}.{instance}", bounce_method="down", service=service, cluster=cluster, instance=instance, marathon_jobid=full_appid, client=client, soa_dir=soa_dir, ) paasta_print("Sleeping for 10 seconds to give the tasks time to drain") time.sleep(10) paasta_print(f"Successfully killed {full_appid}")
def test_do_bounce_when_tasks_to_drain(self): fake_task_to_drain = mock.Mock(app_id="fake_app_to_kill_1") fake_bounce_func_return = {"create_app": False, "tasks_to_drain": [fake_task_to_drain]} fake_bounce_func = mock.create_autospec(bounce_lib.brutal_bounce, return_value=fake_bounce_func_return) fake_config = {"instances": 5} fake_new_app_running = True fake_happy_new_tasks = ["fake_one", "fake_two", "fake_three"] fake_old_app_live_tasks = {"fake_app_to_kill_1": set([fake_task_to_drain])} fake_old_app_draining_tasks = {"fake_app_to_kill_1": set([])} fake_service = "fake_service" fake_serviceinstance = "fake_service.fake_instance" self.fake_cluster = "fake_cluster" fake_instance = "fake_instance" fake_bounce_method = "fake_bounce_method" fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False) fake_marathon_jobid = "fake.marathon.jobid" fake_client = mock.create_autospec(marathon.MarathonClient) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) expected_drain_task_count = len(fake_bounce_func_return["tasks_to_drain"]) with contextlib.nested( mock.patch("paasta_tools.setup_marathon_job._log", autospec=True), mock.patch("paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app", autospec=True), mock.patch("paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids", autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir="fake_soa_dir", ) # assert mock_log.call_count == 3 first_logged_line = mock_log.mock_calls[0][2]["line"] assert "%s new tasks" % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert ( "draining %s old tasks with app_id %s" % (expected_drain_task_count, "fake_app_to_kill_1") in second_logged_line ) assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == 0 assert mock_kill_old_ids.call_count == 0 assert fake_drain_method.drain.call_count == expected_drain_task_count
def test_do_bounce_when_apps_to_kill(self): fake_bounce_func_return = {"create_app": False, "tasks_to_drain": []} fake_bounce_func = mock.create_autospec(bounce_lib.brutal_bounce, return_value=fake_bounce_func_return) fake_config = {"instances": 5} fake_new_app_running = True fake_happy_new_tasks = ["fake_one", "fake_two", "fake_three"] fake_old_app_live_tasks = {"fake_app_to_kill_1": set()} fake_old_app_draining_tasks = {"fake_app_to_kill_1": set()} fake_service = "fake_service" fake_serviceinstance = "fake_service.fake_instance" self.fake_cluster = "fake_cluster" fake_instance = "fake_instance" fake_bounce_method = "fake_bounce_method" fake_drain_method = mock.Mock() fake_marathon_jobid = "fake.marathon.jobid" fake_client = mock.create_autospec(marathon.MarathonClient) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) with contextlib.nested( mock.patch("paasta_tools.setup_marathon_job._log", autospec=True), mock.patch("paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app", autospec=True), mock.patch("paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids", autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir="fake_soa_dir", ) assert mock_log.call_count == 3 first_logged_line = mock_log.mock_calls[0][2]["line"] assert "%s new tasks" % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert "removing old unused apps with app_ids: %s" % "fake_app_to_kill_1" in second_logged_line assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == len(fake_bounce_func_return["tasks_to_drain"]) assert mock_kill_old_ids.call_count == 1 third_logged_line = mock_log.mock_calls[2][2]["line"] assert "%s bounce on %s finish" % (fake_bounce_method, fake_serviceinstance) in third_logged_line assert "Now running %s" % fake_marathon_jobid in third_logged_line
def test_do_bounce_when_nothing_to_do(self): fake_bounce_func_return = {"create_app": False, "tasks_to_drain": []} fake_bounce_func = mock.create_autospec(bounce_lib.brutal_bounce, return_value=fake_bounce_func_return) fake_config = {"instances": 3} fake_new_app_running = True fake_happy_new_tasks = ["fake_one", "fake_two", "fake_three"] fake_old_app_live_tasks = {} fake_old_app_draining_tasks = {} fake_service = "fake_service" fake_serviceinstance = "fake_service.fake_instance" self.fake_cluster = "fake_cluster" fake_instance = "fake_instance" fake_bounce_method = "fake_bounce_method" fake_drain_method = mock.Mock() fake_marathon_jobid = "fake.marathon.jobid" fake_client = mock.create_autospec(marathon.MarathonClient) with contextlib.nested( mock.patch("paasta_tools.setup_marathon_job._log", autospec=True), mock.patch("paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app", autospec=True), mock.patch("paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids", autospec=True), mock.patch("paasta_tools.setup_marathon_job.send_sensu_bounce_keepalive", autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids, mock_send_sensu_bounce_keepalive): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir="fake_soa_dir", ) assert mock_log.call_count == 0 assert mock_create_marathon_app.call_count == 0 assert fake_drain_method.drain.call_count == 0 assert mock_kill_old_ids.call_count == 0 # When doing nothing, we need to send the keepalive heartbeat to Sensu mock_send_sensu_bounce_keepalive.assert_called_once_with( service=fake_service, instance=fake_instance, cluster=self.fake_cluster, soa_dir="fake_soa_dir" )
def test_do_bounce_when_nothing_to_do(self): fake_bounce_func_return = { 'create_app': False, 'tasks_to_drain': [], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 3} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {} fake_old_app_draining_tasks = {} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock() fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), mock.patch('paasta_tools.setup_marathon_job.send_sensu_bounce_keepalive', autospec=True), ) as ( mock_log, mock_create_marathon_app, mock_kill_old_ids, mock_send_sensu_bounce_keepalive, ): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 0 assert mock_create_marathon_app.call_count == 0 assert fake_drain_method.drain.call_count == 0 assert mock_kill_old_ids.call_count == 0 # When doing nothing, we need to send the keepalive heartbeat to Sensu mock_send_sensu_bounce_keepalive.assert_called_once_with( service=fake_service, instance=fake_instance, cluster=self.fake_cluster, soa_dir='fake_soa_dir', )
def test_do_bounce_when_apps_to_kill(self): fake_bounce_func_return = { 'create_app': False, 'tasks_to_drain': [], } fake_bounce_func = mock.create_autospec( bounce_lib.brutal_bounce, return_value=fake_bounce_func_return, ) fake_config = {'instances': 5} fake_new_app_running = True fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three'] fake_old_app_live_tasks = {'fake_app_to_kill_1': set()} fake_old_app_draining_tasks = {'fake_app_to_kill_1': set()} fake_service = 'fake_service' fake_serviceinstance = 'fake_service.fake_instance' self.fake_cluster = 'fake_cluster' fake_instance = 'fake_instance' fake_bounce_method = 'fake_bounce_method' fake_drain_method = mock.Mock() fake_marathon_jobid = 'fake.marathon.jobid' fake_client = mock.create_autospec( marathon.MarathonClient ) expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks) with contextlib.nested( mock.patch('paasta_tools.setup_marathon_job._log', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True), mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True), ) as (mock_log, mock_create_marathon_app, mock_kill_old_ids): setup_marathon_job.do_bounce( bounce_func=fake_bounce_func, drain_method=fake_drain_method, config=fake_config, new_app_running=fake_new_app_running, happy_new_tasks=fake_happy_new_tasks, old_app_live_tasks=fake_old_app_live_tasks, old_app_draining_tasks=fake_old_app_draining_tasks, service=fake_service, bounce_method=fake_bounce_method, serviceinstance=fake_serviceinstance, cluster=self.fake_cluster, instance=fake_instance, marathon_jobid=fake_marathon_jobid, client=fake_client, soa_dir='fake_soa_dir', ) assert mock_log.call_count == 3 first_logged_line = mock_log.mock_calls[0][2]["line"] assert '%s new tasks' % expected_new_task_count in first_logged_line second_logged_line = mock_log.mock_calls[1][2]["line"] assert 'removing old unused apps with app_ids: %s' % 'fake_app_to_kill_1' in second_logged_line assert mock_create_marathon_app.call_count == 0 assert fake_client.kill_task.call_count == len(fake_bounce_func_return["tasks_to_drain"]) assert mock_kill_old_ids.call_count == 1 third_logged_line = mock_log.mock_calls[2][2]["line"] assert '%s bounce on %s finish' % (fake_bounce_method, fake_serviceinstance) in third_logged_line assert 'Now running %s' % fake_marathon_jobid in third_logged_line
def main(): args = parse_args() full_appid = args.appname.lstrip('/') soa_dir = args.soa_dir marathon_config = marathon_tools.load_marathon_config() client = marathon_tools.get_marathon_client( url=marathon_config.get_url(), user=marathon_config.get_username(), passwd=marathon_config.get_password(), ) if not marathon_tools.is_app_id_running(app_id=full_appid, client=client): print("Couldn't find an app named {0}".format(full_appid)) sys.exit(1) service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid)) complete_config = marathon_tools.create_complete_config( service, instance, marathon_config) cluster = load_system_paasta_config().get_cluster() service_instance_config = marathon_tools.load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, ) nerve_ns = service_instance_config.get_nerve_namespace() service_namespace_config = marathon_tools.load_service_namespace_config( service=service, namespace=nerve_ns) drain_method = drain_lib.get_drain_method( service_instance_config.get_drain_method(service_namespace_config), service=service, instance=instance, nerve_ns=nerve_ns, drain_method_params=service_instance_config.get_drain_method_params( service_namespace_config), ) bounce_func = bounce_lib.get_bounce_method_func('down') while marathon_tools.is_app_id_running(app_id=full_appid, client=client): app_to_kill = client.get_app(full_appid) old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks( [app_to_kill], drain_method) do_bounce( bounce_func=bounce_func, drain_method=drain_method, config=complete_config, new_app_running='', happy_new_tasks=[], old_app_live_tasks=old_app_live_tasks, old_app_draining_tasks=old_app_draining_tasks, serviceinstance="{0}.{1}".format(service, instance), bounce_method='down', service=service, cluster=cluster, instance=instance, marathon_jobid=full_appid, client=client, soa_dir=soa_dir, ) print "Sleeping for 10 seconds to give the tasks time to drain" time.sleep(10) print("Sucessfully killed {0}".format(full_appid))
def main(): args = parse_args() full_appid = args.appname.lstrip('/') soa_dir = args.soa_dir marathon_config = marathon_tools.load_marathon_config() client = marathon_tools.get_marathon_client( url=marathon_config.get_url(), user=marathon_config.get_username(), passwd=marathon_config.get_password(), ) if not marathon_tools.is_app_id_running(app_id=full_appid, client=client): print("Couldn't find an app named {0}".format(full_appid)) sys.exit(1) service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid)) complete_config = marathon_tools.create_complete_config(service, instance, marathon_config) cluster = load_system_paasta_config().get_cluster() service_instance_config = marathon_tools.load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, ) nerve_ns = service_instance_config.get_nerve_namespace() service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns) drain_method = drain_lib.get_drain_method( service_instance_config.get_drain_method(service_namespace_config), service=service, instance=instance, nerve_ns=nerve_ns, drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config), ) bounce_func = bounce_lib.get_bounce_method_func('down') while marathon_tools.is_app_id_running(app_id=full_appid, client=client): app_to_kill = client.get_app(full_appid) old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks([app_to_kill], drain_method) do_bounce( bounce_func=bounce_func, drain_method=drain_method, config=complete_config, new_app_running='', happy_new_tasks=[], old_app_live_tasks=old_app_live_tasks, old_app_draining_tasks=old_app_draining_tasks, serviceinstance="{0}.{1}".format(service, instance), bounce_method='down', service=service, cluster=cluster, instance=instance, marathon_jobid=full_appid, client=client, soa_dir=soa_dir, ) print "Sleeping for 10 seconds to give the tasks time to drain" time.sleep(10) print("Sucessfully killed {0}".format(full_appid))