def test_do_bounce_when_create_app_and_new_app_running(self):
fake_task_to_drain = mock.Mock(app_id='fake_app_to_kill_1')
fake_bounce_func_return = {
'create_app': True,
'tasks_to_drain': [fake_task_to_drain],
}
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce,
return_value=fake_bounce_func_return,
)
fake_config = {'instances': 5}
fake_new_app_running = True
fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three']
fake_old_app_live_tasks = {'fake_app_to_kill_1': set([fake_task_to_drain])}
fake_old_app_draining_tasks = {'fake_app_to_kill_1': set()}
fake_service = 'fake_service'
fake_serviceinstance = 'fake_service.fake_instance'
self.fake_cluster = 'fake_cluster'
fake_instance = 'fake_instance'
fake_bounce_method = 'fake_bounce_method'
fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False)
fake_marathon_jobid = 'fake.marathon.jobid'
fake_client = mock.create_autospec(
marathon.MarathonClient
)
expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks)
expected_drain_task_count = len(fake_bounce_func_return['tasks_to_drain'])
with contextlib.nested(
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True),
) as (mock_log, mock_create_marathon_app, mock_kill_old_ids):
setup_marathon_job.do_bounce(
bounce_func=fake_bounce_func,
drain_method=fake_drain_method,
config=fake_config,
new_app_running=fake_new_app_running,
happy_new_tasks=fake_happy_new_tasks,
old_app_live_tasks=fake_old_app_live_tasks,
old_app_draining_tasks=fake_old_app_draining_tasks,
service=fake_service,
bounce_method=fake_bounce_method,
serviceinstance=fake_serviceinstance,
cluster=self.fake_cluster,
instance=fake_instance,
marathon_jobid=fake_marathon_jobid,
client=fake_client,
soa_dir='fake_soa_dir',
)
first_logged_line = mock_log.mock_calls[0][2]["line"]
assert '%s new tasks' % expected_new_task_count in first_logged_line
second_logged_line = mock_log.mock_calls[1][2]["line"]
assert 'draining %s old tasks' % expected_drain_task_count in second_logged_line
assert mock_log.call_count == 2
assert mock_create_marathon_app.call_count == 0
assert fake_client.kill_task.call_count == 0
assert mock_kill_old_ids.call_count == 0
assert fake_drain_method.drain.call_count == len(fake_bounce_func_return["tasks_to_drain"])
def kill_marathon_app(full_appid, cluster, client, soa_dir):
service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid))
service_instance_config = marathon_tools.load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
complete_config = service_instance_config.format_marathon_app_dict()
nerve_ns = service_instance_config.get_nerve_namespace()
service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns)
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func('down')
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
(
old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps=[app_to_kill],
drain_method=drain_method,
service=service,
nerve_ns=nerve_ns,
bounce_health_params=service_instance_config.get_bounce_health_params(service_namespace_config),
)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running='',
happy_new_tasks=[],
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
serviceinstance="{}.{}".format(service, instance),
bounce_method='down',
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
paasta_print("Sleeping for 10 seconds to give the tasks time to drain")
time.sleep(10)
paasta_print("Sucessfully killed {}".format(full_appid))
def kill_marathon_app(full_appid, cluster, client, soa_dir):
service, instance, _, __ = (s.replace("--", "_")
for s in decompose_job_id(full_appid))
service_instance_config = marathon_tools.load_marathon_service_config(
service=service, instance=instance, cluster=cluster, soa_dir=soa_dir)
complete_config = service_instance_config.format_marathon_app_dict()
registrations = service_instance_config.get_registrations()
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=registrations[0])
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
registrations=registrations,
drain_method_params=service_instance_config.get_drain_method_params(
service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func("down")
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
(
old_app_live_happy_tasks,
old_app_live_unhappy_tasks,
old_app_draining_tasks,
old_app_at_risk_tasks,
) = get_tasks_by_state(
other_apps=[app_to_kill],
drain_method=drain_method,
service=service,
nerve_ns=registrations[0],
bounce_health_params=service_instance_config.
get_bounce_health_params(service_namespace_config),
)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running="",
happy_new_tasks=[],
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
old_app_draining_tasks=old_app_draining_tasks,
old_app_at_risk_tasks=old_app_at_risk_tasks,
serviceinstance=f"{service}.{instance}",
bounce_method="down",
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
paasta_print("Sleeping for 10 seconds to give the tasks time to drain")
time.sleep(10)
paasta_print(f"Successfully killed {full_appid}")
def test_do_bounce_when_tasks_to_drain(self):
fake_task_to_drain = mock.Mock(app_id="fake_app_to_kill_1")
fake_bounce_func_return = {"create_app": False, "tasks_to_drain": [fake_task_to_drain]}
fake_bounce_func = mock.create_autospec(bounce_lib.brutal_bounce, return_value=fake_bounce_func_return)
fake_config = {"instances": 5}
fake_new_app_running = True
fake_happy_new_tasks = ["fake_one", "fake_two", "fake_three"]
fake_old_app_live_tasks = {"fake_app_to_kill_1": set([fake_task_to_drain])}
fake_old_app_draining_tasks = {"fake_app_to_kill_1": set([])}
fake_service = "fake_service"
fake_serviceinstance = "fake_service.fake_instance"
self.fake_cluster = "fake_cluster"
fake_instance = "fake_instance"
fake_bounce_method = "fake_bounce_method"
fake_drain_method = mock.Mock(is_safe_to_kill=lambda t: False)
fake_marathon_jobid = "fake.marathon.jobid"
fake_client = mock.create_autospec(marathon.MarathonClient)
expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks)
expected_drain_task_count = len(fake_bounce_func_return["tasks_to_drain"])
with contextlib.nested(
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids", autospec=True),
) as (mock_log, mock_create_marathon_app, mock_kill_old_ids):
setup_marathon_job.do_bounce(
bounce_func=fake_bounce_func,
drain_method=fake_drain_method,
config=fake_config,
new_app_running=fake_new_app_running,
happy_new_tasks=fake_happy_new_tasks,
old_app_live_tasks=fake_old_app_live_tasks,
old_app_draining_tasks=fake_old_app_draining_tasks,
service=fake_service,
bounce_method=fake_bounce_method,
serviceinstance=fake_serviceinstance,
cluster=self.fake_cluster,
instance=fake_instance,
marathon_jobid=fake_marathon_jobid,
client=fake_client,
soa_dir="fake_soa_dir",
)
# assert mock_log.call_count == 3
first_logged_line = mock_log.mock_calls[0][2]["line"]
assert "%s new tasks" % expected_new_task_count in first_logged_line
second_logged_line = mock_log.mock_calls[1][2]["line"]
assert (
"draining %s old tasks with app_id %s" % (expected_drain_task_count, "fake_app_to_kill_1")
in second_logged_line
)
assert mock_create_marathon_app.call_count == 0
assert fake_client.kill_task.call_count == 0
assert mock_kill_old_ids.call_count == 0
assert fake_drain_method.drain.call_count == expected_drain_task_count
def test_do_bounce_when_apps_to_kill(self):
fake_bounce_func_return = {"create_app": False, "tasks_to_drain": []}
fake_bounce_func = mock.create_autospec(bounce_lib.brutal_bounce, return_value=fake_bounce_func_return)
fake_config = {"instances": 5}
fake_new_app_running = True
fake_happy_new_tasks = ["fake_one", "fake_two", "fake_three"]
fake_old_app_live_tasks = {"fake_app_to_kill_1": set()}
fake_old_app_draining_tasks = {"fake_app_to_kill_1": set()}
fake_service = "fake_service"
fake_serviceinstance = "fake_service.fake_instance"
self.fake_cluster = "fake_cluster"
fake_instance = "fake_instance"
fake_bounce_method = "fake_bounce_method"
fake_drain_method = mock.Mock()
fake_marathon_jobid = "fake.marathon.jobid"
fake_client = mock.create_autospec(marathon.MarathonClient)
expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks)
with contextlib.nested(
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids", autospec=True),
) as (mock_log, mock_create_marathon_app, mock_kill_old_ids):
setup_marathon_job.do_bounce(
bounce_func=fake_bounce_func,
drain_method=fake_drain_method,
config=fake_config,
new_app_running=fake_new_app_running,
happy_new_tasks=fake_happy_new_tasks,
old_app_live_tasks=fake_old_app_live_tasks,
old_app_draining_tasks=fake_old_app_draining_tasks,
service=fake_service,
bounce_method=fake_bounce_method,
serviceinstance=fake_serviceinstance,
cluster=self.fake_cluster,
instance=fake_instance,
marathon_jobid=fake_marathon_jobid,
client=fake_client,
soa_dir="fake_soa_dir",
)
assert mock_log.call_count == 3
first_logged_line = mock_log.mock_calls[0][2]["line"]
assert "%s new tasks" % expected_new_task_count in first_logged_line
second_logged_line = mock_log.mock_calls[1][2]["line"]
assert "removing old unused apps with app_ids: %s" % "fake_app_to_kill_1" in second_logged_line
assert mock_create_marathon_app.call_count == 0
assert fake_client.kill_task.call_count == len(fake_bounce_func_return["tasks_to_drain"])
assert mock_kill_old_ids.call_count == 1
third_logged_line = mock_log.mock_calls[2][2]["line"]
assert "%s bounce on %s finish" % (fake_bounce_method, fake_serviceinstance) in third_logged_line
assert "Now running %s" % fake_marathon_jobid in third_logged_line
def test_do_bounce_when_nothing_to_do(self):
fake_bounce_func_return = {"create_app": False, "tasks_to_drain": []}
fake_bounce_func = mock.create_autospec(bounce_lib.brutal_bounce, return_value=fake_bounce_func_return)
fake_config = {"instances": 3}
fake_new_app_running = True
fake_happy_new_tasks = ["fake_one", "fake_two", "fake_three"]
fake_old_app_live_tasks = {}
fake_old_app_draining_tasks = {}
fake_service = "fake_service"
fake_serviceinstance = "fake_service.fake_instance"
self.fake_cluster = "fake_cluster"
fake_instance = "fake_instance"
fake_bounce_method = "fake_bounce_method"
fake_drain_method = mock.Mock()
fake_marathon_jobid = "fake.marathon.jobid"
fake_client = mock.create_autospec(marathon.MarathonClient)
with contextlib.nested(
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.send_sensu_bounce_keepalive", autospec=True),
) as (mock_log, mock_create_marathon_app, mock_kill_old_ids, mock_send_sensu_bounce_keepalive):
setup_marathon_job.do_bounce(
bounce_func=fake_bounce_func,
drain_method=fake_drain_method,
config=fake_config,
new_app_running=fake_new_app_running,
happy_new_tasks=fake_happy_new_tasks,
old_app_live_tasks=fake_old_app_live_tasks,
old_app_draining_tasks=fake_old_app_draining_tasks,
service=fake_service,
bounce_method=fake_bounce_method,
serviceinstance=fake_serviceinstance,
cluster=self.fake_cluster,
instance=fake_instance,
marathon_jobid=fake_marathon_jobid,
client=fake_client,
soa_dir="fake_soa_dir",
)
assert mock_log.call_count == 0
assert mock_create_marathon_app.call_count == 0
assert fake_drain_method.drain.call_count == 0
assert mock_kill_old_ids.call_count == 0
# When doing nothing, we need to send the keepalive heartbeat to Sensu
mock_send_sensu_bounce_keepalive.assert_called_once_with(
service=fake_service, instance=fake_instance, cluster=self.fake_cluster, soa_dir="fake_soa_dir"
)
def test_do_bounce_when_nothing_to_do(self):
fake_bounce_func_return = {
'create_app': False,
'tasks_to_drain': [],
}
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce,
return_value=fake_bounce_func_return,
)
fake_config = {'instances': 3}
fake_new_app_running = True
fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three']
fake_old_app_live_tasks = {}
fake_old_app_draining_tasks = {}
fake_service = 'fake_service'
fake_serviceinstance = 'fake_service.fake_instance'
self.fake_cluster = 'fake_cluster'
fake_instance = 'fake_instance'
fake_bounce_method = 'fake_bounce_method'
fake_drain_method = mock.Mock()
fake_marathon_jobid = 'fake.marathon.jobid'
fake_client = mock.create_autospec(
marathon.MarathonClient
)
with contextlib.nested(
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.send_sensu_bounce_keepalive', autospec=True),
) as (
mock_log,
mock_create_marathon_app,
mock_kill_old_ids,
mock_send_sensu_bounce_keepalive,
):
setup_marathon_job.do_bounce(
bounce_func=fake_bounce_func,
drain_method=fake_drain_method,
config=fake_config,
new_app_running=fake_new_app_running,
happy_new_tasks=fake_happy_new_tasks,
old_app_live_tasks=fake_old_app_live_tasks,
old_app_draining_tasks=fake_old_app_draining_tasks,
service=fake_service,
bounce_method=fake_bounce_method,
serviceinstance=fake_serviceinstance,
cluster=self.fake_cluster,
instance=fake_instance,
marathon_jobid=fake_marathon_jobid,
client=fake_client,
soa_dir='fake_soa_dir',
)
assert mock_log.call_count == 0
assert mock_create_marathon_app.call_count == 0
assert fake_drain_method.drain.call_count == 0
assert mock_kill_old_ids.call_count == 0
# When doing nothing, we need to send the keepalive heartbeat to Sensu
mock_send_sensu_bounce_keepalive.assert_called_once_with(
service=fake_service,
instance=fake_instance,
cluster=self.fake_cluster,
soa_dir='fake_soa_dir',
)
def test_do_bounce_when_apps_to_kill(self):
fake_bounce_func_return = {
'create_app': False,
'tasks_to_drain': [],
}
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce,
return_value=fake_bounce_func_return,
)
fake_config = {'instances': 5}
fake_new_app_running = True
fake_happy_new_tasks = ['fake_one', 'fake_two', 'fake_three']
fake_old_app_live_tasks = {'fake_app_to_kill_1': set()}
fake_old_app_draining_tasks = {'fake_app_to_kill_1': set()}
fake_service = 'fake_service'
fake_serviceinstance = 'fake_service.fake_instance'
self.fake_cluster = 'fake_cluster'
fake_instance = 'fake_instance'
fake_bounce_method = 'fake_bounce_method'
fake_drain_method = mock.Mock()
fake_marathon_jobid = 'fake.marathon.jobid'
fake_client = mock.create_autospec(
marathon.MarathonClient
)
expected_new_task_count = fake_config["instances"] - len(fake_happy_new_tasks)
with contextlib.nested(
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.create_marathon_app', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.kill_old_ids', autospec=True),
) as (mock_log, mock_create_marathon_app, mock_kill_old_ids):
setup_marathon_job.do_bounce(
bounce_func=fake_bounce_func,
drain_method=fake_drain_method,
config=fake_config,
new_app_running=fake_new_app_running,
happy_new_tasks=fake_happy_new_tasks,
old_app_live_tasks=fake_old_app_live_tasks,
old_app_draining_tasks=fake_old_app_draining_tasks,
service=fake_service,
bounce_method=fake_bounce_method,
serviceinstance=fake_serviceinstance,
cluster=self.fake_cluster,
instance=fake_instance,
marathon_jobid=fake_marathon_jobid,
client=fake_client,
soa_dir='fake_soa_dir',
)
assert mock_log.call_count == 3
first_logged_line = mock_log.mock_calls[0][2]["line"]
assert '%s new tasks' % expected_new_task_count in first_logged_line
second_logged_line = mock_log.mock_calls[1][2]["line"]
assert 'removing old unused apps with app_ids: %s' % 'fake_app_to_kill_1' in second_logged_line
assert mock_create_marathon_app.call_count == 0
assert fake_client.kill_task.call_count == len(fake_bounce_func_return["tasks_to_drain"])
assert mock_kill_old_ids.call_count == 1
third_logged_line = mock_log.mock_calls[2][2]["line"]
assert '%s bounce on %s finish' % (fake_bounce_method, fake_serviceinstance) in third_logged_line
assert 'Now running %s' % fake_marathon_jobid in third_logged_line
def main():
args = parse_args()
full_appid = args.appname.lstrip('/')
soa_dir = args.soa_dir
marathon_config = marathon_tools.load_marathon_config()
client = marathon_tools.get_marathon_client(
url=marathon_config.get_url(),
user=marathon_config.get_username(),
passwd=marathon_config.get_password(),
)
if not marathon_tools.is_app_id_running(app_id=full_appid, client=client):
print("Couldn't find an app named {0}".format(full_appid))
sys.exit(1)
service, instance, _, __ = (s.replace('--', '_')
for s in decompose_job_id(full_appid))
complete_config = marathon_tools.create_complete_config(
service, instance, marathon_config)
cluster = load_system_paasta_config().get_cluster()
service_instance_config = marathon_tools.load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
nerve_ns = service_instance_config.get_nerve_namespace()
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=nerve_ns)
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=service_instance_config.get_drain_method_params(
service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func('down')
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks(
[app_to_kill], drain_method)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running='',
happy_new_tasks=[],
old_app_live_tasks=old_app_live_tasks,
old_app_draining_tasks=old_app_draining_tasks,
serviceinstance="{0}.{1}".format(service, instance),
bounce_method='down',
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
print "Sleeping for 10 seconds to give the tasks time to drain"
time.sleep(10)
print("Sucessfully killed {0}".format(full_appid))
def main():
args = parse_args()
full_appid = args.appname.lstrip('/')
soa_dir = args.soa_dir
marathon_config = marathon_tools.load_marathon_config()
client = marathon_tools.get_marathon_client(
url=marathon_config.get_url(),
user=marathon_config.get_username(),
passwd=marathon_config.get_password(),
)
if not marathon_tools.is_app_id_running(app_id=full_appid, client=client):
print("Couldn't find an app named {0}".format(full_appid))
sys.exit(1)
service, instance, _, __ = (s.replace('--', '_') for s in decompose_job_id(full_appid))
complete_config = marathon_tools.create_complete_config(service, instance, marathon_config)
cluster = load_system_paasta_config().get_cluster()
service_instance_config = marathon_tools.load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
nerve_ns = service_instance_config.get_nerve_namespace()
service_namespace_config = marathon_tools.load_service_namespace_config(service=service, namespace=nerve_ns)
drain_method = drain_lib.get_drain_method(
service_instance_config.get_drain_method(service_namespace_config),
service=service,
instance=instance,
nerve_ns=nerve_ns,
drain_method_params=service_instance_config.get_drain_method_params(service_namespace_config),
)
bounce_func = bounce_lib.get_bounce_method_func('down')
while marathon_tools.is_app_id_running(app_id=full_appid, client=client):
app_to_kill = client.get_app(full_appid)
old_app_live_tasks, old_app_draining_tasks = get_old_live_draining_tasks([app_to_kill], drain_method)
do_bounce(
bounce_func=bounce_func,
drain_method=drain_method,
config=complete_config,
new_app_running='',
happy_new_tasks=[],
old_app_live_tasks=old_app_live_tasks,
old_app_draining_tasks=old_app_draining_tasks,
serviceinstance="{0}.{1}".format(service, instance),
bounce_method='down',
service=service,
cluster=cluster,
instance=instance,
marathon_jobid=full_appid,
client=client,
soa_dir=soa_dir,
)
print "Sleeping for 10 seconds to give the tasks time to drain"
time.sleep(10)
print("Sucessfully killed {0}".format(full_appid))
