def test_crossover_bounce_lots_of_unhappy_old_some_happy_old_new_app_exists_no_new_tasks(self):
"""When marathon has a new app and multiple old apps, no new tasks are up, one of the old apps is healthy and
the other is not, only unhealthy tasks should get killed.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(5)),
}
old_app_live_unhappy_tasks = {
'app2': set(mock.Mock() for _ in xrange(5)),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert actual['tasks_to_drain'] == old_app_live_unhappy_tasks['app2']
# Since there are plenty of unhappy old tasks, we should not kill any new ones.
assert len(actual['tasks_to_drain'] & old_app_live_happy_tasks['app1']) == 0
def test_crossover_bounce_some_unhappy_old_no_happy_old_no_new_tasks_no_excess(
self):
"""When marathon only has old apps for this service, and all of their tasks are unhappy, and there are no excess
tasks, the crossover bounce should start a new app and not kill any old tasks.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {}
old_app_live_unhappy_tasks = {
'app1': {mock.Mock()
for _ in range(2)},
'app2': {mock.Mock()
for _ in range(3)},
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(),
}
def test_crossover_bounce_cleanup(self):
"""When marathon has the desired app, and there are other copies of
the service running, which have no remaining tasks, those apps should
be killed."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_happy_tasks = {
'app1': set(),
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(),
}
def test_crossover_bounce_lots_of_unhappy_old_no_happy_old_no_new(self):
"""When marathon has a new app and multiple old apps, no new tasks are up, all old tasks are unhappy, and there
are too many tasks running, the crossover bounce should kill some (but not all) of the old
tasks.
This represents a situation where
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {}
old_app_live_unhappy_tasks = {
'app1': {mock.Mock() for _ in range(5)},
'app2': {mock.Mock() for _ in range(5)},
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 5
def test_crossover_bounce_cleanup(self):
"""When marathon has the desired app, and there are other copies of
the service running, which have no remaining tasks, those apps should
be killed."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in xrange(5)]
old_app_live_happy_tasks = {
'app1': set(),
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(),
}
def test_crossover_bounce_old_app_is_happy_but_no_new_app_happy_tasks(self):
"""When marathon only has old apps for this service and margin_factor != 1,
crossover bounce should start the new app and kill some old tasks."""
new_config = {'id': 'foo.bar.12345', 'instances': 100}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(60)},
'app2': {mock.Mock() for _ in range(40)},
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
margin_factor=0.95,
)
assert actual["create_app"] is True
assert len(actual["tasks_to_drain"]) == 5
def test_crossover_bounce_mid_bounce_some_happy_old_some_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, and some of those
older tasks are unhappy, we should prefer killing the unhappy tasks."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in xrange(3)]
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(3)),
'app2': set(mock.Mock() for _ in xrange(2)),
}
old_app_live_unhappy_tasks = {
'app1': set(mock.Mock() for _ in xrange(1)),
'app2': set(),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 4
# There are fewer unhappy old tasks than excess tasks, so we should kill all unhappy old ones, plus a few
# happy ones.
assert old_app_live_unhappy_tasks['app1'].issubset(actual['tasks_to_drain'])
def test_crossover_bounce_lots_of_unhappy_old_no_happy_old_no_new(self):
"""When marathon has a new app and multiple old apps, no new tasks are up, all old tasks are unhappy, and there
are too many tasks running, the crossover bounce should kill some (but not all) of the old
tasks.
This represents a situation where
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {}
old_app_live_unhappy_tasks = {
'app1': set(mock.Mock() for _ in xrange(5)),
'app2': set(mock.Mock() for _ in xrange(5)),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 5
def test_crossover_bounce_mid_bounce_no_happy_old_lots_of_unhappy_old(
self):
"""When marathon has the desired app, and there are other copies of the service running, but none of the old
tasks are happy, and there are excess tasks, we should kill some (but not all) unhappy old tasks."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = {
'app1': set(),
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': {mock.Mock()
for _ in range(3)},
'app2': {mock.Mock()
for _ in range(3)},
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 4
def test_crossover_bounce_some_unhappy_old_some_happy_old_no_new(self):
"""When marathon only has old apps for this service, and some of them are unhappy (maybe they've been recently
started), the crossover bounce should start a new app and prefer killing the unhappy tasks over the happy ones.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(3)),
'app2': set(mock.Mock() for _ in xrange(2)),
}
old_app_live_unhappy_tasks = {
'app1': set(mock.Mock() for _ in xrange(2)),
'app2': set(mock.Mock() for _ in xrange(3)),
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(old_app_live_unhappy_tasks['app1'] | old_app_live_unhappy_tasks['app2']),
}
def test_crossover_bounce_old_but_no_new(self):
"""When marathon only has old apps for this service, crossover bounce should start the new one, but not kill any
old tasks yet."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(3)},
'app2': {mock.Mock() for _ in range(2)},
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": True,
"tasks_to_drain": set(),
}
def test_crossover_bounce_old_but_no_new(self):
"""When marathon only has old apps for this service, crossover bounce should start the new one, but not kill any
old tasks yet."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(3)),
'app2': set(mock.Mock() for _ in xrange(2)),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": True,
"tasks_to_drain": set(),
}
def test_crossover_bounce_old_app_is_happy_but_no_new_app_happy_tasks(self):
"""When marathon only has old apps for this service and margin_factor != 1,
crossover bounce should start the new app and kill some old tasks."""
new_config = {'id': 'foo.bar.12345', 'instances': 100}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(60)),
'app2': set(mock.Mock() for _ in xrange(40)),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
margin_factor=0.95,
)
assert actual["create_app"] is True
assert len(actual["tasks_to_drain"]) == 5
def test_crossover_bounce_mid_bounce_some_happy_old_lots_of_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, and there are more
unhappy old tasks than excess tasks, we should only kill unhappy tasks.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in xrange(3)]
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(2)),
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(mock.Mock() for _ in xrange(5)),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
# There are as many unhappy old tasks as excess tasks, so all tasks that we kill should be old unhappy ones.
assert len(actual['tasks_to_drain']) == 5
assert actual['tasks_to_drain'] == old_app_live_unhappy_tasks['app2']
def kill_tasks_if_necessary(self, driver):
base_task = self.service_config.base_task(self.system_paasta_config)
new_tasks = self.get_new_tasks(base_task.name, self.tasks_with_flags.keys())
happy_new_tasks = self.get_happy_tasks(new_tasks)
desired_instances = self.service_config.get_instances()
# this puts the most-desired tasks first. I would have left them in order of bad->good and used
# new_tasks_by_desirability[:-desired_instances] instead, but list[:-0] is an empty list, rather than the full
# list.
new_tasks_by_desirability = sorted(list(new_tasks), key=self.make_healthiness_sorter(base_task.name),
reverse=True)
new_tasks_to_kill = new_tasks_by_desirability[desired_instances:]
old_tasks = self.get_old_tasks(base_task.name, self.tasks_with_flags.keys())
old_happy_tasks = self.get_happy_tasks(old_tasks)
old_draining_tasks = self.get_draining_tasks(old_tasks)
old_unhappy_tasks = set(old_tasks) - set(old_happy_tasks) - set(old_draining_tasks)
actions = bounce_lib.crossover_bounce(
new_config={"instances": desired_instances},
new_app_running=True,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=self.group_tasks_by_version(old_happy_tasks + new_tasks_to_kill),
old_app_live_unhappy_tasks=self.group_tasks_by_version(old_unhappy_tasks),
)
for task in set(new_tasks) - set(actions['tasks_to_drain']):
self.undrain_task(task)
for task in actions['tasks_to_drain']:
self.drain_task(task)
for task in [task for task, parameters in self.tasks_with_flags.iteritems() if parameters.is_draining]:
if self.drain_method.is_safe_to_kill(DrainTask(id=task)):
self.kill_task(driver, task)
def test_crossover_bounce_some_unhappy_old_some_happy_old_no_new(self):
"""When marathon only has old apps for this service, and some of them are unhappy (maybe they've been recently
started), the crossover bounce should start a new app and prefer killing the unhappy tasks over the happy ones.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(3)},
'app2': {mock.Mock() for _ in range(2)},
}
old_app_live_unhappy_tasks = {
'app1': {mock.Mock() for _ in range(2)},
'app2': {mock.Mock() for _ in range(3)},
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(old_app_live_unhappy_tasks['app1'] | old_app_live_unhappy_tasks['app2']),
}
def test_crossover_bounce_lots_of_unhappy_old_some_happy_old_new_app_exists_no_new_tasks(self):
"""When marathon has a new app and multiple old apps, no new tasks are up, one of the old apps is healthy and
the other is not, only unhealthy tasks should get killed.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(5)},
}
old_app_live_unhappy_tasks = {
'app2': {mock.Mock() for _ in range(5)},
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert actual['tasks_to_drain'] == old_app_live_unhappy_tasks['app2']
# Since there are plenty of unhappy old tasks, we should not kill any new ones.
assert len(actual['tasks_to_drain'] & old_app_live_happy_tasks['app1']) == 0
def test_crossover_bounce_mid_bounce_some_happy_old_some_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, and some of those
older tasks are unhappy, we should prefer killing the unhappy tasks."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(3)},
'app2': {mock.Mock() for _ in range(2)},
}
old_app_live_unhappy_tasks = {
'app1': {mock.Mock() for _ in range(1)},
'app2': set(),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 4
# There are fewer unhappy old tasks than excess tasks, so we should kill all unhappy old ones, plus a few
# happy ones.
assert old_app_live_unhappy_tasks['app1'].issubset(actual['tasks_to_drain'])
def test_crossover_bounce_mid_bounce_some_happy_old_lots_of_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, and there are more
unhappy old tasks than excess tasks, we should only kill unhappy tasks.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(2)},
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': {mock.Mock() for _ in range(5)},
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
# There are as many unhappy old tasks as excess tasks, so all tasks that we kill should be old unhappy ones.
assert len(actual['tasks_to_drain']) == 5
assert actual['tasks_to_drain'] == old_app_live_unhappy_tasks['app2']
def kill_tasks_if_necessary(self, driver: MesosSchedulerDriver):
base_task = self.service_config.base_task(self.system_paasta_config)
all_tasks_with_params = self.task_store.get_all_tasks()
new_tasks_with_params = self.get_new_tasks(base_task['name'], all_tasks_with_params)
happy_new_tasks_with_params = self.get_happy_tasks(new_tasks_with_params)
desired_instances = self.service_config.get_desired_instances()
# this puts the most-desired tasks first. I would have left them in order of bad->good and used
# new_tasks_by_desirability[:-desired_instances] instead, but list[:-0] is an empty list, rather than the full
# list.
new_task_ids_by_desirability = sorted(
list(new_tasks_with_params.keys()),
key=self.make_healthiness_sorter(base_task['name'], all_tasks_with_params),
reverse=True,
)
new_task_ids_to_kill = new_task_ids_by_desirability[desired_instances:]
old_tasks_with_params = self.get_old_tasks(base_task['name'], all_tasks_with_params)
old_draining_tasks_with_params = self.get_draining_tasks(old_tasks_with_params)
old_non_draining_tasks = sorted(
list(
set(old_tasks_with_params.keys()) -
set(old_draining_tasks_with_params),
),
key=self.make_healthiness_sorter(base_task['name'], all_tasks_with_params),
reverse=True,
)
actions = bounce_lib.crossover_bounce(
new_config={"instances": desired_instances},
new_app_running=True,
happy_new_tasks=happy_new_tasks_with_params.keys(),
old_non_draining_tasks=new_task_ids_to_kill + old_non_draining_tasks,
)
with a_sync.idle_event_loop():
futures = []
for task in set(new_tasks_with_params.keys()) - set(actions['tasks_to_drain']):
futures.append(asyncio.ensure_future(self.undrain_task(task)))
for task in actions['tasks_to_drain']:
futures.append(asyncio.ensure_future(self.drain_task(task)))
if futures:
a_sync.block(asyncio.wait, futures)
async def kill_if_safe_to_kill(task_id: str):
if await self.drain_method.is_safe_to_kill(self.make_drain_task(task_id)):
self.kill_task(driver, task_id)
futures = []
for task, parameters in all_tasks_with_params.items():
if parameters.is_draining and parameters.mesos_task_state in LIVE_TASK_STATES:
futures.append(asyncio.ensure_future(kill_if_safe_to_kill(task)))
if futures:
a_sync.block(asyncio.wait, futures)
def kill_tasks_if_necessary(self, driver: MesosSchedulerDriver):
base_task = self.service_config.base_task(self.system_paasta_config)
all_tasks_with_params = self.task_store.get_all_tasks()
new_tasks_with_params = self.get_new_tasks(base_task['name'],
all_tasks_with_params)
happy_new_tasks_with_params = self.get_happy_tasks(
new_tasks_with_params)
desired_instances = self.service_config.get_desired_instances()
# this puts the most-desired tasks first. I would have left them in order of bad->good and used
# new_tasks_by_desirability[:-desired_instances] instead, but list[:-0] is an empty list, rather than the full
# list.
new_task_ids_by_desirability = sorted(
list(new_tasks_with_params.keys()),
key=self.make_healthiness_sorter(base_task['name'],
all_tasks_with_params),
reverse=True,
)
new_task_ids_to_kill = new_task_ids_by_desirability[desired_instances:]
old_tasks_with_params = self.get_old_tasks(base_task['name'],
all_tasks_with_params)
old_happy_tasks_with_params = self.get_happy_tasks(
old_tasks_with_params)
old_draining_tasks_with_params = self.get_draining_tasks(
old_tasks_with_params)
old_unhappy_task_ids = list(
set(old_tasks_with_params.keys()) -
set(old_happy_tasks_with_params.keys()) -
set(old_draining_tasks_with_params), )
actions = bounce_lib.crossover_bounce(
new_config={"instances": desired_instances},
new_app_running=True,
happy_new_tasks=happy_new_tasks_with_params.keys(),
old_app_live_happy_tasks=self.group_tasks_by_version(
list(old_happy_tasks_with_params.keys()) +
new_task_ids_to_kill, ),
old_app_live_unhappy_tasks=self.group_tasks_by_version(
old_unhappy_task_ids),
)
for task in set(new_tasks_with_params.keys()) - set(
actions['tasks_to_drain']):
self.undrain_task(task)
for task in actions['tasks_to_drain']:
self.drain_task(task)
for task, parameters in all_tasks_with_params.items():
if parameters.is_draining and \
self.drain_method.is_safe_to_kill(self.make_drain_task(task)) and \
parameters.mesos_task_state in LIVE_TASK_STATES:
self.kill_task(driver, task)
def test_crossover_bounce_no_existing_apps(self):
"""When marathon is unaware of a service, crossover bounce should try to
create a marathon app."""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = []
old_app_live_happy_tasks = []
old_app_live_unhappy_tasks = []
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
) == {"create_app": True, "tasks_to_drain": set()}
def test_crossover_bounce_old_but_no_new(self):
"""When marathon only has old apps for this service, crossover bounce should start the new one, but not kill any
old tasks yet."""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = []
old_app_live_happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_unhappy_tasks = []
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
) == {"create_app": True, "tasks_to_drain": set()}
def test_crossover_bounce_no_existing_apps(self):
"""When marathon is unaware of a service, crossover bounce should try to
create a marathon app."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_tasks = {}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_app_live_tasks=old_app_live_tasks,
) == {
"create_app": True,
"tasks_to_drain": set(),
}
def test_crossover_bounce_using_margin_factor_big_numbers(self):
new_config = {"id": "foo.bar.12345", "instances": 500}
happy_tasks = [mock.Mock() for _ in range(100)]
old_app_live_happy_tasks = [mock.Mock() for _ in range(300)]
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(100)]
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks +
old_app_live_unhappy_tasks,
margin_factor=0.95,
)
assert actual["create_app"] is False
assert len(actual["tasks_to_drain"]) == 25
def test_crossover_bounce_using_margin_factor_small_numbers(self):
new_config = {'id': 'foo.bar.12345', 'instances': 3}
happy_tasks = []
old_app_live_happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_unhappy_tasks = []
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks +
old_app_live_unhappy_tasks,
margin_factor=0.66,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 1
def test_crossover_bounce_done(self):
"""When marathon has the desired app, and there are no other copies of
the service running, crossover bounce should neither start nor stop
anything."""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_happy_tasks = []
old_app_live_unhappy_tasks = []
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
) == {"create_app": False, "tasks_to_drain": set()}
def test_crossover_bounce_mid_bounce_no_happy_old_lots_of_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, but none of the old
tasks are happy, and there are excess tasks, we should kill some (but not all) unhappy old tasks."""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = []
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(6)]
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
)
assert actual["create_app"] is False
assert len(actual["tasks_to_drain"]) == 4
def test_crossover_bounce_some_unhappy_old_some_happy_old_no_new(self):
"""When marathon only has old apps for this service, and some of them are unhappy (maybe they've been recently
started), the crossover bounce should start a new app and prefer killing the unhappy tasks over the happy ones.
"""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = []
old_app_live_happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(5)]
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
) == {"create_app": False, "tasks_to_drain": set(old_app_live_unhappy_tasks)}
def kill_tasks_if_necessary(self, driver):
base_task = self.service_config.base_task(self.system_paasta_config)
new_tasks = self.get_new_tasks(base_task.name,
self.tasks_with_flags.keys())
happy_new_tasks = self.get_happy_tasks(new_tasks)
desired_instances = self.service_config.get_desired_instances()
# this puts the most-desired tasks first. I would have left them in order of bad->good and used
# new_tasks_by_desirability[:-desired_instances] instead, but list[:-0] is an empty list, rather than the full
# list.
new_tasks_by_desirability = sorted(list(new_tasks),
key=self.make_healthiness_sorter(
base_task.name),
reverse=True)
new_tasks_to_kill = new_tasks_by_desirability[desired_instances:]
old_tasks = self.get_old_tasks(base_task.name,
self.tasks_with_flags.keys())
old_happy_tasks = self.get_happy_tasks(old_tasks)
old_draining_tasks = self.get_draining_tasks(old_tasks)
old_unhappy_tasks = set(old_tasks) - set(old_happy_tasks) - set(
old_draining_tasks)
actions = bounce_lib.crossover_bounce(
new_config={"instances": desired_instances},
new_app_running=True,
happy_new_tasks=happy_new_tasks,
old_app_live_happy_tasks=self.group_tasks_by_version(
old_happy_tasks + new_tasks_to_kill),
old_app_live_unhappy_tasks=self.group_tasks_by_version(
old_unhappy_tasks),
)
for task in set(new_tasks) - set(actions['tasks_to_drain']):
self.undrain_task(task)
for task in actions['tasks_to_drain']:
self.drain_task(task)
for task in [
task for task, parameters in self.tasks_with_flags.iteritems()
if parameters.is_draining
]:
if self.drain_method.is_safe_to_kill(DrainTask(id=task)):
self.kill_task(driver, task)
def test_crossover_bounce_done(self):
"""When marathon has the desired app, and there are no other copies of
the service running, crossover bounce should neither start nor stop
anything."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in xrange(5)]
old_app_live_tasks = {}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_tasks=old_app_live_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(),
}
def test_crossover_bounce_old_app_is_happy_but_no_new_app_happy_tasks(self):
"""When marathon only has old apps for this service and margin_factor != 1,
crossover bounce should start the new app and kill some old tasks."""
new_config = {"id": "foo.bar.12345", "instances": 100}
happy_tasks = []
old_app_live_happy_tasks = [mock.Mock() for _ in range(100)]
old_app_live_unhappy_tasks = []
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=False,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
margin_factor=0.95,
)
assert actual["create_app"] is True
assert len(actual["tasks_to_drain"]) == 5
def test_crossover_bounce_mid_bounce(self):
"""When marathon has the desired app, and there are other copies of the service running, but the new app is not
fully up, crossover bounce should only stop a few of the old instances."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_unhappy_tasks = []
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks +
old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 3
def test_crossover_bounce_mid_bounce(self):
"""When marathon has the desired app, and there are other copies of the service running, but the new app is not
fully up, crossover bounce should only stop a few of the old instances."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in xrange(3)]
old_app_live_tasks = {
'app1': set(mock.Mock() for _ in xrange(3)),
'app2': set(mock.Mock() for _ in xrange(2)),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_tasks=old_app_live_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 3
def test_crossover_bounce_some_unhappy_old_no_happy_old_no_new_tasks_no_excess(
self, ):
"""When marathon only has old apps for this service, and all of their tasks are unhappy, and there are no excess
tasks, the crossover bounce should start a new app and not kill any old tasks.
"""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = []
old_app_live_happy_tasks = []
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(5)]
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks +
old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set()
}
def test_crossover_bounce_using_margin_factor_big_numbers(self):
new_config = {'id': 'foo.bar.12345', 'instances': 500}
happy_tasks = [mock.Mock() for _ in range(100)]
old_app_live_happy_tasks = {
'app1': {mock.Mock() for _ in range(300)},
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': {mock.Mock() for _ in range(100)},
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
margin_factor=0.95,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 25
def test_crossover_bounce_mid_bounce_some_happy_old_some_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, and some of those
older tasks are unhappy, we should prefer killing the unhappy tasks."""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(1)]
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
)
assert actual["create_app"] is False
assert len(actual["tasks_to_drain"]) == 4
# There are fewer unhappy old tasks than excess tasks, so we should kill all unhappy old ones, plus a few
# happy ones.
assert set(old_app_live_unhappy_tasks).issubset(actual["tasks_to_drain"])
def test_crossover_bounce_mid_bounce_some_happy_old_lots_of_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, and there are more
unhappy old tasks than excess tasks, we should only kill unhappy tasks.
"""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = [mock.Mock() for _ in range(3)]
old_app_live_happy_tasks = [mock.Mock() for _ in range(2)]
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(5)]
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks
+ old_app_live_unhappy_tasks,
)
assert actual["create_app"] is False
# There are as many unhappy old tasks as excess tasks, so all tasks that we kill should be old unhappy ones.
assert len(actual["tasks_to_drain"]) == 5
assert actual["tasks_to_drain"] == set(old_app_live_unhappy_tasks)
def test_crossover_bounce_lots_of_unhappy_old_no_happy_old_no_new(self):
"""When marathon has a new app and multiple old apps, no new tasks are up, all old tasks are unhappy, and there
are too many tasks running, the crossover bounce should kill some (but not all) of the old
tasks.
This represents a situation where
"""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = []
old_app_live_happy_tasks = []
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(10)]
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks +
old_app_live_unhappy_tasks,
)
assert actual["create_app"] is False
assert len(actual["tasks_to_drain"]) == 5
def test_crossover_bounce_using_margin_factor_small_numbers(self):
new_config = {'id': 'foo.bar.12345', 'instances': 3}
happy_tasks = []
old_app_live_happy_tasks = {
'app1': set(mock.Mock() for _ in xrange(3)),
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(),
'app2': set(),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
margin_factor=0.66,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 1
def test_crossover_bounce_lots_of_unhappy_old_some_happy_old_new_app_exists_no_new_tasks(
self, ):
"""When marathon has a new app and multiple old apps, no new tasks are up, one of the old apps is healthy and
the other is not, only unhealthy tasks should get killed.
"""
new_config = {"id": "foo.bar.12345", "instances": 5}
happy_tasks = []
old_app_live_happy_tasks = [mock.Mock() for _ in range(5)]
old_app_live_unhappy_tasks = [mock.Mock() for _ in range(5)]
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_non_draining_tasks=old_app_live_happy_tasks +
old_app_live_unhappy_tasks,
)
assert actual["create_app"] is False
assert actual["tasks_to_drain"] == set(old_app_live_unhappy_tasks)
# Since there are plenty of unhappy old tasks, we should not kill any new ones.
assert len(actual["tasks_to_drain"]
& set(old_app_live_happy_tasks)) == 0
def test_crossover_bounce_some_unhappy_old_no_happy_old_no_new_tasks_no_excess(self):
"""When marathon only has old apps for this service, and all of their tasks are unhappy, and there are no excess
tasks, the crossover bounce should start a new app and not kill any old tasks.
"""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = []
old_app_live_happy_tasks = {}
old_app_live_unhappy_tasks = {
'app1': set(mock.Mock() for _ in xrange(2)),
'app2': set(mock.Mock() for _ in xrange(3)),
}
assert bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
) == {
"create_app": False,
"tasks_to_drain": set(),
}
def test_crossover_bounce_mid_bounce_no_happy_old_lots_of_unhappy_old(self):
"""When marathon has the desired app, and there are other copies of the service running, but none of the old
tasks are happy, and there are excess tasks, we should kill some (but not all) unhappy old tasks."""
new_config = {'id': 'foo.bar.12345', 'instances': 5}
happy_tasks = [mock.Mock() for _ in xrange(3)]
old_app_live_happy_tasks = {
'app1': set(),
'app2': set(),
}
old_app_live_unhappy_tasks = {
'app1': set(mock.Mock() for _ in xrange(3)),
'app2': set(mock.Mock() for _ in xrange(3)),
}
actual = bounce_lib.crossover_bounce(
new_config=new_config,
new_app_running=True,
happy_new_tasks=happy_tasks,
old_app_live_happy_tasks=old_app_live_happy_tasks,
old_app_live_unhappy_tasks=old_app_live_unhappy_tasks,
)
assert actual['create_app'] is False
assert len(actual['tasks_to_drain']) == 4
