def test_deploy_service_logs_exceptions(self):
fake_bounce = 'WHEEEEEEEEEEEEEEEE'
fake_drain_method = 'noop'
fake_name = 'whoa'
fake_instance = 'the_earth_is_tiny'
fake_id = marathon_tools.format_job_id(fake_name, fake_instance)
fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[])]
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=fake_apps))
fake_config = {'id': fake_id, 'instances': 2}
with contextlib.nested(
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.bounce_lib.get_bounce_method_func', side_effect=IOError('foo')),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
) as (mock_log, mock_bounce, mock_load_system_paasta_config):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster')
with raises(IOError):
setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir='fake_soa_dir',
)
assert fake_name in mock_log.mock_calls[0][2]["line"]
assert 'Traceback' in mock_log.mock_calls[1][2]["line"]
def test_deploy_service_logs_exceptions(self):
fake_bounce = "WHEEEEEEEEEEEEEEEE"
fake_drain_method = "noop"
fake_name = "whoa"
fake_instance = "the_earth_is_tiny"
fake_id = marathon_tools.format_job_id(fake_name, fake_instance)
fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=("%s2" % fake_id), tasks=[])]
fake_client = mock.MagicMock(list_apps=mock.Mock(return_value=fake_apps))
fake_config = {"id": fake_id, "instances": 2}
with contextlib.nested(
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.bounce_lib.get_bounce_method_func", side_effect=IOError("foo")),
mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True),
) as (mock_log, mock_bounce, mock_load_system_paasta_config):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value="fake_cluster")
with raises(IOError):
setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir="fake_soa_dir",
)
assert fake_name in mock_log.mock_calls[0][2]["line"]
assert "Traceback" in mock_log.mock_calls[1][2]["line"]
def when_deploy_service_initiated(context, bounce_method, drain_method):
with contextlib.nested(
mock.patch(
'paasta_tools.bounce_lib.get_happy_tasks',
autospec=True,
# Wrap function call so we can select a subset of tasks or test
# intermediate steps, like when an app is not completely up
side_effect=lambda app, _, __, **kwargs: get_happy_tasks(
app, context.service, "fake_nerve_ns")[:context.max_tasks],
),
mock.patch('paasta_tools.bounce_lib.bounce_lock_zookeeper', autospec=True),
mock.patch('paasta_tools.bounce_lib.create_app_lock', autospec=True),
mock.patch('paasta_tools.bounce_lib.time.sleep', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
) as (
_,
_,
_,
_,
mock_load_system_paasta_config,
):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value=context.cluster)
setup_marathon_job.deploy_service(
service=context.service,
instance=context.instance,
marathon_jobid=context.new_config['id'],
config=context.new_config,
client=context.marathon_client,
bounce_method=bounce_method,
drain_method_name=drain_method,
drain_method_params={},
nerve_ns=context.instance,
bounce_health_params={},
soa_dir=None,
)
def test_deploy_service_unknown_drain_method(self):
fake_bounce = "exists"
fake_drain_method = "doesntexist"
fake_name = "whoa"
fake_instance = "the_earth_is_tiny"
fake_id = marathon_tools.format_job_id(fake_name, fake_instance)
fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=("%s2" % fake_id), tasks=[])]
fake_client = mock.MagicMock(list_apps=mock.Mock(return_value=fake_apps))
fake_config = {"id": fake_id, "instances": 2}
errormsg = "ERROR: drain_method not recognized: doesntexist. Must be one of (exists1, exists2)"
expected = (1, errormsg)
with contextlib.nested(
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True),
mock.patch("paasta_tools.drain_lib._drain_methods", new={"exists1": mock.Mock(), "exists2": mock.Mock()}),
) as (mock_log, mock_load_system_paasta_config, mock_drain_methods):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value="fake_cluster")
actual = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir="fake_soa_dir",
)
assert mock_log.call_count == 1
assert expected == actual
def when_deploy_service_initiated(context, bounce_method, drain_method):
with contextlib.nested(
mock.patch(
"paasta_tools.bounce_lib.get_happy_tasks",
autospec=True,
# Wrap function call so we can select a subset of tasks or test
# intermediate steps, like when an app is not completely up
side_effect=lambda app, _, __, ___, **kwargs: get_happy_tasks(
app, context.service, "fake_nerve_ns", context.system_paasta_config
)[: context.max_tasks],
),
mock.patch("paasta_tools.bounce_lib.bounce_lock_zookeeper", autospec=True),
mock.patch("paasta_tools.bounce_lib.create_app_lock", autospec=True),
mock.patch("paasta_tools.bounce_lib.time.sleep", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True),
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
) as (_, _, _, _, mock_load_system_paasta_config, _):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value=context.cluster)
# 120 * 0.5 = 60 seconds
for _ in xrange(120):
try:
setup_marathon_job.deploy_service(
service=context.service,
instance=context.instance,
marathon_jobid=context.new_config["id"],
config=context.new_config,
client=context.marathon_client,
bounce_method=bounce_method,
drain_method_name=drain_method,
drain_method_params={},
nerve_ns=context.instance,
bounce_health_params={},
soa_dir=None,
)
return
except MarathonHttpError:
time.sleep(0.5)
raise Exception("Unable to qcuiqre app lock for setup_marathon_job.deploy_service")
def test_deploy_service_already_bouncing(self):
fake_bounce = "areallygoodbouncestrategy"
fake_drain_method = "noop"
fake_name = "how_many_strings"
fake_instance = "will_i_need_to_think_of"
fake_id = marathon_tools.format_job_id(fake_name, fake_instance, "gityourmom", "configyourdad")
fake_config = {"id": fake_id, "instances": 2}
old_app_id = "%s2" % fake_id
old_task = mock.Mock(id="old_task_id", app_id=old_app_id)
old_app = mock.Mock(id=old_app_id, tasks=[old_task])
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=[old_app]), kill_task=mock.Mock(spec=lambda app_id, id, scale=False: None)
)
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce, return_value={"create_app": True, "tasks_to_drain": [old_task]}
)
fake_short_id = marathon_tools.format_job_id(fake_name, fake_instance)
with contextlib.nested(
mock.patch("paasta_tools.bounce_lib.get_bounce_method_func", return_value=fake_bounce_func, autospec=True),
mock.patch(
"paasta_tools.bounce_lib.bounce_lock_zookeeper", side_effect=bounce_lib.LockHeldException, autospec=True
),
mock.patch(
"paasta_tools.bounce_lib.get_happy_tasks", autospec=True, side_effect=lambda x, _, __, **kwargs: x
),
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True),
) as (_, _, _, _, mock_load_system_paasta_config):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value="fake_cluster")
result = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir="fake_soa_dir",
)
assert result == (1, "Instance %s is already being bounced." % fake_short_id)
def test_deploy_service_unknown_drain_method(self):
fake_bounce = 'exists'
fake_drain_method = 'doesntexist'
fake_name = 'whoa'
fake_instance = 'the_earth_is_tiny'
fake_id = marathon_tools.format_job_id(fake_name, fake_instance)
fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[])]
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=fake_apps))
fake_config = {'id': fake_id, 'instances': 2}
errormsg = 'ERROR: drain_method not recognized: doesntexist. Must be one of (exists1, exists2)'
expected = (1, errormsg)
with contextlib.nested(
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
mock.patch(
'paasta_tools.drain_lib._drain_methods',
new={'exists1': mock.Mock(), 'exists2': mock.Mock()},
)
) as (mock_log, mock_load_system_paasta_config, mock_drain_methods):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster')
actual = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir='fake_soa_dir',
)
assert mock_log.call_count == 1
assert expected == actual
def test_deploy_service_unknown_bounce(self):
fake_bounce = "WHEEEEEEEEEEEEEEEE"
fake_drain_method = "noop"
fake_name = "whoa"
fake_instance = "the_earth_is_tiny"
fake_id = marathon_tools.format_job_id(fake_name, fake_instance)
fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=("%s2" % fake_id), tasks=[])]
fake_client = mock.MagicMock(list_apps=mock.Mock(return_value=fake_apps))
fake_config = {"id": fake_id, "instances": 2}
errormsg = "ERROR: bounce_method not recognized: %s. Must be one of (%s)" % (
fake_bounce,
", ".join(list_bounce_methods()),
)
expected = (1, errormsg)
with contextlib.nested(
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True),
) as (mock_log, mock_load_system_paasta_config):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value="fake_cluster")
actual = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir="fake_soa_dir",
)
assert mock_log.call_count == 1
assert expected == actual
fake_client.list_apps.assert_called_once_with(embed_failures=True)
assert fake_client.create_app.call_count == 0
def test_deploy_service_unknown_bounce(self):
fake_bounce = 'WHEEEEEEEEEEEEEEEE'
fake_drain_method = 'noop'
fake_name = 'whoa'
fake_instance = 'the_earth_is_tiny'
fake_id = marathon_tools.format_job_id(fake_name, fake_instance)
fake_apps = [mock.Mock(id=fake_id, tasks=[]), mock.Mock(id=('%s2' % fake_id), tasks=[])]
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=fake_apps))
fake_config = {'id': fake_id, 'instances': 2}
errormsg = 'ERROR: bounce_method not recognized: %s. Must be one of (%s)' % \
(fake_bounce, ', '.join(list_bounce_methods()))
expected = (1, errormsg)
with contextlib.nested(
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
) as (mock_log, mock_load_system_paasta_config):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster')
actual = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir='fake_soa_dir',
)
assert mock_log.call_count == 1
assert expected == actual
fake_client.list_apps.assert_called_once_with(embed_failures=True)
assert fake_client.create_app.call_count == 0
def test_deploy_service_already_bouncing(self):
fake_bounce = 'areallygoodbouncestrategy'
fake_drain_method = 'noop'
fake_name = 'how_many_strings'
fake_instance = 'will_i_need_to_think_of'
fake_id = marathon_tools.format_job_id(fake_name, fake_instance, 'gityourmom', 'configyourdad')
fake_config = {'id': fake_id, 'instances': 2}
old_app_id = ('%s2' % fake_id)
old_task = mock.Mock(id="old_task_id", app_id=old_app_id)
old_app = mock.Mock(id=old_app_id, tasks=[old_task])
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=[old_app]),
kill_task=mock.Mock(spec=lambda app_id, id, scale=False: None),
)
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce,
return_value={
"create_app": True,
"tasks_to_drain": [old_task],
}
)
fake_short_id = marathon_tools.format_job_id(fake_name, fake_instance)
with contextlib.nested(
mock.patch(
'paasta_tools.bounce_lib.get_bounce_method_func',
return_value=fake_bounce_func,
autospec=True,
),
mock.patch(
'paasta_tools.bounce_lib.bounce_lock_zookeeper',
side_effect=bounce_lib.LockHeldException,
autospec=True
),
mock.patch(
'paasta_tools.bounce_lib.get_happy_tasks',
autospec=True,
side_effect=lambda x, _, __, **kwargs: x,
),
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
) as (_, _, _, _, mock_load_system_paasta_config):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster')
result = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir='fake_soa_dir',
)
assert result == (1, "Instance %s is already being bounced." % fake_short_id)
def test_deploy_service_known_bounce(self):
fake_bounce = 'areallygoodbouncestrategy'
fake_drain_method_name = 'noop'
fake_name = 'how_many_strings'
fake_instance = 'will_i_need_to_think_of'
fake_id = marathon_tools.format_job_id(fake_name, fake_instance, 'git11111111', 'config11111111')
fake_config = {'id': fake_id, 'instances': 2}
old_app_id = marathon_tools.format_job_id(fake_name, fake_instance, 'git22222222', 'config22222222')
old_task_to_drain = mock.Mock(id="old_task_to_drain", app_id=old_app_id)
old_task_is_draining = mock.Mock(id="old_task_is_draining", app_id=old_app_id)
old_task_dont_drain = mock.Mock(id="old_task_dont_drain", app_id=old_app_id)
old_app = mock.Mock(id="/%s" % old_app_id, tasks=[old_task_to_drain, old_task_is_draining, old_task_dont_drain])
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=[old_app]),
kill_task=mock.Mock(spec=lambda app_id, id, scale=False: None),
)
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce,
return_value={
"create_app": True,
"tasks_to_drain": [old_task_to_drain],
}
)
fake_drain_method = mock.Mock(is_draining=lambda t: t is old_task_is_draining, is_safe_to_kill=lambda t: True)
with contextlib.nested(
mock.patch(
'paasta_tools.bounce_lib.get_bounce_method_func',
return_value=fake_bounce_func,
autospec=True,
),
mock.patch(
'paasta_tools.bounce_lib.bounce_lock_zookeeper',
autospec=True
),
mock.patch(
'paasta_tools.bounce_lib.get_happy_tasks',
autospec=True,
side_effect=lambda x, _, __, **kwargs: x,
),
mock.patch('paasta_tools.bounce_lib.kill_old_ids', autospec=True),
mock.patch('paasta_tools.bounce_lib.create_marathon_app', autospec=True),
mock.patch('paasta_tools.setup_marathon_job._log', autospec=True),
mock.patch('paasta_tools.setup_marathon_job.load_system_paasta_config', autospec=True),
mock.patch('paasta_tools.drain_lib.get_drain_method', return_value=fake_drain_method),
) as (_, _, _, kill_old_ids_patch, create_marathon_app_patch, mock_log, mock_load_system_paasta_config, _):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value='fake_cluster')
result = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method_name,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir='fake_soa_dir',
)
assert result[0] == 0, "Expected successful result; got (%d, %s)" % result
fake_client.list_apps.assert_called_once_with(embed_failures=True)
assert fake_client.create_app.call_count == 0
fake_bounce_func.assert_called_once_with(
new_config=fake_config,
new_app_running=False,
happy_new_tasks=[],
old_app_live_tasks={old_app.id: set([old_task_to_drain, old_task_dont_drain])},
)
assert fake_drain_method.drain.call_count == 2
fake_drain_method.drain.assert_any_call(old_task_is_draining)
fake_drain_method.drain.assert_any_call(old_task_to_drain)
assert fake_client.kill_task.call_count == 2
fake_client.kill_task.assert_any_call(old_app_id, old_task_is_draining.id, scale=True)
fake_client.kill_task.assert_any_call(old_app_id, old_task_to_drain.id, scale=True)
create_marathon_app_patch.assert_called_once_with(fake_config['id'], fake_config, fake_client)
assert kill_old_ids_patch.call_count == 0
# We should call _log 5 times:
# 1. bounce starts
# 2. create new app
# 3. draining old tasks
# 4. remove old apps
# 5. bounce finishes
assert mock_log.call_count == 5
def test_deploy_service_known_bounce(self):
fake_bounce = "areallygoodbouncestrategy"
fake_drain_method_name = "noop"
fake_name = "how_many_strings"
fake_instance = "will_i_need_to_think_of"
fake_id = marathon_tools.format_job_id(fake_name, fake_instance, "git11111111", "config11111111")
fake_config = {"id": fake_id, "instances": 2}
old_app_id = marathon_tools.format_job_id(fake_name, fake_instance, "git22222222", "config22222222")
old_task_to_drain = mock.Mock(id="old_task_to_drain", app_id=old_app_id)
old_task_is_draining = mock.Mock(id="old_task_is_draining", app_id=old_app_id)
old_task_dont_drain = mock.Mock(id="old_task_dont_drain", app_id=old_app_id)
old_app = mock.Mock(id="/%s" % old_app_id, tasks=[old_task_to_drain, old_task_is_draining, old_task_dont_drain])
fake_client = mock.MagicMock(
list_apps=mock.Mock(return_value=[old_app]), kill_task=mock.Mock(spec=lambda app_id, id, scale=False: None)
)
fake_bounce_func = mock.create_autospec(
bounce_lib.brutal_bounce, return_value={"create_app": True, "tasks_to_drain": [old_task_to_drain]}
)
fake_drain_method = mock.Mock(is_draining=lambda t: t is old_task_is_draining, is_safe_to_kill=lambda t: True)
with contextlib.nested(
mock.patch("paasta_tools.bounce_lib.get_bounce_method_func", return_value=fake_bounce_func, autospec=True),
mock.patch("paasta_tools.bounce_lib.bounce_lock_zookeeper", autospec=True),
mock.patch(
"paasta_tools.bounce_lib.get_happy_tasks", autospec=True, side_effect=lambda x, _, __, **kwargs: x
),
mock.patch("paasta_tools.bounce_lib.kill_old_ids", autospec=True),
mock.patch("paasta_tools.bounce_lib.create_marathon_app", autospec=True),
mock.patch("paasta_tools.setup_marathon_job._log", autospec=True),
mock.patch("paasta_tools.setup_marathon_job.load_system_paasta_config", autospec=True),
mock.patch("paasta_tools.drain_lib.get_drain_method", return_value=fake_drain_method),
) as (_, _, _, kill_old_ids_patch, create_marathon_app_patch, mock_log, mock_load_system_paasta_config, _):
mock_load_system_paasta_config.return_value.get_cluster = mock.Mock(return_value="fake_cluster")
result = setup_marathon_job.deploy_service(
service=fake_name,
instance=fake_instance,
marathon_jobid=fake_id,
config=fake_config,
client=fake_client,
bounce_method=fake_bounce,
drain_method_name=fake_drain_method_name,
drain_method_params={},
nerve_ns=fake_instance,
bounce_health_params={},
soa_dir="fake_soa_dir",
)
assert result[0] == 0, "Expected successful result; got (%d, %s)" % result
fake_client.list_apps.assert_called_once_with(embed_failures=True)
assert fake_client.create_app.call_count == 0
fake_bounce_func.assert_called_once_with(
new_config=fake_config,
new_app_running=False,
happy_new_tasks=[],
old_app_live_tasks={old_app.id: set([old_task_to_drain, old_task_dont_drain])},
)
assert fake_drain_method.drain.call_count == 2
fake_drain_method.drain.assert_any_call(old_task_is_draining)
fake_drain_method.drain.assert_any_call(old_task_to_drain)
assert fake_client.kill_task.call_count == 2
fake_client.kill_task.assert_any_call(app_id=old_app_id, task_id=old_task_is_draining.id, scale=True)
fake_client.kill_task.assert_any_call(app_id=old_app_id, task_id=old_task_to_drain.id, scale=True)
create_marathon_app_patch.assert_called_once_with(fake_config["id"], fake_config, fake_client)
assert kill_old_ids_patch.call_count == 0
# We should call _log 5 times:
# 1. bounce starts
# 2. create new app
# 3. draining old tasks
# 4. remove old apps
# 5. bounce finishes
assert mock_log.call_count == 5
