def main():
args = parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
autoscale_services(soa_dir=args.soa_dir, services=args.services)
def main():
args = parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
autoscale_services(soa_dir=args.soa_dir)
def test_autoscale_services_happy_path():
fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
service='fake-service',
instance='fake-instance',
cluster='fake-cluster',
config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
branch_dict={},
)
mock_mesos_tasks = [{'id': 'fake-service.fake-instance.sha123.sha456'}]
mock_healthcheck_results = mock.Mock(alive=True)
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456',
health_check_results=[mock_healthcheck_results])]
with mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True,
) as mock_autoscale_marathon_instance, mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
return_value=mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks)),
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_all_running_tasks',
autospec=True,
return_value=mock_mesos_tasks,
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_cluster=mock.Mock()),
), mock.patch(
'paasta_tools.utils.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_zk_hosts=mock.Mock()),
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
return_value=[('fake-service', 'fake-instance')],
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
return_value=fake_marathon_service_config,
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True,
), mock.patch(
'paasta_tools.utils.KazooClient', autospec=True,
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True,
), mock.patch(
'paasta_tools.marathon_tools.MarathonServiceConfig.format_marathon_app_dict', autospec=True,
) as mock_format_marathon_app_dict:
mock_format_marathon_app_dict.return_value = {'id': 'fake-service.fake-instance.sha123.sha456'}
autoscaling_service_lib.autoscale_services()
mock_autoscale_marathon_instance.assert_called_once_with(
fake_marathon_service_config, mock_marathon_tasks, mock_mesos_tasks)
def test_autoscale_services_happy_path():
fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
service='fake-service',
instance='fake-instance',
cluster='fake-cluster',
config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
branch_dict={},
)
mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
mock_healthcheck_results = mock.Mock(alive=True)
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
health_check_results=[mock_healthcheck_results])]
with contextlib.nested(
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
return_value=mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks))),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_active_frameworks',
autospec=True,
return_value=mock_mesos_tasks),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_cluster=mock.Mock())),
mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_zk_hosts=mock.Mock())),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
return_value=[('fake-service', 'fake-instance')]),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
return_value=fake_marathon_service_config),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True),
mock.patch('paasta_tools.utils.KazooClient', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True),
) as (
mock_autoscale_marathon_instance,
_,
_,
_,
_,
_,
_,
_,
_,
_,
):
autoscaling_service_lib.autoscale_services()
mock_autoscale_marathon_instance.assert_called_once_with(
fake_marathon_service_config, mock_marathon_tasks, mock_mesos_tasks)
def test_autoscale_services_bespoke_doesnt_autoscale():
fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
service='fake-service',
instance='fake-instance',
cluster='fake-cluster',
config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start',
'autoscaling': {'decision_policy': 'bespoke'}},
branch_dict={},
)
mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance')]
with contextlib.nested(
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
return_value=mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks))),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_frameworks',
autospec=True,
return_value=mock_mesos_tasks),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_cluster=mock.Mock())),
mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_zk_hosts=mock.Mock())),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
return_value=[('fake-service', 'fake-instance')]),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
return_value=fake_marathon_service_config),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True),
mock.patch('paasta_tools.utils.KazooClient', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True),
) as (
mock_autoscale_marathon_instance,
_,
_,
_,
_,
_,
_,
_,
_,
_,
):
autoscaling_service_lib.autoscale_services()
assert not mock_autoscale_marathon_instance.called
def test_autoscale_services_not_healthy():
fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
service='fake-service',
instance='fake-instance',
cluster='fake-cluster',
config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
branch_dict={},
)
mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
with contextlib.nested(
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.write_to_log', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_active_frameworks',
autospec=True,
return_value=mock_mesos_tasks),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_cluster=mock.Mock())),
mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_zk_hosts=mock.Mock())),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
return_value=[('fake-service', 'fake-instance')]),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
return_value=fake_marathon_service_config),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True),
mock.patch('paasta_tools.utils.KazooClient', autospec=True),
mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True),
) as (
mock_autoscale_marathon_instance,
mock_write_to_log,
mock_marathon_client,
_,
_,
_,
_,
_,
_,
_,
_,
):
# Test missing health_check_results
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
health_check_results=[])]
mock_marathon_app = mock.Mock()
mock_marathon_app.health_checks = ["some-healthcheck-definition"]
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
assert not mock_autoscale_marathon_instance.called
# Test present results but not yet passing
mock_healthcheck_results = mock.Mock(alive=False)
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
health_check_results=[mock_healthcheck_results])]
mock_marathon_app = mock.Mock()
mock_marathon_app.health_checks = ["some-healthcheck-definition"]
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
assert not mock_autoscale_marathon_instance.called
# Test no healthcheck defined
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
health_check_results=[])]
mock_marathon_app = mock.Mock()
mock_marathon_app.health_checks = []
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
assert mock_autoscale_marathon_instance.called
def test_autoscale_services_not_healthy():
fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
service='fake-service',
instance='fake-instance',
cluster='fake-cluster',
config_dict={
'min_instances': 1,
'max_instances': 10,
'desired_state': 'start'
},
branch_dict={},
)
mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
with contextlib.nested(
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance',
autospec=True),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.write_to_log',
autospec=True),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client',
autospec=True),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_active_frameworks',
autospec=True,
return_value=mock_mesos_tasks),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config',
autospec=True,
return_value=mock.Mock(get_cluster=mock.Mock())),
mock.patch('paasta_tools.utils.load_system_paasta_config',
autospec=True,
return_value=mock.Mock(get_zk_hosts=mock.Mock())),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster',
autospec=True,
return_value=[('fake-service', 'fake-instance')]),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config',
autospec=True,
return_value=fake_marathon_service_config),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config',
autospec=True),
mock.patch('paasta_tools.utils.KazooClient', autospec=True),
mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock',
autospec=True),
) as (
mock_autoscale_marathon_instance,
mock_write_to_log,
mock_marathon_client,
_,
_,
_,
_,
_,
_,
_,
_,
):
# Test missing health_check_results
mock_marathon_tasks = [
mock.Mock(id='fake-service.fake-instance', health_check_results=[])
]
mock_marathon_app = mock.Mock()
mock_marathon_app.health_checks = ["some-healthcheck-definition"]
mock_marathon_client.return_value = mock.Mock(
list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
mock_write_to_log.assert_called_with(
config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
assert not mock_autoscale_marathon_instance.called
# Test present results but not yet passing
mock_healthcheck_results = mock.Mock(alive=False)
mock_marathon_tasks = [
mock.Mock(id='fake-service.fake-instance',
health_check_results=[mock_healthcheck_results])
]
mock_marathon_app = mock.Mock()
mock_marathon_app.health_checks = ["some-healthcheck-definition"]
mock_marathon_client.return_value = mock.Mock(
list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
mock_write_to_log.assert_called_with(
config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
assert not mock_autoscale_marathon_instance.called
# Test no healthcheck defined
mock_marathon_tasks = [
mock.Mock(id='fake-service.fake-instance', health_check_results=[])
]
mock_marathon_app = mock.Mock()
mock_marathon_app.health_checks = []
mock_marathon_client.return_value = mock.Mock(
list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
mock_write_to_log.assert_called_with(
config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
assert mock_autoscale_marathon_instance.called
def test_autoscale_services_not_healthy():
fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
service='fake-service',
instance='fake-instance',
cluster='fake-cluster',
config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
branch_dict={},
)
mock_mesos_tasks = [{'id': 'fake-service.fake-instance.sha123.sha456.uuid'}]
with mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True,
) as mock_autoscale_marathon_instance, mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.write_to_log', autospec=True,
) as mock_write_to_log, mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
) as mock_marathon_client, mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_all_running_tasks',
autospec=True,
return_value=mock_mesos_tasks,
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_cluster=mock.Mock()),
), mock.patch(
'paasta_tools.utils.load_system_paasta_config', autospec=True,
return_value=mock.Mock(get_zk_hosts=mock.Mock()),
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
return_value=[('fake-service', 'fake-instance')],
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
return_value=fake_marathon_service_config,
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True,
), mock.patch(
'paasta_tools.utils.KazooClient', autospec=True,
), mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True,
), mock.patch(
'paasta_tools.marathon_tools.MarathonServiceConfig.format_marathon_app_dict', autospec=True,
) as mock_format_marathon_app_dict, mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.is_task_healthy', autospec=True,
) as mock_is_task_healthy, mock.patch(
'paasta_tools.autoscaling.autoscaling_service_lib.is_old_task_missing_healthchecks', autospec=True,
) as mock_is_old_task_missing_healthchecks:
mock_is_task_healthy.return_value = True
mock_is_old_task_missing_healthchecks.return_value = False
mock_format_marathon_app_dict.return_value = {'id': 'fake-service.fake-instance.sha123.sha456'}
# Test healthy task
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
mock_health_check = mock.Mock()
mock_marathon_app = mock.Mock(health_checks=[mock_health_check])
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
assert mock_autoscale_marathon_instance.called
mock_autoscale_marathon_instance.reset_mock()
# Test unhealthy task
mock_is_task_healthy.reset_mock()
mock_is_task_healthy.return_value = False
mock_is_old_task_missing_healthchecks.return_value = False
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
mock_health_check = mock.Mock()
mock_marathon_app = mock.Mock(health_checks=[mock_health_check])
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
assert not mock_autoscale_marathon_instance.called
mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
line="Caught Exception Couldn't find any healthy marathon tasks")
mock_autoscale_marathon_instance.reset_mock()
# Test no healthcheck defined
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
mock_health_check = mock.Mock()
mock_marathon_app = mock.Mock(health_checks=[])
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
assert mock_autoscale_marathon_instance.called
mock_autoscale_marathon_instance.reset_mock()
# Test unhealthy but old missing hcr
mock_is_task_healthy.reset_mock()
mock_is_task_healthy.return_value = False
mock_is_old_task_missing_healthchecks.return_value = True
mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
mock_health_check = mock.Mock()
mock_marathon_app = mock.Mock(health_checks=[mock_health_check])
mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
get_app=mock.Mock(return_value=mock_marathon_app))
autoscaling_service_lib.autoscale_services()
assert mock_autoscale_marathon_instance.called
