示例#1
0
def main():
    args = parse_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.WARNING)
    autoscale_services(soa_dir=args.soa_dir, services=args.services)
示例#2
0
def main():
    args = parse_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.WARNING)
    autoscale_services(soa_dir=args.soa_dir)
示例#3
0
def test_autoscale_services_happy_path():
    fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
        service='fake-service',
        instance='fake-instance',
        cluster='fake-cluster',
        config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
        branch_dict={},
    )
    mock_mesos_tasks = [{'id': 'fake-service.fake-instance.sha123.sha456'}]
    mock_healthcheck_results = mock.Mock(alive=True)
    mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456',
                                     health_check_results=[mock_healthcheck_results])]
    with mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True,
    ) as mock_autoscale_marathon_instance, mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
        return_value=mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks)),
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.get_all_running_tasks',
        autospec=True,
        return_value=mock_mesos_tasks,
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
        return_value=mock.Mock(get_cluster=mock.Mock()),
    ), mock.patch(
        'paasta_tools.utils.load_system_paasta_config', autospec=True,
        return_value=mock.Mock(get_zk_hosts=mock.Mock()),
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
        return_value=[('fake-service', 'fake-instance')],
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
        return_value=fake_marathon_service_config,
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True,
    ), mock.patch(
        'paasta_tools.utils.KazooClient', autospec=True,
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True,
    ), mock.patch(
        'paasta_tools.marathon_tools.MarathonServiceConfig.format_marathon_app_dict', autospec=True,
    ) as mock_format_marathon_app_dict:
        mock_format_marathon_app_dict.return_value = {'id': 'fake-service.fake-instance.sha123.sha456'}
        autoscaling_service_lib.autoscale_services()
        mock_autoscale_marathon_instance.assert_called_once_with(
            fake_marathon_service_config, mock_marathon_tasks, mock_mesos_tasks)
def test_autoscale_services_happy_path():
    fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
        service='fake-service',
        instance='fake-instance',
        cluster='fake-cluster',
        config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
        branch_dict={},
    )
    mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
    mock_healthcheck_results = mock.Mock(alive=True)
    mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
                                     health_check_results=[mock_healthcheck_results])]
    with contextlib.nested(
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
                   return_value=mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks))),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_active_frameworks',
                   autospec=True,
                   return_value=mock_mesos_tasks),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
                   return_value=mock.Mock(get_cluster=mock.Mock())),
        mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True,
                   return_value=mock.Mock(get_zk_hosts=mock.Mock())),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
                   return_value=[('fake-service', 'fake-instance')]),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
                   return_value=fake_marathon_service_config),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True),
        mock.patch('paasta_tools.utils.KazooClient', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True),
    ) as (
        mock_autoscale_marathon_instance,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
    ):
        autoscaling_service_lib.autoscale_services()
        mock_autoscale_marathon_instance.assert_called_once_with(
            fake_marathon_service_config, mock_marathon_tasks, mock_mesos_tasks)
示例#5
0
def test_autoscale_services_bespoke_doesnt_autoscale():
    fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
        service='fake-service',
        instance='fake-instance',
        cluster='fake-cluster',
        config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start',
                     'autoscaling': {'decision_policy': 'bespoke'}},
        branch_dict={},
    )
    mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
    mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance')]
    with contextlib.nested(
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
                   return_value=mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks))),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_frameworks',
                   autospec=True,
                   return_value=mock_mesos_tasks),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
                   return_value=mock.Mock(get_cluster=mock.Mock())),
        mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True,
                   return_value=mock.Mock(get_zk_hosts=mock.Mock())),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
                   return_value=[('fake-service', 'fake-instance')]),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
                   return_value=fake_marathon_service_config),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True),
        mock.patch('paasta_tools.utils.KazooClient', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True),
    ) as (
        mock_autoscale_marathon_instance,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
    ):
        autoscaling_service_lib.autoscale_services()
        assert not mock_autoscale_marathon_instance.called
def test_autoscale_services_not_healthy():
    fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
        service='fake-service',
        instance='fake-instance',
        cluster='fake-cluster',
        config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
        branch_dict={},
    )
    mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
    with contextlib.nested(
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.write_to_log', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_active_frameworks',
                   autospec=True,
                   return_value=mock_mesos_tasks),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
                   return_value=mock.Mock(get_cluster=mock.Mock())),
        mock.patch('paasta_tools.utils.load_system_paasta_config', autospec=True,
                   return_value=mock.Mock(get_zk_hosts=mock.Mock())),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
                   return_value=[('fake-service', 'fake-instance')]),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
                   return_value=fake_marathon_service_config),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True),
        mock.patch('paasta_tools.utils.KazooClient', autospec=True),
        mock.patch('paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True),
    ) as (
        mock_autoscale_marathon_instance,
        mock_write_to_log,
        mock_marathon_client,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
        _,
    ):

        # Test missing health_check_results
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
                                         health_check_results=[])]
        mock_marathon_app = mock.Mock()
        mock_marathon_app.health_checks = ["some-healthcheck-definition"]
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
                                             line="Caught Exception Couldn't find any healthy marathon tasks")
        assert not mock_autoscale_marathon_instance.called

        # Test present results but not yet passing
        mock_healthcheck_results = mock.Mock(alive=False)
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
                                         health_check_results=[mock_healthcheck_results])]
        mock_marathon_app = mock.Mock()
        mock_marathon_app.health_checks = ["some-healthcheck-definition"]
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
                                             line="Caught Exception Couldn't find any healthy marathon tasks")
        assert not mock_autoscale_marathon_instance.called

        # Test no healthcheck defined
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance',
                                         health_check_results=[])]
        mock_marathon_app = mock.Mock()
        mock_marathon_app.health_checks = []
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
                                             line="Caught Exception Couldn't find any healthy marathon tasks")
        assert mock_autoscale_marathon_instance.called
def test_autoscale_services_not_healthy():
    fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
        service='fake-service',
        instance='fake-instance',
        cluster='fake-cluster',
        config_dict={
            'min_instances': 1,
            'max_instances': 10,
            'desired_state': 'start'
        },
        branch_dict={},
    )
    mock_mesos_tasks = [{'id': 'fake-service.fake-instance'}]
    with contextlib.nested(
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance',
                autospec=True),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.write_to_log',
                autospec=True),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client',
                autospec=True),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.get_running_tasks_from_active_frameworks',
                autospec=True,
                return_value=mock_mesos_tasks),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config',
                autospec=True,
                return_value=mock.Mock(get_cluster=mock.Mock())),
            mock.patch('paasta_tools.utils.load_system_paasta_config',
                       autospec=True,
                       return_value=mock.Mock(get_zk_hosts=mock.Mock())),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster',
                autospec=True,
                return_value=[('fake-service', 'fake-instance')]),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config',
                autospec=True,
                return_value=fake_marathon_service_config),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config',
                autospec=True),
            mock.patch('paasta_tools.utils.KazooClient', autospec=True),
            mock.patch(
                'paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock',
                autospec=True),
    ) as (
            mock_autoscale_marathon_instance,
            mock_write_to_log,
            mock_marathon_client,
            _,
            _,
            _,
            _,
            _,
            _,
            _,
            _,
    ):

        # Test missing health_check_results
        mock_marathon_tasks = [
            mock.Mock(id='fake-service.fake-instance', health_check_results=[])
        ]
        mock_marathon_app = mock.Mock()
        mock_marathon_app.health_checks = ["some-healthcheck-definition"]
        mock_marathon_client.return_value = mock.Mock(
            list_tasks=mock.Mock(return_value=mock_marathon_tasks),
            get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        mock_write_to_log.assert_called_with(
            config=fake_marathon_service_config,
            line="Caught Exception Couldn't find any healthy marathon tasks")
        assert not mock_autoscale_marathon_instance.called

        # Test present results but not yet passing
        mock_healthcheck_results = mock.Mock(alive=False)
        mock_marathon_tasks = [
            mock.Mock(id='fake-service.fake-instance',
                      health_check_results=[mock_healthcheck_results])
        ]
        mock_marathon_app = mock.Mock()
        mock_marathon_app.health_checks = ["some-healthcheck-definition"]
        mock_marathon_client.return_value = mock.Mock(
            list_tasks=mock.Mock(return_value=mock_marathon_tasks),
            get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        mock_write_to_log.assert_called_with(
            config=fake_marathon_service_config,
            line="Caught Exception Couldn't find any healthy marathon tasks")
        assert not mock_autoscale_marathon_instance.called

        # Test no healthcheck defined
        mock_marathon_tasks = [
            mock.Mock(id='fake-service.fake-instance', health_check_results=[])
        ]
        mock_marathon_app = mock.Mock()
        mock_marathon_app.health_checks = []
        mock_marathon_client.return_value = mock.Mock(
            list_tasks=mock.Mock(return_value=mock_marathon_tasks),
            get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        mock_write_to_log.assert_called_with(
            config=fake_marathon_service_config,
            line="Caught Exception Couldn't find any healthy marathon tasks")
        assert mock_autoscale_marathon_instance.called
示例#8
0
def test_autoscale_services_not_healthy():
    fake_marathon_service_config = marathon_tools.MarathonServiceConfig(
        service='fake-service',
        instance='fake-instance',
        cluster='fake-cluster',
        config_dict={'min_instances': 1, 'max_instances': 10, 'desired_state': 'start'},
        branch_dict={},
    )
    mock_mesos_tasks = [{'id': 'fake-service.fake-instance.sha123.sha456.uuid'}]
    with mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.autoscale_marathon_instance', autospec=True,
    ) as mock_autoscale_marathon_instance, mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.write_to_log', autospec=True,
    ) as mock_write_to_log, mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.get_marathon_client', autospec=True,
    ) as mock_marathon_client, mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.get_all_running_tasks',
        autospec=True,
        return_value=mock_mesos_tasks,
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.load_system_paasta_config', autospec=True,
        return_value=mock.Mock(get_cluster=mock.Mock()),
    ), mock.patch(
        'paasta_tools.utils.load_system_paasta_config', autospec=True,
        return_value=mock.Mock(get_zk_hosts=mock.Mock()),
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.get_services_for_cluster', autospec=True,
        return_value=[('fake-service', 'fake-instance')],
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_service_config', autospec=True,
        return_value=fake_marathon_service_config,
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.load_marathon_config', autospec=True,
    ), mock.patch(
        'paasta_tools.utils.KazooClient', autospec=True,
    ), mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.create_autoscaling_lock', autospec=True,
    ), mock.patch(
        'paasta_tools.marathon_tools.MarathonServiceConfig.format_marathon_app_dict', autospec=True,
    ) as mock_format_marathon_app_dict, mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.is_task_healthy', autospec=True,
    ) as mock_is_task_healthy, mock.patch(
        'paasta_tools.autoscaling.autoscaling_service_lib.is_old_task_missing_healthchecks', autospec=True,
    ) as mock_is_old_task_missing_healthchecks:
        mock_is_task_healthy.return_value = True
        mock_is_old_task_missing_healthchecks.return_value = False
        mock_format_marathon_app_dict.return_value = {'id': 'fake-service.fake-instance.sha123.sha456'}
        # Test healthy task
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
        mock_health_check = mock.Mock()
        mock_marathon_app = mock.Mock(health_checks=[mock_health_check])
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        assert mock_autoscale_marathon_instance.called

        mock_autoscale_marathon_instance.reset_mock()
        # Test unhealthy task
        mock_is_task_healthy.reset_mock()
        mock_is_task_healthy.return_value = False
        mock_is_old_task_missing_healthchecks.return_value = False
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
        mock_health_check = mock.Mock()
        mock_marathon_app = mock.Mock(health_checks=[mock_health_check])
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        assert not mock_autoscale_marathon_instance.called
        mock_write_to_log.assert_called_with(config=fake_marathon_service_config,
                                             line="Caught Exception Couldn't find any healthy marathon tasks")

        mock_autoscale_marathon_instance.reset_mock()
        # Test no healthcheck defined
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
        mock_health_check = mock.Mock()
        mock_marathon_app = mock.Mock(health_checks=[])
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        assert mock_autoscale_marathon_instance.called

        mock_autoscale_marathon_instance.reset_mock()
        # Test unhealthy but old missing hcr
        mock_is_task_healthy.reset_mock()
        mock_is_task_healthy.return_value = False
        mock_is_old_task_missing_healthchecks.return_value = True
        mock_marathon_tasks = [mock.Mock(id='fake-service.fake-instance.sha123.sha456')]
        mock_health_check = mock.Mock()
        mock_marathon_app = mock.Mock(health_checks=[mock_health_check])
        mock_marathon_client.return_value = mock.Mock(list_tasks=mock.Mock(return_value=mock_marathon_tasks),
                                                      get_app=mock.Mock(return_value=mock_marathon_app))
        autoscaling_service_lib.autoscale_services()
        assert mock_autoscale_marathon_instance.called