Exemplo n.º 1
0
 def test_should_throw_vcl_load_exception_on_any_error_while_connecting_to_varnish_api(
         self):
     with patch.object(VarnishApi, '__init__', side_effect=Exception):
         with patch.object(VarnishApi, '__del__'):
             varnish_cluster = VarnishApiProvider()
             with self.assertRaises(VclLoadException):
                 varnish_cluster.get_api(servers[0])
Exemplo n.º 2
0
    def test_should_create_varnish_api_clients_for_all_servers(self):
        expected_construct_args = [
            call(['127.0.0.1', '6082', 1.0], 'secret-1'),
            call(['127.0.0.2', '6083', 1.0], 'secret-2'),
            call(['127.0.0.3', '6084', 1.0], 'secret-3')
        ]
        sample_extractor = Mock(servers=servers)

        with patch('vaas.cluster.cluster.ServerExtractor',
                   Mock(return_value=sample_extractor)):
            with patch.object(VarnishApi, '__init__',
                              return_value=None) as construct_mock:
                varnish_cluster = VarnishApiProvider()
                api_objects = []
                for api in varnish_cluster.get_varnish_api():
                    """
                        Workaround - we cannot mock __del__ method:
                        https://docs.python.org/3/library/unittest.mock.html

                        We inject sock field to eliminate warning raised by cleaning actions in __del__ method
                        """
                    api.sock = None
                    api_objects.append(api)

                assert_equals(3, len(api_objects))
                assert_list_equal(expected_construct_args,
                                  construct_mock.call_args_list)
Exemplo n.º 3
0
    def test_should_create_varnish_api_for_connected_servers(self):
        expected_construct_args = [
            call(['127.0.0.1', '6082', 1.0], 'secret-1'),
            call(['127.0.0.2', '6083', 1.0], 'secret-2'),
            call(['127.0.0.3', '6084', 1.0], 'secret-3')]
        sample_extractor = Mock(servers=servers)

        api_init_side_effect = {
            'secret-1': Exception(),
            'secret-2': None,
            'secret-3': None
        }

        with patch('vaas.cluster.cluster.ServerExtractor', Mock(return_value=sample_extractor)):
            with patch.object(
                VarnishApi, '__init__', side_effect=lambda host_port_timeout, secret: api_init_side_effect[secret]
            ) as construct_mock:
                with patch('telnetlib.Telnet.close', Mock()):
                    varnish_cluster = VarnishApiProvider()
                    api_objects = []
                    for api in varnish_cluster.get_connected_varnish_api():
                        """
                        Workaround - we cannot mock __del__ method:
                        https://docs.python.org/3/library/unittest.mock.html

                        We inject sock field to eliminate warning raised by cleaning actions in __del__ method
                        """
                        api.sock = None
                        api_objects.append(api)

                    assert_equals(2, len(api_objects))
                    assert_list_equal(expected_construct_args, construct_mock.call_args_list)
Exemplo n.º 4
0
Arquivo: admin.py Projeto: bieli/vaas
class VarnishServerAdmin(admin.ModelAdmin):
    search_fields = ['dc__symbol', 'ip', 'hostname', 'template__name']
    list_display = (
        'hostname',
        'ip',
        'port',
        'dc',
        'cluster',
        'cluster_weight',
        'template',
        'custom_enabled',
        'is_connected',
        'vcl'
    )
    actions = [enable_varnish_servers, disable_varnish_servers]
    varnish_api_provider = None

    def get_list_display(self, request):
        self.varnish_api_provider = VarnishApiProvider()
        return super(VarnishServerAdmin, self).get_list_display(request)

    def custom_enabled(self, obj):
        if obj.enabled:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini btn-success' href='#'>" +
                "<i class='icon-ok-circle'></i></a>" +
                "</div>"
            )
        else:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini' href='#'><i class='icon-ban-circle'></i></a>" +
                "</div>"
            )
    custom_enabled.short_description = 'Enabled'

    def is_connected(self, obj):
        try:
            self.varnish_api_provider.get_api(obj)
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini btn-success' href='#'><i class='icon-ok'></i></a>" +
                "</div>"
            )
        except:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini btn-danger' href='#'><i class='icon-off'></i></a>" +
                "</div>"
            )

    def vcl(self, obj):
        return format_html(
            ("<div class='span13 text-center'>" +
             "<button class='btn btn-success' data-remote='/manager/varnish/vcl/%s/' " +
             "data-toggle='modal' data-target='#vclModal'>Show vcl</button>" +
             "</div>") % obj.id
        )
Exemplo n.º 5
0
class VarnishServerAdmin(admin.ModelAdmin):
    search_fields = ['dc__symbol', 'ip', 'hostname', 'template__name']
    list_filter = ['cluster__name']
    list_display = ('hostname', 'ip', 'port', 'dc', 'cluster',
                    'cluster_weight', 'template', 'template_version',
                    'custom_enabled', 'is_connected', 'vcl')
    actions = [enable_varnish_servers, disable_varnish_servers]
    varnish_api_provider = None

    def get_list_display(self, request):
        self.varnish_api_provider = VarnishApiProvider()
        return super(VarnishServerAdmin, self).get_list_display(request)

    def template_version(self, obj):
        return obj.template.get_template_version()

    def custom_enabled(self, obj):
        if obj.enabled:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini btn-success' href='#'>" +
                "<i class='icon-ok-circle'></i></a>" + "</div>")
        else:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini' href='#'><i class='icon-ban-circle'></i></a>"
                + "</div>")

    custom_enabled.short_description = 'Enabled'

    def is_connected(self, obj):
        try:
            self.varnish_api_provider.get_api(obj)
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini btn-success' href='#'><i class='icon-ok'></i></a>"
                + "</div>")
        except:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-mini btn-danger' href='#'><i class='icon-off'></i></a>"
                + "</div>")

    def vcl(self, obj):
        return format_html((
            "<div class='span13 text-center'>" +
            "<button class='btn btn-success' data-remote='/manager/varnish/vcl/%s/' "
            + "data-toggle='modal' data-target='#vclModal'>Show vcl</button>" +
            "</div>") % obj.id)
Exemplo n.º 6
0
    def test_should_create_varnish_api_clients_for_all_servers(self):
        expected_construct_args = [
            call(['127.0.0.1', '6082', 1.0], 'secret-1'),
            call(['127.0.0.2', '6083', 1.0], 'secret-2'),
            call(['127.0.0.3', '6084', 1.0], 'secret-3')]
        sample_extractor = Mock(servers=servers)

        with patch('vaas.cluster.cluster.ServerExtractor', Mock(return_value=sample_extractor)):
                with patch.object(VarnishApi, '__init__', return_value=None) as construct_mock:
                    varnish_cluster = VarnishApiProvider()
                    api_objects = []
                    for api in varnish_cluster.get_varnish_api():
                        """
                        Workaround - we cannot mock __del__ method:
                        https://docs.python.org/3/library/unittest.mock.html

                        We inject sock field to eliminate warning raised by cleaning actions in __del__ method
                        """
                        api.sock = None
                        api_objects.append(api)

                    assert_equals(3, len(api_objects))
                    assert_list_equal(expected_construct_args, construct_mock.call_args_list)
Exemplo n.º 7
0
    def test_should_create_varnish_api_for_connected_servers(self):
        expected_construct_args = [
            call(['127.0.0.1', '6082', 1.0], 'secret-1'),
            call(['127.0.0.2', '6083', 1.0], 'secret-2'),
            call(['127.0.0.3', '6084', 1.0], 'secret-3')
        ]
        sample_extractor = Mock(servers=servers)

        api_init_side_effect = {
            'secret-1': Exception(),
            'secret-2': None,
            'secret-3': None
        }

        with patch('vaas.cluster.cluster.ServerExtractor',
                   Mock(return_value=sample_extractor)):
            with patch.object(VarnishApi,
                              '__init__',
                              side_effect=lambda host_port_timeout, secret:
                              api_init_side_effect[secret]) as construct_mock:
                with patch('telnetlib.Telnet.close', Mock()):
                    varnish_cluster = VarnishApiProvider()
                    api_objects = []
                    for api in varnish_cluster.get_connected_varnish_api():
                        """
                        Workaround - we cannot mock __del__ method:
                        https://docs.python.org/3/library/unittest.mock.html

                        We inject sock field to eliminate warning raised by cleaning actions in __del__ method
                        """
                        api.sock = None
                        api_objects.append(api)

                    assert_equals(2, len(api_objects))
                    assert_list_equal(expected_construct_args,
                                      construct_mock.call_args_list)
Exemplo n.º 8
0
class BackendStatusManager(object):
    def __init__(self):
        self.varnish_api_provider = VarnishApiProvider()
        self.logger = logging.getLogger('vaas')
        self.timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)

    def load_from_varnish(self):
        pattern = re.compile("^((?:.*_){5}[^(\s]*)")
        backend_to_status_map = {}
        backends = {x.pk: "{}:{}".format(x.address, x.port) for x in Backend.objects.all()}

        try:
            for varnish_api in self.varnish_api_provider.get_connected_varnish_api():
                backend_statuses = map(lambda x: x.split(), varnish_api.fetch('backend.list')[1][0:].split('\n'))

                for backend_status in backend_statuses:
                    if len(backend_status):
                        backend = re.search(pattern, backend_status[0])

                        if backend is not None:
                            backend_id = int(backend.group(1).split('_')[-5])
                            status = backend_status[-2]
                            if backend_id not in backend_to_status_map or status == 'Sick':
                                backend_address = backends.get(backend_id)
                                if backend_address is not None:
                                    backend_to_status_map[backend_address] = status

        except VclLoadException as e:
            self.logger.warning("Some backends' status could not be refreshed: %s" % e)

        return backend_to_status_map

    def store_backend_statuses(self, backend_to_status_map):
        for key, status in backend_to_status_map.items():
            address, port = key.split(":")
            try:
                backend_status = BackendStatus.objects.get(address=address, port=port)
                if backend_status.timestamp < self.timestamp:
                    backend_status.status = status
                    backend_status.timestamp = self.timestamp
                    backend_status.save()
            except BackendStatus.DoesNotExist:
                BackendStatus.objects.create(address=address, port=port, status=status, timestamp=self.timestamp)

        BackendStatus.objects.filter(timestamp__lt=self.timestamp).delete()

    def refresh_statuses(self):
        self.store_backend_statuses(self.load_from_varnish())
Exemplo n.º 9
0
Arquivo: health.py Projeto: bieli/vaas
class BackendStatusManager(object):
    def __init__(self):
        self.varnish_api_provider = VarnishApiProvider()

    def load_from_varnish(self):
        pattern = re.compile(
            "\((\d+\.\d+\.\d+\.\d+),[^,]*,(\d+)\)\s+\w+\s+\w+\s+(\w+)")
        backend_to_status_map = {}

        for varnish_api in self.varnish_api_provider.get_varnish_api():
            backend_statuses = varnish_api.fetch('backend.list')[1][0:].split(
                '\n')

            for backend in backend_statuses:
                ips = re.search(pattern, backend)
                if ips is not None:
                    backend_address = str(ips.group(1)) + ':' + str(
                        ips.group(2))

                    if backend_address not in backend_to_status_map or ips.group(
                            3) == 'Sick':
                        backend_to_status_map[backend_address] = ips.group(3)

        return backend_to_status_map

    def store_backend_statuses(self, backend_to_status_map):
        now = datetime.datetime.utcnow().replace(tzinfo=utc)
        for key, status in backend_to_status_map.items():
            address, port = key.split(":")
            try:
                backend_status = BackendStatus.objects.get(address=address,
                                                           port=port)
                backend_status.status = status
                backend_status.timestamp = now
                backend_status.save()
            except BackendStatus.DoesNotExist:
                BackendStatus.objects.create(address=address,
                                             port=port,
                                             status=status,
                                             timestamp=now)

        BackendStatus.objects.filter(timestamp__lt=now).delete()

    def refresh_statuses(self):
        self.store_backend_statuses(self.load_from_varnish())
Exemplo n.º 10
0
Arquivo: health.py Projeto: bieli/vaas
class BackendStatusManager(object):
    def __init__(self):
        self.varnish_api_provider = VarnishApiProvider()

    def load_from_varnish(self):
        pattern = re.compile("\((\d+\.\d+\.\d+\.\d+),[^,]*,(\d+)\)\s+\w+\s+\w+\s+(\w+)")
        backend_to_status_map = {}

        for varnish_api in self.varnish_api_provider.get_varnish_api():
            backend_statuses = varnish_api.fetch('backend.list')[1][0:].split('\n')

            for backend in backend_statuses:
                ips = re.search(pattern, backend)
                if ips is not None:
                    backend_address = str(ips.group(1)) + ':' + str(ips.group(2))

                    if backend_address not in backend_to_status_map or ips.group(3) == 'Sick':
                        backend_to_status_map[backend_address] = ips.group(3)

        return backend_to_status_map

    def store_backend_statuses(self, backend_to_status_map):
        now = datetime.datetime.utcnow().replace(tzinfo=utc)
        for key, status in backend_to_status_map.items():
            address, port = key.split(":")
            try:
                backend_status = BackendStatus.objects.get(address=address, port=port)
                backend_status.status = status
                backend_status.timestamp = now
                backend_status.save()
            except BackendStatus.DoesNotExist:
                BackendStatus.objects.create(address=address, port=port, status=status, timestamp=now)

        BackendStatus.objects.filter(timestamp__lt=now).delete()

    def refresh_statuses(self):
        self.store_backend_statuses(self.load_from_varnish())
Exemplo n.º 11
0
 def get_list_display(self, request):
     self.varnish_api_provider = VarnishApiProvider()
     return super(VarnishServerAdmin, self).get_list_display(request)
Exemplo n.º 12
0
 def get_list_display(self, request):
     self.varnish_api_provider = VarnishApiProvider()
     return super(VarnishServerAdmin, self).get_list_display(request)
Exemplo n.º 13
0
class VarnishServerAdmin(admin.ModelAdmin):
    search_fields = ['dc__symbol', 'ip', 'hostname', 'template__name']
    list_filter = ['cluster__name', OutdatedFilter]
    list_display = (
        'hostname',
        'ip',
        'port',
        'http_port',
        'dc',
        'cluster',
        'cluster_weight',
        'template',
        'template_version',
        'custom_enabled',
        'is_connected',
        'custom_is_canary',
        'vcl'
    )
    actions = [enable_varnish_servers, maintenance_varnish_servers, disable_varnish_servers]
    varnish_api_provider = None

    def get_list_display(self, request):
        self.varnish_api_provider = VarnishApiProvider()
        return super(VarnishServerAdmin, self).get_list_display(request)

    def template_version(self, obj):
        return obj.template.get_template_version()

    def custom_enabled(self, obj):
        if obj.status == 'active':
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs btn-success' href='#'>" +
                "<i class='glyphicon glyphicon-ok-sign'></i></a>" +
                "</div>"
            )
        elif obj.status == 'maintenance':
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs btn-warning' href='#'>" +
                "<i class='glyphicon glyphicon-wrench'></i></a>" +
                "</div>"
            )
        else:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs' href='#'><i class='glyphicon glyphicon-ban-circle'></i></a>" +
                "</div>"
            )
    custom_enabled.short_description = 'Enabled'

    def is_connected(self, obj):
        if obj.status == 'active':
            try:
                self.varnish_api_provider.get_api(obj)
                return format_html(
                    "<div class='span13 text-center'>" +
                    "<a class='btn btn-xs btn-success' href='#'><i class='glyphicon glyphicon-ok'></i></a>" +
                    "</div>"
                )
            except:
                return format_html(
                    "<div class='span13 text-center'>" +
                    "<a class='btn btn-xs btn-danger' href='#'><i class='glyphicon glyphicon-off'></i></a>" +
                    "</div>"
                )
        elif obj.status == 'maintenance':
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs btn-warning' href='#'>" +
                "<i class='glyphicon glyphicon-wrench'></i></a>" +
                "</div>"
            )
        else:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs' href='#'><i class='glyphicon glyphicon-ban-circle'></i></a>" +
                "</div>"
            )

    def custom_is_canary(self, obj):
        if obj.is_canary:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs btn-success' href='#'>" +
                "<i class='glyphicon glyphicon-ok-sign'></i></a>" +
                "</div>"
            )
        else:
            return format_html(
                "<div class='span13 text-center'>" +
                "<a class='btn btn-xs' href='#'><i class='glyphicon glyphicon-ban-circle'></i></a>" +
                "</div>"
            )
    custom_is_canary.short_description = 'Canary'

    def vcl(self, obj):
        if obj.status in ('active', 'maintenance'):
            return format_html(
                ("<div class='span13 text-center'>" +
                 "<button type='button' class='btn btn-success' data-toggle='modal' " +
                 "data-vcl='/manager/varnish/vcl/%s/'" +
                 "data-target='#vclModal'>Show vcl</button>" +
                 "</div>") % obj.id
            )
        else:
            return format_html(
                ("<div class='span13 text-center'>" +
                 "<button class='btn btn-danger' disabled>Show vcl</button>" +
                 "</div>")
            )
Exemplo n.º 14
0
Arquivo: health.py Projeto: bieli/vaas
 def __init__(self):
     self.varnish_api_provider = VarnishApiProvider()
Exemplo n.º 15
0
 def test_should_throw_vcl_load_exception_on_any_error_while_connecting_to_varnish_api(self):
     with patch.object(VarnishApi, '__init__', side_effect=Exception):
         with patch.object(VarnishApi, '__del__'):
             varnish_cluster = VarnishApiProvider()
             with self.assertRaises(VclLoadException):
                 varnish_cluster.get_api(servers[0])
Exemplo n.º 16
0
 def __init__(self):
     self.varnish_api_provider = VarnishApiProvider()
     self.logger = logging.getLogger('vaas')
Exemplo n.º 17
0
class BackendStatusManager(object):
    def __init__(self):
        self.varnish_api_provider = VarnishApiProvider()
        self.logger = logging.getLogger('vaas')
        self.timestamp = datetime.datetime.utcnow().replace(tzinfo=utc,
                                                            microsecond=0)

    def load_from_varnish(self):
        pattern = re.compile("^((?:.*_){5}[^(\s]*)")
        backend_to_status_map = {}
        backends = {
            x.pk: "{}:{}".format(x.address, x.port)
            for x in Backend.objects.all()
        }

        try:
            for varnish_api in self.varnish_api_provider.get_connected_varnish_api(
            ):
                backend_statuses = map(
                    lambda x: x.split(),
                    varnish_api.fetch('backend.list')[1][0:].split('\n'))

                for backend_status in backend_statuses:
                    if len(backend_status):
                        backend = re.search(pattern, backend_status[0])
                        if backend is not None:
                            backend_id = None
                            regex_result = re.findall(
                                BaseHelpers.dynamic_regex_with_datacenters(),
                                backend.group(1))

                            if len(regex_result) == 1:
                                try:
                                    backend_id = int(regex_result[0][0])
                                except ValueError:
                                    self.logger.error(
                                        'Mapping backend id failed. Expected parsable string to int, got {}'
                                        .format(regex_result[0][0]))
                            else:
                                self.logger.error(
                                    'Regex patterns matches for possible backend id: {} '
                                    .format(regex_result))

                            status = backend_status[-2]
                            if backend_id and backend_id not in backend_to_status_map or status == 'Sick':
                                backend_address = backends.get(backend_id)
                                if backend_address is not None:
                                    backend_to_status_map[
                                        backend_address] = status

        except VclLoadException as e:
            self.logger.warning(
                "Some backends' status could not be refreshed: %s" % e)

        return backend_to_status_map

    def store_backend_statuses(self, backend_to_status_map):
        for key, status in backend_to_status_map.items():
            address, port = key.split(":")
            try:
                backend_status = BackendStatus.objects.get(address=address,
                                                           port=port)
                if backend_status.timestamp < self.timestamp:
                    backend_status.status = status
                    backend_status.timestamp = self.timestamp
                    backend_status.save()
            except BackendStatus.DoesNotExist:
                BackendStatus.objects.create(address=address,
                                             port=port,
                                             status=status,
                                             timestamp=self.timestamp)

        BackendStatus.objects.filter(timestamp__lt=self.timestamp).delete()

    def refresh_statuses(self):
        self.store_backend_statuses(self.load_from_varnish())
Exemplo n.º 18
0
Arquivo: health.py Projeto: bieli/vaas
 def __init__(self):
     self.varnish_api_provider = VarnishApiProvider()
Exemplo n.º 19
0
 def __init__(self):
     self.varnish_api_provider = VarnishApiProvider()
     self.logger = logging.getLogger('vaas')
     self.timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)
Exemplo n.º 20
0
def provide_backend_status_manager():
    return BackendStatusManager(VarnishApiProvider(),
                                ServerExtractor().servers,
                                settings.VAAS_GATHER_STATUSES_CONNECT_TIMEOUT,
                                settings.VAAS_GATHER_STATUSES_MAX_WORKERS)