def test_should_throw_vcl_load_exception_on_any_error_while_connecting_to_varnish_api(
self):
with patch.object(VarnishApi, '__init__', side_effect=Exception):
with patch.object(VarnishApi, '__del__'):
varnish_cluster = VarnishApiProvider()
with self.assertRaises(VclLoadException):
varnish_cluster.get_api(servers[0])
def test_should_create_varnish_api_clients_for_all_servers(self):
expected_construct_args = [
call(['127.0.0.1', '6082', 1.0], 'secret-1'),
call(['127.0.0.2', '6083', 1.0], 'secret-2'),
call(['127.0.0.3', '6084', 1.0], 'secret-3')
]
sample_extractor = Mock(servers=servers)
with patch('vaas.cluster.cluster.ServerExtractor',
Mock(return_value=sample_extractor)):
with patch.object(VarnishApi, '__init__',
return_value=None) as construct_mock:
varnish_cluster = VarnishApiProvider()
api_objects = []
for api in varnish_cluster.get_varnish_api():
"""
Workaround - we cannot mock __del__ method:
https://docs.python.org/3/library/unittest.mock.html
We inject sock field to eliminate warning raised by cleaning actions in __del__ method
"""
api.sock = None
api_objects.append(api)
assert_equals(3, len(api_objects))
assert_list_equal(expected_construct_args,
construct_mock.call_args_list)
def test_should_create_varnish_api_for_connected_servers(self):
expected_construct_args = [
call(['127.0.0.1', '6082', 1.0], 'secret-1'),
call(['127.0.0.2', '6083', 1.0], 'secret-2'),
call(['127.0.0.3', '6084', 1.0], 'secret-3')]
sample_extractor = Mock(servers=servers)
api_init_side_effect = {
'secret-1': Exception(),
'secret-2': None,
'secret-3': None
}
with patch('vaas.cluster.cluster.ServerExtractor', Mock(return_value=sample_extractor)):
with patch.object(
VarnishApi, '__init__', side_effect=lambda host_port_timeout, secret: api_init_side_effect[secret]
) as construct_mock:
with patch('telnetlib.Telnet.close', Mock()):
varnish_cluster = VarnishApiProvider()
api_objects = []
for api in varnish_cluster.get_connected_varnish_api():
"""
Workaround - we cannot mock __del__ method:
https://docs.python.org/3/library/unittest.mock.html
We inject sock field to eliminate warning raised by cleaning actions in __del__ method
"""
api.sock = None
api_objects.append(api)
assert_equals(2, len(api_objects))
assert_list_equal(expected_construct_args, construct_mock.call_args_list)
class VarnishServerAdmin(admin.ModelAdmin):
search_fields = ['dc__symbol', 'ip', 'hostname', 'template__name']
list_display = (
'hostname',
'ip',
'port',
'dc',
'cluster',
'cluster_weight',
'template',
'custom_enabled',
'is_connected',
'vcl'
)
actions = [enable_varnish_servers, disable_varnish_servers]
varnish_api_provider = None
def get_list_display(self, request):
self.varnish_api_provider = VarnishApiProvider()
return super(VarnishServerAdmin, self).get_list_display(request)
def custom_enabled(self, obj):
if obj.enabled:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini btn-success' href='#'>" +
"<i class='icon-ok-circle'></i></a>" +
"</div>"
)
else:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini' href='#'><i class='icon-ban-circle'></i></a>" +
"</div>"
)
custom_enabled.short_description = 'Enabled'
def is_connected(self, obj):
try:
self.varnish_api_provider.get_api(obj)
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini btn-success' href='#'><i class='icon-ok'></i></a>" +
"</div>"
)
except:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini btn-danger' href='#'><i class='icon-off'></i></a>" +
"</div>"
)
def vcl(self, obj):
return format_html(
("<div class='span13 text-center'>" +
"<button class='btn btn-success' data-remote='/manager/varnish/vcl/%s/' " +
"data-toggle='modal' data-target='#vclModal'>Show vcl</button>" +
"</div>") % obj.id
)
class VarnishServerAdmin(admin.ModelAdmin):
search_fields = ['dc__symbol', 'ip', 'hostname', 'template__name']
list_filter = ['cluster__name']
list_display = ('hostname', 'ip', 'port', 'dc', 'cluster',
'cluster_weight', 'template', 'template_version',
'custom_enabled', 'is_connected', 'vcl')
actions = [enable_varnish_servers, disable_varnish_servers]
varnish_api_provider = None
def get_list_display(self, request):
self.varnish_api_provider = VarnishApiProvider()
return super(VarnishServerAdmin, self).get_list_display(request)
def template_version(self, obj):
return obj.template.get_template_version()
def custom_enabled(self, obj):
if obj.enabled:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini btn-success' href='#'>" +
"<i class='icon-ok-circle'></i></a>" + "</div>")
else:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini' href='#'><i class='icon-ban-circle'></i></a>"
+ "</div>")
custom_enabled.short_description = 'Enabled'
def is_connected(self, obj):
try:
self.varnish_api_provider.get_api(obj)
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini btn-success' href='#'><i class='icon-ok'></i></a>"
+ "</div>")
except:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-mini btn-danger' href='#'><i class='icon-off'></i></a>"
+ "</div>")
def vcl(self, obj):
return format_html((
"<div class='span13 text-center'>" +
"<button class='btn btn-success' data-remote='/manager/varnish/vcl/%s/' "
+ "data-toggle='modal' data-target='#vclModal'>Show vcl</button>" +
"</div>") % obj.id)
def test_should_create_varnish_api_clients_for_all_servers(self):
expected_construct_args = [
call(['127.0.0.1', '6082', 1.0], 'secret-1'),
call(['127.0.0.2', '6083', 1.0], 'secret-2'),
call(['127.0.0.3', '6084', 1.0], 'secret-3')]
sample_extractor = Mock(servers=servers)
with patch('vaas.cluster.cluster.ServerExtractor', Mock(return_value=sample_extractor)):
with patch.object(VarnishApi, '__init__', return_value=None) as construct_mock:
varnish_cluster = VarnishApiProvider()
api_objects = []
for api in varnish_cluster.get_varnish_api():
"""
Workaround - we cannot mock __del__ method:
https://docs.python.org/3/library/unittest.mock.html
We inject sock field to eliminate warning raised by cleaning actions in __del__ method
"""
api.sock = None
api_objects.append(api)
assert_equals(3, len(api_objects))
assert_list_equal(expected_construct_args, construct_mock.call_args_list)
def test_should_create_varnish_api_for_connected_servers(self):
expected_construct_args = [
call(['127.0.0.1', '6082', 1.0], 'secret-1'),
call(['127.0.0.2', '6083', 1.0], 'secret-2'),
call(['127.0.0.3', '6084', 1.0], 'secret-3')
]
sample_extractor = Mock(servers=servers)
api_init_side_effect = {
'secret-1': Exception(),
'secret-2': None,
'secret-3': None
}
with patch('vaas.cluster.cluster.ServerExtractor',
Mock(return_value=sample_extractor)):
with patch.object(VarnishApi,
'__init__',
side_effect=lambda host_port_timeout, secret:
api_init_side_effect[secret]) as construct_mock:
with patch('telnetlib.Telnet.close', Mock()):
varnish_cluster = VarnishApiProvider()
api_objects = []
for api in varnish_cluster.get_connected_varnish_api():
"""
Workaround - we cannot mock __del__ method:
https://docs.python.org/3/library/unittest.mock.html
We inject sock field to eliminate warning raised by cleaning actions in __del__ method
"""
api.sock = None
api_objects.append(api)
assert_equals(2, len(api_objects))
assert_list_equal(expected_construct_args,
construct_mock.call_args_list)
class BackendStatusManager(object):
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
self.logger = logging.getLogger('vaas')
self.timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)
def load_from_varnish(self):
pattern = re.compile("^((?:.*_){5}[^(\s]*)")
backend_to_status_map = {}
backends = {x.pk: "{}:{}".format(x.address, x.port) for x in Backend.objects.all()}
try:
for varnish_api in self.varnish_api_provider.get_connected_varnish_api():
backend_statuses = map(lambda x: x.split(), varnish_api.fetch('backend.list')[1][0:].split('\n'))
for backend_status in backend_statuses:
if len(backend_status):
backend = re.search(pattern, backend_status[0])
if backend is not None:
backend_id = int(backend.group(1).split('_')[-5])
status = backend_status[-2]
if backend_id not in backend_to_status_map or status == 'Sick':
backend_address = backends.get(backend_id)
if backend_address is not None:
backend_to_status_map[backend_address] = status
except VclLoadException as e:
self.logger.warning("Some backends' status could not be refreshed: %s" % e)
return backend_to_status_map
def store_backend_statuses(self, backend_to_status_map):
for key, status in backend_to_status_map.items():
address, port = key.split(":")
try:
backend_status = BackendStatus.objects.get(address=address, port=port)
if backend_status.timestamp < self.timestamp:
backend_status.status = status
backend_status.timestamp = self.timestamp
backend_status.save()
except BackendStatus.DoesNotExist:
BackendStatus.objects.create(address=address, port=port, status=status, timestamp=self.timestamp)
BackendStatus.objects.filter(timestamp__lt=self.timestamp).delete()
def refresh_statuses(self):
self.store_backend_statuses(self.load_from_varnish())
class BackendStatusManager(object):
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
def load_from_varnish(self):
pattern = re.compile(
"\((\d+\.\d+\.\d+\.\d+),[^,]*,(\d+)\)\s+\w+\s+\w+\s+(\w+)")
backend_to_status_map = {}
for varnish_api in self.varnish_api_provider.get_varnish_api():
backend_statuses = varnish_api.fetch('backend.list')[1][0:].split(
'\n')
for backend in backend_statuses:
ips = re.search(pattern, backend)
if ips is not None:
backend_address = str(ips.group(1)) + ':' + str(
ips.group(2))
if backend_address not in backend_to_status_map or ips.group(
3) == 'Sick':
backend_to_status_map[backend_address] = ips.group(3)
return backend_to_status_map
def store_backend_statuses(self, backend_to_status_map):
now = datetime.datetime.utcnow().replace(tzinfo=utc)
for key, status in backend_to_status_map.items():
address, port = key.split(":")
try:
backend_status = BackendStatus.objects.get(address=address,
port=port)
backend_status.status = status
backend_status.timestamp = now
backend_status.save()
except BackendStatus.DoesNotExist:
BackendStatus.objects.create(address=address,
port=port,
status=status,
timestamp=now)
BackendStatus.objects.filter(timestamp__lt=now).delete()
def refresh_statuses(self):
self.store_backend_statuses(self.load_from_varnish())
class BackendStatusManager(object):
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
def load_from_varnish(self):
pattern = re.compile("\((\d+\.\d+\.\d+\.\d+),[^,]*,(\d+)\)\s+\w+\s+\w+\s+(\w+)")
backend_to_status_map = {}
for varnish_api in self.varnish_api_provider.get_varnish_api():
backend_statuses = varnish_api.fetch('backend.list')[1][0:].split('\n')
for backend in backend_statuses:
ips = re.search(pattern, backend)
if ips is not None:
backend_address = str(ips.group(1)) + ':' + str(ips.group(2))
if backend_address not in backend_to_status_map or ips.group(3) == 'Sick':
backend_to_status_map[backend_address] = ips.group(3)
return backend_to_status_map
def store_backend_statuses(self, backend_to_status_map):
now = datetime.datetime.utcnow().replace(tzinfo=utc)
for key, status in backend_to_status_map.items():
address, port = key.split(":")
try:
backend_status = BackendStatus.objects.get(address=address, port=port)
backend_status.status = status
backend_status.timestamp = now
backend_status.save()
except BackendStatus.DoesNotExist:
BackendStatus.objects.create(address=address, port=port, status=status, timestamp=now)
BackendStatus.objects.filter(timestamp__lt=now).delete()
def refresh_statuses(self):
self.store_backend_statuses(self.load_from_varnish())
def get_list_display(self, request):
self.varnish_api_provider = VarnishApiProvider()
return super(VarnishServerAdmin, self).get_list_display(request)
def get_list_display(self, request):
self.varnish_api_provider = VarnishApiProvider()
return super(VarnishServerAdmin, self).get_list_display(request)
class VarnishServerAdmin(admin.ModelAdmin):
search_fields = ['dc__symbol', 'ip', 'hostname', 'template__name']
list_filter = ['cluster__name', OutdatedFilter]
list_display = (
'hostname',
'ip',
'port',
'http_port',
'dc',
'cluster',
'cluster_weight',
'template',
'template_version',
'custom_enabled',
'is_connected',
'custom_is_canary',
'vcl'
)
actions = [enable_varnish_servers, maintenance_varnish_servers, disable_varnish_servers]
varnish_api_provider = None
def get_list_display(self, request):
self.varnish_api_provider = VarnishApiProvider()
return super(VarnishServerAdmin, self).get_list_display(request)
def template_version(self, obj):
return obj.template.get_template_version()
def custom_enabled(self, obj):
if obj.status == 'active':
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs btn-success' href='#'>" +
"<i class='glyphicon glyphicon-ok-sign'></i></a>" +
"</div>"
)
elif obj.status == 'maintenance':
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs btn-warning' href='#'>" +
"<i class='glyphicon glyphicon-wrench'></i></a>" +
"</div>"
)
else:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs' href='#'><i class='glyphicon glyphicon-ban-circle'></i></a>" +
"</div>"
)
custom_enabled.short_description = 'Enabled'
def is_connected(self, obj):
if obj.status == 'active':
try:
self.varnish_api_provider.get_api(obj)
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs btn-success' href='#'><i class='glyphicon glyphicon-ok'></i></a>" +
"</div>"
)
except:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs btn-danger' href='#'><i class='glyphicon glyphicon-off'></i></a>" +
"</div>"
)
elif obj.status == 'maintenance':
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs btn-warning' href='#'>" +
"<i class='glyphicon glyphicon-wrench'></i></a>" +
"</div>"
)
else:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs' href='#'><i class='glyphicon glyphicon-ban-circle'></i></a>" +
"</div>"
)
def custom_is_canary(self, obj):
if obj.is_canary:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs btn-success' href='#'>" +
"<i class='glyphicon glyphicon-ok-sign'></i></a>" +
"</div>"
)
else:
return format_html(
"<div class='span13 text-center'>" +
"<a class='btn btn-xs' href='#'><i class='glyphicon glyphicon-ban-circle'></i></a>" +
"</div>"
)
custom_is_canary.short_description = 'Canary'
def vcl(self, obj):
if obj.status in ('active', 'maintenance'):
return format_html(
("<div class='span13 text-center'>" +
"<button type='button' class='btn btn-success' data-toggle='modal' " +
"data-vcl='/manager/varnish/vcl/%s/'" +
"data-target='#vclModal'>Show vcl</button>" +
"</div>") % obj.id
)
else:
return format_html(
("<div class='span13 text-center'>" +
"<button class='btn btn-danger' disabled>Show vcl</button>" +
"</div>")
)
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
def test_should_throw_vcl_load_exception_on_any_error_while_connecting_to_varnish_api(self):
with patch.object(VarnishApi, '__init__', side_effect=Exception):
with patch.object(VarnishApi, '__del__'):
varnish_cluster = VarnishApiProvider()
with self.assertRaises(VclLoadException):
varnish_cluster.get_api(servers[0])
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
self.logger = logging.getLogger('vaas')
class BackendStatusManager(object):
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
self.logger = logging.getLogger('vaas')
self.timestamp = datetime.datetime.utcnow().replace(tzinfo=utc,
microsecond=0)
def load_from_varnish(self):
pattern = re.compile("^((?:.*_){5}[^(\s]*)")
backend_to_status_map = {}
backends = {
x.pk: "{}:{}".format(x.address, x.port)
for x in Backend.objects.all()
}
try:
for varnish_api in self.varnish_api_provider.get_connected_varnish_api(
):
backend_statuses = map(
lambda x: x.split(),
varnish_api.fetch('backend.list')[1][0:].split('\n'))
for backend_status in backend_statuses:
if len(backend_status):
backend = re.search(pattern, backend_status[0])
if backend is not None:
backend_id = None
regex_result = re.findall(
BaseHelpers.dynamic_regex_with_datacenters(),
backend.group(1))
if len(regex_result) == 1:
try:
backend_id = int(regex_result[0][0])
except ValueError:
self.logger.error(
'Mapping backend id failed. Expected parsable string to int, got {}'
.format(regex_result[0][0]))
else:
self.logger.error(
'Regex patterns matches for possible backend id: {} '
.format(regex_result))
status = backend_status[-2]
if backend_id and backend_id not in backend_to_status_map or status == 'Sick':
backend_address = backends.get(backend_id)
if backend_address is not None:
backend_to_status_map[
backend_address] = status
except VclLoadException as e:
self.logger.warning(
"Some backends' status could not be refreshed: %s" % e)
return backend_to_status_map
def store_backend_statuses(self, backend_to_status_map):
for key, status in backend_to_status_map.items():
address, port = key.split(":")
try:
backend_status = BackendStatus.objects.get(address=address,
port=port)
if backend_status.timestamp < self.timestamp:
backend_status.status = status
backend_status.timestamp = self.timestamp
backend_status.save()
except BackendStatus.DoesNotExist:
BackendStatus.objects.create(address=address,
port=port,
status=status,
timestamp=self.timestamp)
BackendStatus.objects.filter(timestamp__lt=self.timestamp).delete()
def refresh_statuses(self):
self.store_backend_statuses(self.load_from_varnish())
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
def __init__(self):
self.varnish_api_provider = VarnishApiProvider()
self.logger = logging.getLogger('vaas')
self.timestamp = datetime.datetime.utcnow().replace(tzinfo=utc)
def provide_backend_status_manager():
return BackendStatusManager(VarnishApiProvider(),
ServerExtractor().servers,
settings.VAAS_GATHER_STATUSES_CONNECT_TIMEOUT,
settings.VAAS_GATHER_STATUSES_MAX_WORKERS)
