def synapse_replication_is_low(service, instance, system_paasta_config,
local_backends):
crit_threshold = 80
reg_svc, reg_namespace, _, __ = utils.decompose_job_id(
read_registration_for_service_instance(service=service,
instance=instance))
# We only actually care about the replication of where we're registering
service, namespace = reg_svc, reg_namespace
smartstack_replication_info = load_smartstack_info_for_service(
service=service,
namespace=namespace,
blacklist=[],
system_paasta_config=system_paasta_config,
)
expected_count = get_expected_instance_count_for_namespace(
service=service, namespace=namespace)
expected_count_per_location = int(expected_count /
len(smartstack_replication_info))
synapse_name = utils.compose_job_id(service, namespace)
local_replication = get_replication_for_services(
synapse_host=system_paasta_config.get_default_synapse_host(),
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.
get_synapse_haproxy_url_format(),
services=[synapse_name],
)
num_available = local_replication.get(synapse_name, 0)
under_replicated, ratio = utils.is_under_replicated(
num_available, expected_count_per_location, crit_threshold)
log.info('Service %s.%s has %d out of %d expected instances' %
(service, instance, num_available, expected_count_per_location))
return under_replicated
def get_service_replication(self, all_services, synapse_host, synapse_port,
synapse_haproxy_url_format):
# Get the replication data once for performance
synapse_host_port = "%s:%s" % (synapse_host, synapse_port)
self.log.debug("Gathering replication information from {0}".format(
synapse_host_port))
service_replication = {}
try:
service_replication = get_replication_for_services(
synapse_host, synapse_port, synapse_haproxy_url_format,
['%s.main' % name for name in all_services])
except requests.exceptions.ConnectionError:
self.log.error('Failed to connect synapse haproxy on {0}'.format(
synapse_host_port))
self.critical('Failed to connect synapse haproxy on {0}'.format(
synapse_host_port))
except Exception as e:
self.log.error(
'Unable to collect replication information on {0}: {1}'.format(
synapse_host_port, e.message))
self.critical(
'Unable to collect replication information: {0}'.format(
e.message))
self.log.debug(
"Finished gathering replication information from {0}".format(
synapse_host_port))
return service_replication
def synapse_replication_is_low(service, instance, system_paasta_config, local_backends):
crit_threshold = 80
reg_svc, reg_namespace, _, __ = utils.decompose_job_id(
read_registration_for_service_instance(
service=service, instance=instance
)
)
# We only actually care about the replication of where we're registering
service, namespace = reg_svc, reg_namespace
smartstack_replication_info = load_smartstack_info_for_service(
service=service,
namespace=namespace,
blacklist=[],
system_paasta_config=system_paasta_config,
)
expected_count = get_expected_instance_count_for_namespace(service=service, namespace=namespace)
expected_count_per_location = int(expected_count / len(smartstack_replication_info))
synapse_name = utils.compose_job_id(service, namespace)
local_replication = get_replication_for_services(
synapse_host=system_paasta_config.get_default_synapse_host(),
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.get_synapse_haproxy_url_format(),
services=[synapse_name],
)
num_available = local_replication.get(synapse_name, 0)
under_replicated, ratio = utils.is_under_replicated(
num_available, expected_count_per_location, crit_threshold)
log.info('Service %s.%s has %d out of %d expected instances' % (
service, instance, num_available, expected_count_per_location))
return under_replicated
def get_service_replication(self, all_services, synapse_host, synapse_port,
synapse_haproxy_url_format):
# Get the replication data once for performance
synapse_host_port = f"{synapse_host}:{synapse_port}"
self.log.debug(
"Gathering replication information from {}".format(
synapse_host_port), )
service_replication: Dict[str, int] = {}
try:
service_replication = get_replication_for_services(
synapse_host,
synapse_port,
synapse_haproxy_url_format,
['%s.main' % name for name in all_services],
)
except requests.exceptions.ConnectionError:
self.log.error(
'Failed to connect synapse haproxy on {}'.format(
synapse_host_port), )
self.critical(
'Failed to connect synapse haproxy on {}'.format(
synapse_host_port), )
except Exception as e:
self.log.error(
'Unable to collect replication information on {}: {}'.format(
synapse_host_port, str(e)), )
self.critical(
'Unable to collect replication information: {}'.format(
str(e)), )
self.log.debug(
"Finished gathering replication information from {}".format(
synapse_host_port), )
return service_replication
def get_service_replication(self, all_services, synapse_host, synapse_port, synapse_haproxy_url_format):
# Get the replication data once for performance
synapse_host_port = "%s:%s" % (synapse_host, synapse_port)
self.log.debug(
"Gathering replication information from {0}".
format(synapse_host_port))
service_replication = {}
try:
service_replication = get_replication_for_services(
synapse_host,
synapse_port,
synapse_haproxy_url_format,
['%s.main' % name for name in all_services]
)
except requests.exceptions.ConnectionError:
self.log.error(
'Failed to connect synapse haproxy on {0}'.
format(synapse_host_port))
self.critical(
'Failed to connect synapse haproxy on {0}'.
format(synapse_host_port))
except Exception as e:
self.log.error(
'Unable to collect replication information on {0}: {1}'.
format(synapse_host_port, e.message))
self.critical(
'Unable to collect replication information: {0}'.
format(e.message))
self.log.debug(
"Finished gathering replication information from {0}".
format(synapse_host_port))
return service_replication
def synapse_replication_is_low(service, instance, system_paasta_config,
local_backends):
crit_threshold = 80
namespace = read_namespace_for_service_instance(service=service,
instance=instance)
smartstack_replication_info = load_smartstack_info_for_service(
service=service,
namespace=namespace,
blacklist=[],
system_paasta_config=system_paasta_config,
)
expected_count = get_expected_instance_count_for_namespace(
service=service, namespace=namespace)
expected_count_per_location = int(expected_count /
len(smartstack_replication_info))
synapse_name = "%s.%s" % (service, namespace)
local_replication = get_replication_for_services(
synapse_host=system_paasta_config.get_default_synapse_host(),
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.
get_synapse_haproxy_url_format(),
services=[synapse_name],
)
num_available = local_replication.get(synapse_name, 0)
under_replicated, ratio = utils.is_under_replicated(
num_available, expected_count_per_location, crit_threshold)
log.info('Service %s.%s has %d out of %d expected instances' %
(service, instance, num_available, expected_count_per_location))
return under_replicated
def run_synapse_check():
system_paasta_config = utils.load_system_paasta_config()
options = parse_synapse_check_options(system_paasta_config)
try:
service_replications = get_replication_for_services(
options.synapse_host, options.synapse_port,
options.synapse_haproxy_url_format, options.services)
all_codes = []
for name, replication in service_replications.iteritems():
code, message = check_replication(name, replication, options.warn,
options.crit)
all_codes.append(code)
paasta_print(message)
sys.exit(max(all_codes))
except Exception, e:
fail('UNKNOWN: {0}'.format(e), 3)
def test_get_replication_for_service():
testdir = os.path.dirname(os.path.realpath(__file__))
testdata = os.path.join(testdir, "haproxy_snapshot.txt")
with open(testdata, "r") as fd:
mock_haproxy_data = fd.read()
mock_response = mock.Mock()
mock_response.text = mock_haproxy_data
mock_get = mock.Mock(return_value=(mock_response))
with mock.patch.object(requests.Session, "get", mock_get):
replication_result = get_replication_for_services(
"fake_host",
6666,
DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
["service1", "service2", "service3", "service4"],
)
expected = {"service1": 18, "service2": 19, "service3": 0, "service4": 3}
assert expected == replication_result
def run_synapse_check():
system_paasta_config = utils.load_system_paasta_config()
options = parse_synapse_check_options(system_paasta_config)
try:
service_replications = get_replication_for_services(
options.synapse_host,
options.synapse_port,
options.synapse_haproxy_url_format,
options.services
)
all_codes = []
for name, replication in service_replications.iteritems():
code, message = check_replication(name, replication,
options.warn, options.crit)
all_codes.append(code)
print message
sys.exit(max(all_codes))
except Exception, e:
fail('UNKNOWN: {0}'.format(e), 3)
def test_get_replication_for_service():
testdir = os.path.dirname(os.path.realpath(__file__))
testdata = os.path.join(testdir, 'haproxy_snapshot.txt')
with open(testdata, 'r') as fd:
mock_haproxy_data = fd.read()
mock_response = mock.Mock()
mock_response.text = mock_haproxy_data
mock_get = mock.Mock(return_value=(mock_response))
with mock.patch.object(requests.Session, 'get', mock_get):
replication_result = get_replication_for_services(
'fake_host', 6666, DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
['service1', 'service2', 'service3', 'service4'])
expected = {
'service1': 18,
'service2': 19,
'service3': 0,
'service4': 3
}
assert expected == replication_result
def test_get_replication_for_service():
testdir = os.path.dirname(os.path.realpath(__file__))
testdata = os.path.join(testdir, 'haproxy_snapshot.txt')
with open(testdata, 'r') as fd:
mock_haproxy_data = fd.read()
mock_response = mock.Mock()
mock_response.text = mock_haproxy_data
mock_get = mock.Mock(return_value=(mock_response))
with mock.patch.object(requests.Session, 'get', mock_get):
replication_result = get_replication_for_services(
'fake_host',
6666,
DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
['service1', 'service2', 'service3', 'service4']
)
expected = {
'service1': 18,
'service2': 19,
'service3': 0,
'service4': 3
}
assert expected == replication_result
