def synapse_replication_is_low(service, instance, system_paasta_config, local_backends): crit_threshold = 80 reg_svc, reg_namespace, _, __ = utils.decompose_job_id( read_registration_for_service_instance(service=service, instance=instance)) # We only actually care about the replication of where we're registering service, namespace = reg_svc, reg_namespace smartstack_replication_info = load_smartstack_info_for_service( service=service, namespace=namespace, blacklist=[], system_paasta_config=system_paasta_config, ) expected_count = get_expected_instance_count_for_namespace( service=service, namespace=namespace) expected_count_per_location = int(expected_count / len(smartstack_replication_info)) synapse_name = utils.compose_job_id(service, namespace) local_replication = get_replication_for_services( synapse_host=system_paasta_config.get_default_synapse_host(), synapse_port=system_paasta_config.get_synapse_port(), synapse_haproxy_url_format=system_paasta_config. get_synapse_haproxy_url_format(), services=[synapse_name], ) num_available = local_replication.get(synapse_name, 0) under_replicated, ratio = utils.is_under_replicated( num_available, expected_count_per_location, crit_threshold) log.info('Service %s.%s has %d out of %d expected instances' % (service, instance, num_available, expected_count_per_location)) return under_replicated
def get_service_replication(self, all_services, synapse_host, synapse_port, synapse_haproxy_url_format): # Get the replication data once for performance synapse_host_port = "%s:%s" % (synapse_host, synapse_port) self.log.debug("Gathering replication information from {0}".format( synapse_host_port)) service_replication = {} try: service_replication = get_replication_for_services( synapse_host, synapse_port, synapse_haproxy_url_format, ['%s.main' % name for name in all_services]) except requests.exceptions.ConnectionError: self.log.error('Failed to connect synapse haproxy on {0}'.format( synapse_host_port)) self.critical('Failed to connect synapse haproxy on {0}'.format( synapse_host_port)) except Exception as e: self.log.error( 'Unable to collect replication information on {0}: {1}'.format( synapse_host_port, e.message)) self.critical( 'Unable to collect replication information: {0}'.format( e.message)) self.log.debug( "Finished gathering replication information from {0}".format( synapse_host_port)) return service_replication
def synapse_replication_is_low(service, instance, system_paasta_config, local_backends): crit_threshold = 80 reg_svc, reg_namespace, _, __ = utils.decompose_job_id( read_registration_for_service_instance( service=service, instance=instance ) ) # We only actually care about the replication of where we're registering service, namespace = reg_svc, reg_namespace smartstack_replication_info = load_smartstack_info_for_service( service=service, namespace=namespace, blacklist=[], system_paasta_config=system_paasta_config, ) expected_count = get_expected_instance_count_for_namespace(service=service, namespace=namespace) expected_count_per_location = int(expected_count / len(smartstack_replication_info)) synapse_name = utils.compose_job_id(service, namespace) local_replication = get_replication_for_services( synapse_host=system_paasta_config.get_default_synapse_host(), synapse_port=system_paasta_config.get_synapse_port(), synapse_haproxy_url_format=system_paasta_config.get_synapse_haproxy_url_format(), services=[synapse_name], ) num_available = local_replication.get(synapse_name, 0) under_replicated, ratio = utils.is_under_replicated( num_available, expected_count_per_location, crit_threshold) log.info('Service %s.%s has %d out of %d expected instances' % ( service, instance, num_available, expected_count_per_location)) return under_replicated
def get_service_replication(self, all_services, synapse_host, synapse_port, synapse_haproxy_url_format): # Get the replication data once for performance synapse_host_port = f"{synapse_host}:{synapse_port}" self.log.debug( "Gathering replication information from {}".format( synapse_host_port), ) service_replication: Dict[str, int] = {} try: service_replication = get_replication_for_services( synapse_host, synapse_port, synapse_haproxy_url_format, ['%s.main' % name for name in all_services], ) except requests.exceptions.ConnectionError: self.log.error( 'Failed to connect synapse haproxy on {}'.format( synapse_host_port), ) self.critical( 'Failed to connect synapse haproxy on {}'.format( synapse_host_port), ) except Exception as e: self.log.error( 'Unable to collect replication information on {}: {}'.format( synapse_host_port, str(e)), ) self.critical( 'Unable to collect replication information: {}'.format( str(e)), ) self.log.debug( "Finished gathering replication information from {}".format( synapse_host_port), ) return service_replication
def get_service_replication(self, all_services, synapse_host, synapse_port, synapse_haproxy_url_format): # Get the replication data once for performance synapse_host_port = "%s:%s" % (synapse_host, synapse_port) self.log.debug( "Gathering replication information from {0}". format(synapse_host_port)) service_replication = {} try: service_replication = get_replication_for_services( synapse_host, synapse_port, synapse_haproxy_url_format, ['%s.main' % name for name in all_services] ) except requests.exceptions.ConnectionError: self.log.error( 'Failed to connect synapse haproxy on {0}'. format(synapse_host_port)) self.critical( 'Failed to connect synapse haproxy on {0}'. format(synapse_host_port)) except Exception as e: self.log.error( 'Unable to collect replication information on {0}: {1}'. format(synapse_host_port, e.message)) self.critical( 'Unable to collect replication information: {0}'. format(e.message)) self.log.debug( "Finished gathering replication information from {0}". format(synapse_host_port)) return service_replication
def synapse_replication_is_low(service, instance, system_paasta_config, local_backends): crit_threshold = 80 namespace = read_namespace_for_service_instance(service=service, instance=instance) smartstack_replication_info = load_smartstack_info_for_service( service=service, namespace=namespace, blacklist=[], system_paasta_config=system_paasta_config, ) expected_count = get_expected_instance_count_for_namespace( service=service, namespace=namespace) expected_count_per_location = int(expected_count / len(smartstack_replication_info)) synapse_name = "%s.%s" % (service, namespace) local_replication = get_replication_for_services( synapse_host=system_paasta_config.get_default_synapse_host(), synapse_port=system_paasta_config.get_synapse_port(), synapse_haproxy_url_format=system_paasta_config. get_synapse_haproxy_url_format(), services=[synapse_name], ) num_available = local_replication.get(synapse_name, 0) under_replicated, ratio = utils.is_under_replicated( num_available, expected_count_per_location, crit_threshold) log.info('Service %s.%s has %d out of %d expected instances' % (service, instance, num_available, expected_count_per_location)) return under_replicated
def run_synapse_check(): system_paasta_config = utils.load_system_paasta_config() options = parse_synapse_check_options(system_paasta_config) try: service_replications = get_replication_for_services( options.synapse_host, options.synapse_port, options.synapse_haproxy_url_format, options.services) all_codes = [] for name, replication in service_replications.iteritems(): code, message = check_replication(name, replication, options.warn, options.crit) all_codes.append(code) paasta_print(message) sys.exit(max(all_codes)) except Exception, e: fail('UNKNOWN: {0}'.format(e), 3)
def test_get_replication_for_service(): testdir = os.path.dirname(os.path.realpath(__file__)) testdata = os.path.join(testdir, "haproxy_snapshot.txt") with open(testdata, "r") as fd: mock_haproxy_data = fd.read() mock_response = mock.Mock() mock_response.text = mock_haproxy_data mock_get = mock.Mock(return_value=(mock_response)) with mock.patch.object(requests.Session, "get", mock_get): replication_result = get_replication_for_services( "fake_host", 6666, DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT, ["service1", "service2", "service3", "service4"], ) expected = {"service1": 18, "service2": 19, "service3": 0, "service4": 3} assert expected == replication_result
def run_synapse_check(): system_paasta_config = utils.load_system_paasta_config() options = parse_synapse_check_options(system_paasta_config) try: service_replications = get_replication_for_services( options.synapse_host, options.synapse_port, options.synapse_haproxy_url_format, options.services ) all_codes = [] for name, replication in service_replications.iteritems(): code, message = check_replication(name, replication, options.warn, options.crit) all_codes.append(code) print message sys.exit(max(all_codes)) except Exception, e: fail('UNKNOWN: {0}'.format(e), 3)
def test_get_replication_for_service(): testdir = os.path.dirname(os.path.realpath(__file__)) testdata = os.path.join(testdir, 'haproxy_snapshot.txt') with open(testdata, 'r') as fd: mock_haproxy_data = fd.read() mock_response = mock.Mock() mock_response.text = mock_haproxy_data mock_get = mock.Mock(return_value=(mock_response)) with mock.patch.object(requests.Session, 'get', mock_get): replication_result = get_replication_for_services( 'fake_host', 6666, DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT, ['service1', 'service2', 'service3', 'service4']) expected = { 'service1': 18, 'service2': 19, 'service3': 0, 'service4': 3 } assert expected == replication_result
def test_get_replication_for_service(): testdir = os.path.dirname(os.path.realpath(__file__)) testdata = os.path.join(testdir, 'haproxy_snapshot.txt') with open(testdata, 'r') as fd: mock_haproxy_data = fd.read() mock_response = mock.Mock() mock_response.text = mock_haproxy_data mock_get = mock.Mock(return_value=(mock_response)) with mock.patch.object(requests.Session, 'get', mock_get): replication_result = get_replication_for_services( 'fake_host', 6666, DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT, ['service1', 'service2', 'service3', 'service4'] ) expected = { 'service1': 18, 'service2': 19, 'service3': 0, 'service4': 3 } assert expected == replication_result