def __init__(self, config_path):
config = cfg.Config()
statsd_config = config.get_config(['Main', 'Statsd'])
# Create the aggregator (which is the point of communication between the server and reporting threads.
aggregator = agg.MetricsAggregator(
util.get_hostname(),
recent_point_threshold=statsd_config['recent_point_threshold'])
# Start the reporting thread.
interval = int(statsd_config['monasca_statsd_interval'])
assert 0 < interval
self.reporter = reporter.Reporter(
interval, aggregator, statsd_config['forwarder_url'],
statsd_config.get('event_chunk_size'))
# Start the server on an IPv4 stack
if statsd_config['non_local_traffic']:
server_host = ''
else:
server_host = 'localhost'
self.server = udp.Server(
aggregator,
server_host,
statsd_config['monasca_statsd_port'],
forward_to_host=statsd_config.get('monasca_statsd_forward_host'),
forward_to_port=int(
statsd_config.get('monasca_statsd_forward_port')))
def __init__(self, name, init_config, agent_config, instances=None):
self._socket_errors = set()
self._response_not_ready = set()
self._general_exception = set()
self._invalid_token = set()
self._warn_msg = set()
super(HTTPCheck, self).__init__(name, init_config, agent_config,
instances)
# init the keystone client if instance has use_keystone
self._api_config = cfg.Config().get_config('Api')
self._ksclients = {}
init_keystone_config = init_config.get('keystone_config', None)
for instance in instances:
addr, username, password, timeout, headers, response_time, \
disable_ssl_validation, use_keystone, keystone_config, \
instance_name = self._load_http_conf(instance)
if use_keystone:
# keystone is a singleton. It will be initialized once,
# the first config instance used.
if init_keystone_config:
ksclient = keystone.Keystone(init_keystone_config)
elif keystone_config:
# Using Keystone config in each instance is deprecated
# in Rocky.
ksclient = keystone.Keystone(keystone_config)
else:
ksclient = keystone.Keystone(self._api_config)
self._ksclients[instance_name] = ksclient
def get_hostname():
"""Get the canonical host name this agent should identify as. This is
the authoritative source of the host name for the agent.
Tries, in order:
* agent config (agent.yaml, "hostname:")
* 'hostname -f' (on unix)
* socket.gethostname()
"""
hostname = None
# first, try the config
config = configuration.Config()
agent_config = config.get_config(sections='Main')
config_hostname = agent_config.get('hostname')
if config_hostname and is_valid_hostname(config_hostname):
return config_hostname
# then move on to os-specific detection
if hostname is None:
def _get_hostname_unix():
try:
# try fqdn
p = subprocess.Popen(['/bin/hostname', '-f'],
stdout=subprocess.PIPE)
out, err = p.communicate()
if p.returncode == 0:
return encodeutils.safe_decode(out.strip(),
incoming='utf-8')
except Exception:
return None
os_name = get_os()
if os_name in ['mac', 'freebsd', 'linux', 'solaris']:
unix_hostname = _get_hostname_unix()
if unix_hostname and is_valid_hostname(unix_hostname):
hostname = unix_hostname
# fall back on socket.gethostname(), socket.getfqdn() is too unreliable
if hostname is None:
try:
socket_hostname = socket.gethostname()
except socket.error:
socket_hostname = None
if socket_hostname and is_valid_hostname(socket_hostname):
hostname = socket_hostname
if hostname is None:
log.critical(
'Unable to reliably determine host name. You can define one in agent.yaml '
'or in your hosts file')
raise Exception(
'Unable to reliably determine host name. You can define one in agent.yaml '
'or in your hosts file')
else:
return hostname
def _unix_confd_path(self):
try:
return configuration.Config().get_confd_path()
except PathNotFound:
pass
path = os.path.join(os.getcwd(), 'conf.d')
if os.path.exists(path):
return path
raise PathNotFound(path)
def __init__(self, name, init_config, agent_config, instances=None):
super(HTTPCheck, self).__init__(name, init_config, agent_config,
instances)
# init the keystone client if instance has use_keystone
self._api_config = cfg.Config().get_config('Api')
self._ksclients = {}
for instance in instances:
addr, username, password, timeout, headers, response_time, \
disable_ssl_validation, use_keystone, keystone_config, \
instance_name = self._load_http_conf(instance)
if use_keystone:
# keystone is a singleton. It will be initialized once,
# the first config instance used.
if keystone_config:
ksclient = keystone.Keystone(keystone_config)
else:
ksclient = keystone.Keystone(self._api_config)
self._ksclients[instance_name] = ksclient
def init_forwarder(skip_ssl_validation=False, use_simple_http_client=False):
config = cfg.Config()
agent_config = config.get_config(['Main', 'Api', 'Logging'])
port = agent_config['listen_port']
if port is None:
port = 17123
else:
port = int(port)
app = Forwarder(port, agent_config, skip_ssl_validation=skip_ssl_validation,
use_simple_http_client=use_simple_http_client)
def sigterm_handler(signum, frame):
log.info("caught sigterm. stopping")
app.stop()
signal.signal(signal.SIGTERM, sigterm_handler)
signal.signal(signal.SIGINT, sigterm_handler)
return app
def main():
options, args = util.get_parsed_args()
config = cfg.Config()
collector_config = config.get_config(['Main', 'Api', 'Logging'])
# todo autorestart isn't used remove
autorestart = collector_config.get('autorestart', False)
COMMANDS = [
'start',
'stop',
'restart',
'foreground',
'status',
'info',
'check',
'check_all',
'configcheck',
'jmx',
]
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
pid_file = util.PidFile('monasca-agent')
if options.clean:
pid_file.clean()
agent = CollectorDaemon(pid_file.get_path(), autorestart)
if command in START_COMMANDS:
log.info('Agent version %s' % config.get_version())
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return agent.info(verbose=options.verbose)
elif 'foreground' == command:
logging.info('Running in foreground')
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.run()
def parent_func():
agent.start_event = False
monasca_agent.common.daemon.AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.run(collector_config)
elif 'check' == command:
check_name = args[1]
checks = util.load_check_directory()
for check in checks['initialized_checks']:
if check.name == check_name:
run_check(check)
elif 'check_all' == command:
print("Loading check directory...")
checks = util.load_check_directory()
print("...directory loaded.\n")
for check in checks['initialized_checks']:
run_check(check)
elif 'configcheck' == command or 'configtest' == command:
osname = util.get_os()
all_valid = True
paths = util.Paths()
for conf_path in glob.glob(os.path.join(paths.get_confd_path(), "*.yaml")):
basename = os.path.basename(conf_path)
try:
config.check_yaml(conf_path)
except Exception as e:
all_valid = False
print("%s contains errors:\n %s" % (basename, e))
else:
print("%s is valid" % basename)
if all_valid:
print("All yaml files passed. You can now run the Monitoring agent.")
return 0
else:
print("Fix the invalid yaml files above in order to start the Monitoring agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
elif 'jmx' == command:
if len(args) < 2 or args[1] not in jmxfetch.JMX_LIST_COMMANDS.keys():
print("#" * 80)
print("JMX tool to be used to help configure your JMX checks.")
print("See http://docs.datadoghq.com/integrations/java/ for more information")
print("#" * 80)
print("\n")
print("You have to specify one of the following commands:")
for command, desc in jmxfetch.JMX_LIST_COMMANDS.iteritems():
print(" - %s [OPTIONAL: LIST OF CHECKS]: %s" % (command, desc))
print("Example: sudo /etc/init.d/monasca-agent jmx list_matching_attributes tomcat jmx solr")
print("\n")
else:
jmx_command = args[1]
checks_list = args[2:]
paths = util.Paths()
confd_path = paths.get_confd_path()
# Start JMXFetch if needed
should_run = jmxfetch.JMXFetch.init(confd_path,
config,
15,
jmx_command,
checks_list,
reporter="console")
if not should_run:
print("Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_path)
print("Have you enabled any JMX checks ?")
return 0
def get_collector_restart_interval():
config = configuration.Config()
agent_config = config.get_config(sections='Main')
restart_interval = agent_config.get('collector_restart_interval')
restart_interval_in_sec = restart_interval * 60 * 60
return restart_interval_in_sec
def get_sub_collection_warn():
config = configuration.Config()
agent_config = config.get_config(sections='Main')
sub_collection_warn = agent_config.get('sub_collection_warn')
return sub_collection_warn
def initialize_logging(logger_name):
try:
log_format = '%%(asctime)s | %%(levelname)s | %s | %%(name)s(%%(filename)s:%%(lineno)s) | %%(message)s' % logger_name
log_date_format = "%Y-%m-%d %H:%M:%S %Z"
config = configuration.Config()
logging_config = config.get_config(sections='Logging')
logging.basicConfig(
format=log_format,
level=logging_config['log_level'] or logging.INFO,
)
# set up file loggers
log_file = logging_config.get('%s_log_file' % logger_name)
if log_file is not None and not logging_config['disable_file_logging']:
# make sure the log directory is writeable
# NOTE: the entire directory needs to be writable so that rotation works
if os.access(os.path.dirname(log_file), os.R_OK | os.W_OK):
file_handler = logging.handlers.RotatingFileHandler(
log_file, maxBytes=LOGGING_MAX_BYTES, backupCount=1)
formatter = logging.Formatter(log_format, log_date_format)
file_handler.setFormatter(formatter)
root_log = logging.getLogger()
root_log.addHandler(file_handler)
else:
sys.stderr.write("Log file is unwritable: '%s'\n" % log_file)
# set up syslog
if logging_config['log_to_syslog']:
try:
syslog_format = '%s[%%(process)d]: %%(levelname)s (%%(filename)s:%%(lineno)s): %%(message)s' % logger_name
from logging.handlers import SysLogHandler
if logging_config['syslog_host'] is not None and logging_config[
'syslog_port'] is not None:
sys_log_addr = (logging_config['syslog_host'], logging_config['syslog_port'])
else:
sys_log_addr = "/dev/log"
# Special-case macs
if sys.platform == 'darwin':
sys_log_addr = "/var/run/syslog"
handler = SysLogHandler(address=sys_log_addr, facility=SysLogHandler.LOG_DAEMON)
handler.setFormatter(
logging.Formatter(syslog_format, log_date_format))
root_log = logging.getLogger()
root_log.addHandler(handler)
except Exception as e:
sys.stderr.write("Error setting up syslog: '%s'\n" % str(e))
traceback.print_exc()
except Exception as e:
sys.stderr.write("Couldn't initialize logging: %s\n" % str(e))
traceback.print_exc()
# if config fails entirely, enable basic stdout logging as a fallback
logging.basicConfig(
format=log_format,
level=logging.INFO,
)
# re-get the log after logging is initialized
global log
log = logging.getLogger(__name__)
def load_check_directory():
"""Return the initialized checks from checks_d, and a mapping of checks that failed to
initialize. Only checks that have a configuration
file in conf.d will be returned.
"""
from monasca_agent.collector.checks import AgentCheck
config = configuration.Config()
agent_config = config.get_config('Main')
initialized_checks = {}
init_failed_checks = {}
paths = Paths()
checks_paths = [glob.glob(os.path.join(agent_config['additional_checksd'], '*.py'))]
try:
checksd_path = paths.get_checksd_path()
checks_paths.append(glob.glob(os.path.join(checksd_path, '*.py')))
except PathNotFound as e:
log.error(e.args[0])
sys.exit(3)
try:
confd_path = paths.get_confd_path()
except PathNotFound as e:
log.error(
"No conf.d folder found at '%s' or in the directory where the Agent is currently deployed.\n" %
e.args[0])
sys.exit(3)
# For backwards-compatability with old style checks, we have to load every
# checks_d module and check for a corresponding config OR check if the old
# config will "activate" the check.
#
# Once old-style checks aren't supported, we'll just read the configs and
# import the corresponding check module
for check in itertools.chain(*checks_paths):
check_name = os.path.basename(check).split('.')[0]
if check_name in initialized_checks or check_name in init_failed_checks:
log.debug(
'Skipping check %s because it has already been loaded from another location', check)
continue
try:
check_module = imp.load_source('checksd_%s' % check_name, check)
except Exception as e:
traceback_message = traceback.format_exc()
# Let's see if there is a conf.d for this check
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
# There is a configuration file for that check but the module can't be imported
init_failed_checks[check_name] = {'error': e, 'traceback': traceback_message}
log.exception('Unable to import check module %s.py from checks_d' % check_name)
else: # There is no conf for that check. Let's not spam the logs for it.
log.debug('Unable to import check module %s.py from checks_d' % check_name)
continue
check_class = None
classes = inspect.getmembers(check_module, inspect.isclass)
for _, clsmember in classes:
if clsmember == AgentCheck:
continue
if issubclass(clsmember, AgentCheck):
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
if not check_name == '__init__':
log.error('No check class (inheriting from AgentCheck) found in %s.py' % check_name)
continue
# Check if the config exists
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
try:
check_config = config.check_yaml(conf_path)
except Exception as e:
log.exception("Unable to parse yaml config in %s" % conf_path)
traceback_message = traceback.format_exc()
init_failed_checks[check_name] = {'error': e, 'traceback': traceback_message}
continue
else:
log.debug('No conf.d/%s.yaml found for checks_d/%s.py' % (check_name, check_name))
continue
# Look for the per-check config, which *must* exist
if 'instances' not in check_config:
log.error("Config %s is missing 'instances'" % conf_path)
continue
# Init all of the check's classes with
init_config = check_config.get('init_config', {})
# init_config: in the configuration triggers init_config to be defined
# to None.
if init_config is None:
init_config = {}
instances = check_config['instances']
try:
try:
c = check_class(check_name, init_config=init_config,
agent_config=agent_config, instances=instances)
except TypeError as e:
# Backwards compatibility for checks which don't support the
# instances argument in the constructor.
c = check_class(check_name, init_config=init_config,
agent_config=agent_config)
c.instances = instances
except Exception as e:
log.exception('Unable to initialize check %s' % check_name)
traceback_message = traceback.format_exc()
init_failed_checks[check_name] = {'error': e, 'traceback': traceback_message}
else:
initialized_checks[check_name] = c
# Add custom pythonpath(s) if available
if 'pythonpath' in check_config:
pythonpath = check_config['pythonpath']
if not isinstance(pythonpath, list):
pythonpath = [pythonpath]
sys.path.extend(pythonpath)
log.debug('Loaded check.d/%s.py' % check_name)
log.info('Successfully initialized checks: %s' % initialized_checks.keys())
if len(init_failed_checks) > 0:
log.info('Initialization failed for checks: %s' % init_failed_checks.keys())
return {'initialized_checks': initialized_checks.values(),
'init_failed_checks': init_failed_checks,
}
log = logging.getLogger(__name__)
# Relevant configuration options must be set in 'conf.d/libvirt.yaml'
# file under 'init_config'.
# eg.
# init_config:
# libvirt_type: kvm
# libvirt_uri: qemu:///system
#
# 'libvirt_type' valid choices: ('kvm', 'lxc', 'qemu', 'uml', 'xen')
# 'libvirt_uri' is dependent on 'libvirt_type'
paths = util.Paths()
conf_path = os.path.join(paths.get_confd_path(), 'libvirt.yaml')
config = configuration.Config()
if os.path.exists(conf_path):
try:
check_config = config.check_yaml(conf_path)
except Exception as e:
log.exception('Unable to parse yaml config in {}\n{}'.format(
conf_path, str(e)))
else:
log.debug('No conf.d/libvirt.yaml found for virt/libvirt/inspector.py')
init_config = check_config.get('init_config', {})
if init_config is None:
init_config = {}
# (C) Copyright 2017 Hewlett Packard Enterprise Development Company LP
import unittest
import os
import monasca_agent.common.config as configuration
import monasca_agent.common.metrics as metrics_pkg
from monasca_agent.collector.checks import AgentCheck
base_config = configuration.Config(os.path.join(os.path.dirname(__file__),
'test-agent.yaml'))
class TestAgentCheck(unittest.TestCase):
def testBadMetricKeepBatch(self):
agent_config = base_config.get_config(sections='Main')
check = AgentCheck("foo", {}, agent_config)
dimensions = {'A': 'B', 'B': 'C', 'D': 'E'}
check.submit_metric("Foo",
5,
metrics_pkg.Gauge,
dimensions=dimensions,
delegated_tenant=None,
hostname=None,
device_name=None,
value_meta=None)
self.assertEqual(len(check.aggregator.metrics), 1)
def main():
options, args = util.get_parsed_args()
config = cfg.Config()
collector_config = config.get_config(['Main', 'Api', 'Logging'])
# todo autorestart isn't used remove
autorestart = collector_config.get('autorestart', False)
COMMANDS = [
'start',
'stop',
'restart',
'foreground',
'status',
'info',
'check',
'check_all',
'configcheck',
'jmx',
]
if len(args) < 1:
sys.stderr.write("Usage: %s %s\n" % (sys.argv[0], "|".join(COMMANDS)))
return 2
command = args[0]
if command not in COMMANDS:
sys.stderr.write("Unknown command: %s\n" % command)
return 3
pid_file = util.PidFile('monasca-agent')
if options.clean:
pid_file.clean()
agent = CollectorDaemon(pid_file.get_path(), autorestart)
if command in START_COMMANDS:
log.info('Agent version %s' % config.get_version())
if 'start' == command:
log.info('Start daemon')
agent.start()
elif 'stop' == command:
log.info('Stop daemon')
agent.stop()
elif 'restart' == command:
log.info('Restart daemon')
agent.restart()
elif 'status' == command:
agent.status()
elif 'info' == command:
return agent.info(verbose=options.verbose)
elif 'foreground' == command:
logging.info('Running in foreground')
if autorestart:
# Set-up the supervisor callbacks and fork it.
logging.info('Running Agent with auto-restart ON')
def child_func():
agent.run()
def parent_func():
agent.start_event = False
monasca_agent.common.daemon.AgentSupervisor.start(parent_func, child_func)
else:
# Run in the standard foreground.
agent.run(collector_config)
elif 'check' == command:
check_name = args[1]
checks = util.load_check_directory()
for check in checks['initialized_checks']:
if check.name == check_name:
run_check(check)
elif 'check_all' == command:
print("Loading check directory...")
checks = util.load_check_directory()
print("...directory loaded.\n")
for check in checks['initialized_checks']:
run_check(check)
elif 'configcheck' == command or 'configtest' == command:
all_valid = True
paths = util.Paths()
for conf_path in glob.glob(os.path.join(paths.get_confd_path(), "*.yaml")):
basename = os.path.basename(conf_path)
try:
config.check_yaml(conf_path)
except Exception as e:
all_valid = False
print("%s contains errors:\n %s" % (basename, e))
else:
print("%s is valid" % basename)
if all_valid:
print("All yaml files passed. You can now run the Monitoring agent.")
return 0
else:
print("Fix the invalid yaml files above in order to start the Monitoring agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
def _http_check(self, instance):
addr, username, password, timeout, headers, response_time, disable_ssl_validation, use_keystone, keystone_config = self._load_http_conf(
instance)
config = cfg.Config()
api_config = config.get_config('Api')
dimensions = self._set_dimensions({'url': addr}, instance)
start = time.time()
done = False
retry = False
while not done or retry:
if use_keystone:
if keystone_config:
key = keystone.Keystone(keystone_config)
else:
key = keystone.Keystone(api_config)
token = key.get_token()
if token:
headers["X-Auth-Token"] = token
headers["Content-type"] = "application/json"
else:
error_string = """Unable to get token. Keystone API server may be down.
Skipping check for {0}""".format(addr)
self.log.warning(error_string)
return False, error_string
try:
self.log.debug("Connecting to %s" % addr)
if disable_ssl_validation:
self.warning(
"Skipping SSL certificate validation for %s based on configuration"
% addr)
h = Http(
timeout=timeout,
disable_ssl_certificate_validation=disable_ssl_validation)
if username is not None and password is not None:
h.add_credentials(username, password)
resp, content = h.request(addr, "GET", headers=headers)
except (socket.timeout, HttpLib2Error, socket.error) as e:
length = int((time.time() - start) * 1000)
error_string = '{0} is DOWN, error: {1}. Connection failed after {2} ms'.format(
addr, str(e), length)
self.log.info(error_string)
return False, error_string
except httplib.ResponseNotReady as e:
length = int((time.time() - start) * 1000)
error_string = '{0} is DOWN, error: {1}. Network is not routable after {2} ms'.format(
addr, repr(e), length)
self.log.info(error_string)
return False, error_string
except Exception as e:
length = int((time.time() - start) * 1000)
error_string = '{0} is DOWN, error: {1}. Connection failed after {2} ms'.format(
addr, str(e), length)
self.log.error(
'Unhandled exception {0}. Connection failed after {1} ms'.
format(str(e), length))
return False, error_string
if response_time:
# Stop the timer as early as possible
running_time = time.time() - start
self.gauge('http_response_time',
running_time,
dimensions=dimensions)
if int(resp.status) >= 400:
if use_keystone and int(resp.status) == 401:
if retry:
error_string = '{0} is DOWN, unable to get a valid token to connect with'.format(
addr)
self.log.error(error_string)
return False, error_string
else:
# Get a new token and retry
self.log.info(
"Token expired, getting new token and retrying...")
retry = True
key.refresh_token()
continue
else:
error_string = '{0} is DOWN, error code: {1}'.format(
addr, str(resp.status))
self.log.info(error_string)
return False, error_string
done = True
return True, content
def _check(self, instance):
addr, username, password, timeout, headers, response_time, disable_ssl_validation, pattern, use_keystone = self._load_conf(
instance)
config = cfg.Config()
api_config = config.get_config('Api')
content = ''
dimensions = self._set_dimensions({'url': addr}, instance)
start = time.time()
done = False
retry = False
while not done or retry:
if use_keystone:
key = keystone.Keystone(api_config)
token = key.get_token()
if token:
headers["X-Auth-Token"] = token
headers["Content-type"] = "application/json"
else:
self.log.warning("""Unable to get token. Keystone API server may be down.
Skipping check for {0}""".format(addr))
return
try:
self.log.debug("Connecting to %s" % addr)
if disable_ssl_validation:
self.warning(
"Skipping SSL certificate validation for %s based on configuration" % addr)
h = Http(timeout=timeout, disable_ssl_certificate_validation=disable_ssl_validation)
if username is not None and password is not None:
h.add_credentials(username, password)
resp, content = h.request(addr, "GET", headers=headers)
except socket.timeout as e:
length = int((time.time() - start) * 1000)
self.log.info(
"%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "%s is DOWN, error: %s. Connection failed after %s ms" % (
addr, str(e), length)
except HttpLib2Error as e:
length = int((time.time() - start) * 1000)
self.log.info(
"%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "%s is DOWN, error: %s. Connection failed after %s ms" % (
addr, str(e), length)
except socket.error as e:
length = int((time.time() - start) * 1000)
self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (
addr, repr(e), length))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "%s is DOWN, error: %s. Connection failed after %s ms" % (
addr, str(e), length)
except httplib.ResponseNotReady as e:
length = int((time.time() - start) * 1000)
self.log.info("%s is DOWN, error: %s. Network is not routable after %s ms" % (
addr, repr(e), length))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "%s is DOWN, error: %s. Network is not routable after %s ms" % (
addr, str(e), length)
except Exception as e:
length = int((time.time() - start) * 1000)
self.log.error(
"Unhandled exception %s. Connection failed after %s ms" % (str(e), length))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "%s is DOWN, error: %s. Connection failed after %s ms" % (
addr, str(e), length)
if response_time:
# Stop the timer as early as possible
running_time = time.time() - start
self.gauge('http_response_time', running_time, dimensions=dimensions)
# TODO(dschroeder): Save/send content data when supported by API
if int(resp.status) >= 400:
if use_keystone and int(resp.status) == 401:
if retry:
self.log.error("%s is DOWN, unable to get a valid token to connect with" % (addr))
return services_checks.Status.DOWN, "%s is DOWN, unable to get a valid token to connect with" % (
addr)
else:
# Get a new token and retry
self.log.info("Token expired, getting new token and retrying...")
retry = True
key.refresh_token()
continue
else:
self.log.info("%s is DOWN, error code: %s" % (addr, str(resp.status)))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "%s is DOWN, error code: %s" % (addr, str(resp.status))
if pattern is not None:
if re.search(pattern, content, re.DOTALL):
self.log.debug("Pattern match successful")
else:
self.log.info("Pattern match failed! '%s' not in '%s'" % (pattern, content))
self.gauge('http_status', 1, dimensions=dimensions)
return services_checks.Status.DOWN, "Pattern match failed! '%s' not in '%s'" % (
pattern, content)
self.log.debug("%s is UP" % addr)
self.gauge('http_status', 0, dimensions=dimensions)
done = True
return services_checks.Status.UP, "%s is UP" % addr
