def check_status_lines(cs):
check_lines = [" " + cs.name, " " + "-" * len(cs.name)]
if cs.init_failed_error:
check_lines.append(
" - initialize check class [%s]: %s" % (style(STATUS_ERROR, "red"), repr(cs.init_failed_error))
)
if cs.init_failed_traceback:
check_lines.extend(" " + line for line in cs.init_failed_traceback.split("\n"))
else:
for s in cs.instance_statuses:
c = "green"
if s.has_warnings():
c = "yellow"
if s.has_error():
c = "red"
line = " - instance #%s [%s]" % (s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get("run_time")
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split("\n")
if not len(warn):
continue
check_lines.append(u" %s: %s" % (style("Warning", "yellow"), warn[0]))
check_lines.extend(u" %s" % l for l in warn[1:])
if s.traceback is not None:
check_lines.extend(" " + line for line in s.traceback.split("\n"))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s"
% (
cs.metric_count,
plural(cs.metric_count),
cs.event_count,
plural(cs.event_count),
cs.service_check_count,
plural(cs.service_check_count),
)
]
if cs.check_stats is not None:
check_lines += [" - Stats: %s" % pretty_statistics(cs.check_stats)]
if cs.library_versions is not None:
check_lines += [" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [" - %s: %s" % (library, version)]
check_lines += [""]
return check_lines
i,
check_status.STATUS_ERROR,
error=str(e),
tb=traceback.format_exc())
finally:
self._roll_up_instance_metadata()
instance_statuses.append(instance_status)
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
after = AgentCheck._collect_internal_stats()
self._set_internal_profiling_stats(before, after)
log.info("\n \t %s %s" %
(self.name,
pretty_statistics(self._internal_profiling_stats)))
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug(
"Failed to collect Agent Stats after check {0}".format(
self.name))
return instance_statuses
def check(self, instance):
"""
Overriden by the check class. This will be called to run the check.
:param instance: A dict with the instance information. This will vary
depending on your config structure.
"""
raise NotImplementedError()
except Exception, e:
self.log.exception("Check '%s' instance #%s failed" % (self.name, i))
instance_status = check_status.InstanceStatus(
i, check_status.STATUS_ERROR,
error=str(e), tb=traceback.format_exc()
)
finally:
self._roll_up_instance_metadata()
instance_statuses.append(instance_status)
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
after = AgentCheck._collect_internal_stats()
self._set_internal_profiling_stats(before, after)
log.info("\n \t %s %s" % (self.name, pretty_statistics(self._internal_profiling_stats)))
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug("Failed to collect Agent Stats after check {0}".format(self.name))
return instance_statuses
def check(self, instance):
"""
Overriden by the check class. This will be called to run the check.
:param instance: A dict with the instance information. This will vary
depending on your config structure.
"""
raise NotImplementedError()
def stop(self):
def run(self):
""" Run all instances. """
# Store run statistics if needed
before, after = None, None
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
before = AgentCheck._collect_internal_stats()
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug(
"Failed to collect Agent Stats before check {0}".format(
self.name))
instance_statuses = []
for i, instance in enumerate(self.instances):
try:
min_collection_interval = instance.get(
'min_collection_interval', self.min_collection_interval)
now = time.time()
if now - self.last_collection_time[i] < min_collection_interval:
self.log.debug(
"Not running instance #{0} of check {1} as it ran less than {2}s ago"
.format(i, self.name, min_collection_interval))
continue
self.last_collection_time[i] = now
check_start_time = None
if self.in_developer_mode:
check_start_time = timeit.default_timer()
self.check(copy.deepcopy(instance))
instance_check_stats = None
if check_start_time is not None:
instance_check_stats = {
'run_time': timeit.default_timer() - check_start_time
}
if self.has_warnings():
instance_status = check_status.InstanceStatus(
i,
check_status.STATUS_WARNING,
warnings=self.get_warnings(),
instance_check_stats=instance_check_stats)
else:
instance_status = check_status.InstanceStatus(
i,
check_status.STATUS_OK,
instance_check_stats=instance_check_stats)
except Exception as e:
self.log.exception("Check '%s' instance #%s failed" %
(self.name, i))
instance_status = check_status.InstanceStatus(
i,
check_status.STATUS_ERROR,
error=str(e),
tb=traceback.format_exc())
finally:
self._roll_up_instance_metadata()
instance_statuses.append(instance_status)
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
after = AgentCheck._collect_internal_stats()
self._set_internal_profiling_stats(before, after)
log.info("\n \t %s %s" %
(self.name,
pretty_statistics(self._internal_profiling_stats)))
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug(
"Failed to collect Agent Stats after check {0}".format(
self.name))
return instance_statuses
def run(self):
""" Run all instances. """
# Store run statistics if needed
before, after = None, None
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
before = AgentCheck._collect_internal_stats()
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug("Failed to collect Agent Stats before check {0}".format(self.name))
instance_statuses = []
for i, instance in enumerate(self.instances):
try:
min_collection_interval = instance.get(
"min_collection_interval",
self.init_config.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL),
)
now = time.time()
if now - self.last_collection_time[i] < min_collection_interval:
self.log.debug(
"Not running instance #{0} of check {1} as it ran less than {2}s ago".format(
i, self.name, min_collection_interval
)
)
continue
self.last_collection_time[i] = now
check_start_time = None
if self.in_developer_mode:
check_start_time = timeit.default_timer()
self.check(copy.deepcopy(instance))
instance_check_stats = None
if check_start_time is not None:
instance_check_stats = {"run_time": timeit.default_timer() - check_start_time}
if self.has_warnings():
instance_status = check_status.InstanceStatus(
i,
check_status.STATUS_WARNING,
warnings=self.get_warnings(),
instance_check_stats=instance_check_stats,
)
else:
instance_status = check_status.InstanceStatus(
i, check_status.STATUS_OK, instance_check_stats=instance_check_stats
)
except Exception as e:
self.log.exception("Check '%s' instance #%s failed" % (self.name, i))
instance_status = check_status.InstanceStatus(
i, check_status.STATUS_ERROR, error=str(e), tb=traceback.format_exc()
)
finally:
self._roll_up_instance_metadata()
instance_statuses.append(instance_status)
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
after = AgentCheck._collect_internal_stats()
self._set_internal_profiling_stats(before, after)
log.info("\n \t %s %s" % (self.name, pretty_statistics(self._internal_profiling_stats)))
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug("Failed to collect Agent Stats after check {0}".format(self.name))
return instance_statuses
def body_lines(self):
# Metadata whitelist
metadata_whitelist = [
'hostname',
'fqdn',
'ipv4',
'instance-id'
]
lines = [
'Clocks',
'======',
''
]
try:
ntp_offset, ntp_styles = get_ntp_info()
lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles))
except Exception as e:
lines.append(' NTP offset: Unknown (%s)' % str(e))
lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__())
lines.append('')
# Paths to checks.d/conf.d
lines += [
'Paths',
'=====',
''
]
osname = config.get_os()
try:
confd_path = config.get_confd_path(osname)
except config.PathNotFound:
confd_path = 'Not found'
try:
checksd_path = config.get_checksd_path(osname)
except config.PathNotFound:
checksd_path = 'Not found'
lines.append(' conf.d: ' + confd_path)
lines.append(' checks.d: ' + checksd_path)
lines.append('')
# Hostnames
lines += [
'Hostnames',
'=========',
''
]
if not self.host_metadata:
lines.append(" No host information available yet.")
else:
for key, host in self.host_metadata.iteritems():
for whitelist_item in metadata_whitelist:
if whitelist_item in key:
lines.append(" " + key + ": " + host)
break
lines.append('')
# Checks.d Status
lines += [
'Checks',
'======',
''
]
check_statuses = self.check_statuses + get_jmx_status()
if not check_statuses:
lines.append(" No checks have run yet.")
else:
for cs in check_statuses:
check_lines = [
' ' + cs.name + ' ({})'.format(cs.check_version),
' ' + '-' * (len(cs.name) + 3 + len(cs.check_version))
]
if cs.init_failed_error:
check_lines.append(" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'),
repr(cs.init_failed_error)))
if self.verbose and cs.init_failed_traceback:
check_lines.extend(' ' + line for line in
cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (
s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get('run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in
warn[1:])
if self.verbose and s.traceback is not None:
check_lines.extend(' ' + line for line in
s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s" % (
cs.metric_count, plural(cs.metric_count),
cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [
" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [
" - %s: %s" % (library, version)]
check_lines += [""]
lines += check_lines
# Metadata status
metadata_enabled = 1#_is_affirmative(get_config().get('display_service_metadata', False))
if metadata_enabled:
lines += [
"",
"Service metadata",
"================",
""
]
if not check_statuses:
lines.append(" No checks have run yet.")
else:
meta_lines = []
for cs in check_statuses:
# Check title
check_line = [
' ' + cs.name,
' ' + '-' * len(cs.name)
]
instance_lines = []
for i, meta in enumerate(cs.service_metadata):
if not meta:
continue
instance_lines += [" - instance #%s:" % i]
for k, v in meta.iteritems():
instance_lines += [" - %s: %s" % (k, v)]
if instance_lines:
check_line += instance_lines
meta_lines += check_line
if meta_lines:
lines += meta_lines
else:
lines.append(" No metadata were collected.")
# Emitter status
lines += [
"",
"Emitters",
"========",
""
]
if not self.emitter_statuses:
lines.append(" No emitters have run yet.")
else:
for es in self.emitter_statuses:
c = 'green'
if es.has_error():
c = 'red'
line = " - %s [%s]" % (es.name, style(es.status, c))
if es.status != STATUS_OK:
line += ": %s" % es.error
lines.append(line)
return lines
def check_status_lines(cs):
check_lines = [
' ' + cs.name + ' ({})'.format(cs.check_version),
' ' + '-' * (len(cs.name) + 3 + len(cs.check_version))
]
if cs.init_failed_error:
check_lines.append(" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'),
repr(cs.init_failed_error)))
if cs.init_failed_traceback:
check_lines.extend(' ' + line for line in
cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (
s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get('run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in
warn[1:])
if s.traceback is not None:
check_lines.extend(' ' + line for line in
s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s" % (
cs.metric_count, plural(cs.metric_count),
cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [
" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [" - %s: %s" % (library, version)]
check_lines += [""]
return check_lines
def body_lines(self):
# Metadata whitelist
metadata_whitelist = [
'hostname',
'fqdn',
'ipv4',
'instance-id'
]
lines = [
'Clocks',
'======',
''
]
try:
ntp_offset, ntp_styles = get_ntp_info()
lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' + style('%s s' % round(ntp_offset, 4), *ntp_styles))
except Exception as e:
lines.append(' NTP offset: Unknown (%s)' % str(e))
lines.append(' System UTC time: ' + datetime.datetime.utcnow().__str__())
lines.append('')
# Paths to checks.d/conf.d
lines += [
'Paths',
'=====',
''
]
osname = config.get_os()
try:
confd_path = config.get_confd_path(osname)
except config.PathNotFound:
confd_path = 'Not found'
try:
checksd_path = config.get_checksd_path(osname)
except config.PathNotFound:
checksd_path = 'Not found'
lines.append(' conf.d: ' + confd_path)
lines.append(' checks.d: ' + checksd_path)
lines.append('')
# Hostnames
lines += [
'Hostnames',
'=========',
''
]
if not self.host_metadata:
lines.append(" No host information available yet.")
else:
for key, host in self.host_metadata.iteritems():
for whitelist_item in metadata_whitelist:
if whitelist_item in key:
lines.append(" " + key + ": " + host)
break
lines.append('')
# Checks.d Status
lines += [
'Checks',
'======',
''
]
check_statuses = self.check_statuses + get_jmx_status()
if not check_statuses:
lines.append(" No checks have run yet.")
else:
for cs in check_statuses:
check_lines = [
' ' + cs.name + ' ({})'.format(cs.check_version),
' ' + '-' * (len(cs.name) + 3 + len(cs.check_version))
]
if cs.init_failed_error:
check_lines.append(" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'),
repr(cs.init_failed_error)))
if self.verbose and cs.init_failed_traceback:
check_lines.extend(' ' + line for line in
cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (
s.instance_id, style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get('run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in
warn[1:])
if self.verbose and s.traceback is not None:
check_lines.extend(' ' + line for line in
s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s" % (
cs.metric_count, plural(cs.metric_count),
cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [
" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [
" - %s: %s" % (library, version)]
check_lines += [""]
lines += check_lines
# Metadata status
metadata_enabled = _is_affirmative(get_config().get('display_service_metadata', False))
if metadata_enabled:
lines += [
"",
"Service metadata",
"================",
""
]
if not check_statuses:
lines.append(" No checks have run yet.")
else:
meta_lines = []
for cs in check_statuses:
# Check title
check_line = [
' ' + cs.name,
' ' + '-' * len(cs.name)
]
instance_lines = []
for i, meta in enumerate(cs.service_metadata):
if not meta:
continue
instance_lines += [" - instance #%s:" % i]
for k, v in meta.iteritems():
instance_lines += [" - %s: %s" % (k, v)]
if instance_lines:
check_line += instance_lines
meta_lines += check_line
if meta_lines:
lines += meta_lines
else:
lines.append(" No metadata were collected.")
# Emitter status
lines += [
"",
"Emitters",
"========",
""
]
if not self.emitter_statuses:
lines.append(" No emitters have run yet.")
else:
for es in self.emitter_statuses:
c = 'green'
if es.has_error():
c = 'red'
line = " - %s [%s]" % (es.name, style(es.status, c))
if es.status != STATUS_OK:
line += ": %s" % es.error
lines.append(line)
return lines
except Exception, e:
self.log.exception("Check '%s' instance #%s failed" % (self.name, i))
instance_status = check_status.InstanceStatus(
i, check_status.STATUS_ERROR,
error=str(e), tb=traceback.format_exc()
)
finally:
self._roll_up_instance_metadata()
instance_statuses.append(instance_status)
if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME:
try:
after = AgentCheck._collect_internal_stats()
self._set_internal_profiling_stats(before, after)
log.info("\n \t %s %s" % (self.name, pretty_statistics(self._internal_profiling_stats)))
except Exception: # It's fine if we can't collect stats for the run, just log and proceed
self.log.debug("Failed to collect Agent Stats after check {0}".format(self.name))
return instance_statuses
def check(self, instance):
"""
Overriden by the check class. This will be called to run the check.
:param instance: A dict with the instance information. This will vary
depending on your config structure.
"""
raise NotImplementedError()
def stop(self):
check_lines.extend(
' ' + line
for line in s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s"
% (cs.metric_count, plural(
cs.metric_count), cs.event_count,
plural(cs.event_count), cs.service_check_count,
plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" %
pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [" - Dependencies:"]
for library, version in cs.library_versions.iteritems(
):
check_lines += [
" - %s: %s" % (library, version)
]
check_lines += [""]
lines += check_lines
# Metadata status