def _postMetrics(self):
if len(self._metrics) > 0:
self._metrics['uuid'] = get_uuid()
self._metrics['internalHostname'] = get_hostname(self._agentConfig)
self._metrics['apiKey'] = self._agentConfig['api_key']
MetricTransaction(json.dumps(self._metrics),
headers={'Content-Type': 'application/json'})
self._metrics = {}
def serialize_metrics(metrics, hostname, ip):
try:
metrics.append(add_serialization_status_metric("success", hostname))
serialized = json.dumps({
"series": metrics,
'uuid': get_uuid(),
'ip': ip
})
except UnicodeDecodeError as e:
log.exception(
"Unable to serialize payload. Trying to replace bad characters. %s",
e)
metrics.append(add_serialization_status_metric("failure", hostname))
try:
log.error(metrics)
serialized = json.dumps({
"series": unicode_metrics(metrics),
'uuid': get_uuid(),
'ip': ip
})
except Exception as e:
log.exception("Unable to serialize payload. Giving up. %s", e)
serialized = json.dumps({
"series": [
add_serialization_status_metric("permanent_failure",
hostname)
],
'uuid':
get_uuid(),
'ip':
ip
})
if len(serialized) > COMPRESS_THRESHOLD:
headers = {
'Content-Type': 'application/json',
'Content-Encoding': 'deflate'
}
serialized = zlib.compress(serialized)
else:
headers = {'Content-Type': 'application/json'}
return serialized, headers
def submit_service_checks(self, service_checks):
headers = {'Content-Type': 'application/json'}
params = {}
if self.api_key:
params['api_key'] = self.api_key
for check in service_checks:
check['uuid'] = get_uuid()
check['ip'] = self.ip
url = '{0}/api/v1/check_run?{1}'.format(self.api_host,
urlencode(params))
self.submit_http(url, json.dumps(service_checks), headers)
def submit_events(self, events):
headers = {'Content-Type': 'application/json'}
event_chunk_size = self.event_chunk_size
for chunk in chunks(events, event_chunk_size):
payload = {
'apiKey': self.api_key,
'events': {
'api': chunk
},
'uuid': get_uuid(),
'ip': self.ip,
'internalHostname': get_hostname()
}
params = {}
if self.api_key:
params['api_key'] = self.api_key
url = '%s/intake?%s' % (self.api_host, urlencode(params))
self.submit_http(url, json.dumps(payload), headers)
def _build_payload(self, payload):
"""
Build the payload skeleton, so it contains all of the generic payload data.
"""
now = time.time()
payload['ip'] = self.ip
payload['collection_timestamp'] = now
payload['os'] = self.os
payload['python'] = sys.version
payload['agentVersion'] = self.agent_config['version']
payload['apiKey'] = self.agent_config['api_key']
payload['events'] = {}
payload['metrics'] = []
payload['service_checks'] = []
payload['resources'] = {}
payload['internalHostname'] = self.hostname
payload['uuid'] = get_uuid()
payload['host-tags'] = {}
payload['external_host_tags'] = {}
def to_dict(self):
status_info = AgentStatus.to_dict(self)
# Hostnames
status_info['hostnames'] = {}
metadata_whitelist = ['hostname', 'fqdn', 'ipv4', 'instance-id']
if self.host_metadata:
for key, host in self.host_metadata.iteritems():
for whitelist_item in metadata_whitelist:
if whitelist_item in key:
status_info['hostnames'][key] = host
break
# Checks.d Status
status_info['checks'] = {}
check_statuses = self.check_statuses + get_jmx_status()
for cs in check_statuses:
status_info['checks'][cs.name] = {'instances': {}}
if cs.init_failed_error:
status_info['checks'][cs.name]['init_failed'] = True
status_info['checks'][cs.name]['traceback'] = \
cs.init_failed_traceback or cs.init_failed_error
else:
status_info['checks'][cs.name] = {'instances': {}}
status_info['checks'][cs.name]['init_failed'] = False
for s in cs.instance_statuses:
status_info['checks'][cs.name]['instances'][
s.instance_id] = {
'status': s.status,
'has_error': s.has_error(),
'has_warnings': s.has_warnings(),
}
if s.has_error():
status_info['checks'][cs.name]['instances'][
s.instance_id]['error'] = s.error
if s.has_warnings():
status_info['checks'][cs.name]['instances'][
s.instance_id]['warnings'] = s.warnings
status_info['checks'][
cs.name]['metric_count'] = cs.metric_count
status_info['checks'][cs.name]['event_count'] = cs.event_count
status_info['checks'][
cs.name]['service_check_count'] = cs.service_check_count
# Emitter status
status_info['emitter'] = []
for es in self.emitter_statuses:
check_status = {
'name': es.name,
'status': es.status,
'has_error': es.has_error(),
}
if es.has_error():
check_status['error'] = es.error
status_info['emitter'].append(check_status)
osname = config.get_os()
try:
status_info['confd_path'] = config.get_confd_path(osname)
except config.PathNotFound:
status_info['confd_path'] = 'Not found'
try:
status_info['checksd_path'] = config.get_checksd_path(osname)
except config.PathNotFound:
status_info['checksd_path'] = 'Not found'
# Clocks
try:
ntp_offset, ntp_style = get_ntp_info()
warn_ntp = len(ntp_style) > 0
status_info["ntp_offset"] = round(ntp_offset, 4)
except Exception as e:
ntp_offset = "Unknown (%s)" % str(e)
warn_ntp = True
status_info["ntp_offset"] = ntp_offset
status_info["ntp_warning"] = warn_ntp
status_info["utc_time"] = datetime.datetime.utcnow().__str__()
# uuid
try:
uuid = str(get_uuid())
except Exception as e:
uuid = "Unknown (%s)" % str(e)
status_info["uuid"] = uuid
return status_info
class CollectorStatus(AgentStatus):
NAME = 'Collector'
def __init__(self,
check_statuses=None,
emitter_statuses=None,
metadata=None):
AgentStatus.__init__(self)
self.check_statuses = check_statuses or []
self.emitter_statuses = emitter_statuses or []
self.host_metadata = metadata or []
@property
def status(self):
for check_status in self.check_statuses:
if check_status.status == STATUS_ERROR:
return STATUS_ERROR
return STATUS_OK
def has_error(self):
return self.status != STATUS_OK
@staticmethod
def check_status_lines(cs):
check_lines = [' ' + cs.name, ' ' + '-' * len(cs.name)]
if cs.init_failed_error:
check_lines.append(
" - initialize check class [%s]: %s" %
(style(STATUS_ERROR, 'red'), repr(cs.init_failed_error)))
if cs.init_failed_traceback:
check_lines.extend(
' ' + line
for line in cs.init_failed_traceback.split('\n'))
else:
for s in cs.instance_statuses:
c = 'green'
if s.has_warnings():
c = 'yellow'
if s.has_error():
c = 'red'
line = " - instance #%s [%s]" % (s.instance_id,
style(s.status, c))
if s.has_error():
line += u": %s" % s.error
if s.metric_count is not None:
line += " collected %s metrics" % s.metric_count
if s.instance_check_stats is not None:
line += " Last run duration: %s" % s.instance_check_stats.get(
'run_time')
check_lines.append(line)
if s.has_warnings():
for warning in s.warnings:
warn = warning.split('\n')
if not len(warn):
continue
check_lines.append(
u" %s: %s" %
(style("Warning", 'yellow'), warn[0]))
check_lines.extend(u" %s" % l for l in warn[1:])
if s.traceback is not None:
check_lines.extend(' ' + line
for line in s.traceback.split('\n'))
check_lines += [
" - Collected %s metric%s, %s event%s & %s service check%s"
% (cs.metric_count, plural(
cs.metric_count), cs.event_count, plural(cs.event_count),
cs.service_check_count, plural(cs.service_check_count)),
]
if cs.check_stats is not None:
check_lines += [
" - Stats: %s" % pretty_statistics(cs.check_stats)
]
if cs.library_versions is not None:
check_lines += [" - Dependencies:"]
for library, version in cs.library_versions.iteritems():
check_lines += [" - %s: %s" % (library, version)]
check_lines += [""]
return check_lines
@staticmethod
def render_check_status(cs):
indent = " "
lines = [indent + l
for l in CollectorStatus.check_status_lines(cs)] + ["", ""]
return "\n".join(lines)
def body_lines(self):
# Metadata whitelist
metadata_whitelist = ['hostname', 'fqdn', 'ipv4', 'instance-id']
lines = ['Clocks', '======', '']
try:
ntp_offset, ntp_styles = get_ntp_info()
lines.append(' ' + style('NTP offset', *ntp_styles) + ': ' +
style('%s s' % round(ntp_offset, 4), *ntp_styles))
except Exception, e:
lines.append(' NTP offset: Unknown (%s)' % str(e))
lines.append(' System UTC time: ' +
datetime.datetime.utcnow().__str__())
lines.append('')
# uuid
lines = ['UUID', '======', '']
try:
uuid = get_uuid()
lines.append(' System uuid: ' + str(uuid))
except Exception, e:
lines.append(' System uuid: Unknown (%s)' % str(e))