def test_checksd_aggregation_perf(self):
ma = MetricsAggregator('my.host')
for _ in xrange(self.FLUSH_COUNT):
for i in xrange(self.LOOPS_PER_FLUSH):
# Counters
for j in xrange(self.METRIC_COUNT):
ma.increment('counter.%s' % j, i)
ma.gauge('gauge.%s' % j, i)
ma.histogram('histogram.%s' % j, i)
ma.set('set.%s' % j, float(i))
ma.flush()
def test_gauge(self):
stats = MetricsAggregator('myhost')
# Track some counters.
stats.submit_packets('my.first.gauge:1|g')
stats.submit_packets('my.first.gauge:5|g')
stats.submit_packets('my.second.gauge:1.5|g')
# Ensure that gauges roll up correctly.
metrics = self.sort_metrics(stats.flush())
assert len(metrics) == 2
first, second = metrics
nt.assert_equals(first['metric'], 'my.first.gauge')
nt.assert_equals(first['points'][0][1], 5)
nt.assert_equals(first['host'], 'myhost')
nt.assert_equals(second['metric'], 'my.second.gauge')
nt.assert_equals(second['points'][0][1], 1.5)
# Ensure that old gauges get dropped due to old timestamps
stats.gauge('my.first.gauge', 5)
stats.gauge('my.first.gauge', 1, timestamp=1000000000)
stats.gauge('my.second.gauge', 20, timestamp=1000000000)
metrics = self.sort_metrics(stats.flush())
assert len(metrics) == 1
first = metrics[0]
nt.assert_equals(first['metric'], 'my.first.gauge')
nt.assert_equals(first['points'][0][1], 5)
nt.assert_equals(first['host'], 'myhost')
def test_gauge(self):
stats = MetricsAggregator('myhost')
# Track some counters.
stats.submit_packets('my.first.gauge:1|g')
stats.submit_packets('my.first.gauge:5|g')
stats.submit_packets('my.second.gauge:1.5|g')
# Ensure that gauges roll up correctly.
metrics = self.sort_metrics(stats.flush())
assert len(metrics) == 2
first, second = metrics
nt.assert_equals(first['metric'], 'my.first.gauge')
nt.assert_equals(first['points'][0][1], 5)
nt.assert_equals(first['host'], 'myhost')
nt.assert_equals(second['metric'], 'my.second.gauge')
nt.assert_equals(second['points'][0][1], 1.5)
# Ensure that old gauges get dropped due to old timestamps
stats.gauge('my.first.gauge', 5)
stats.gauge('my.first.gauge', 1, timestamp=1000000000)
stats.gauge('my.second.gauge', 20, timestamp=1000000000)
metrics = self.sort_metrics(stats.flush())
assert len(metrics) == 1
first = metrics[0]
nt.assert_equals(first['metric'], 'my.first.gauge')
nt.assert_equals(first['points'][0][1], 5)
nt.assert_equals(first['host'], 'myhost')
class AgentCheck(object):
def __init__(self, name, init_config, agent_config, instances=None):
"""Initialize a new check.
:param name: The name of the check
:param init_config: The config for initializing the check
:param agent_config: The global configuration for the agent
:param instances: A list of configuration objects for each instance.
"""
self.name = name
self.init_config = init_config
self.agent_config = agent_config
self.hostname = monagent.common.util.get_hostname(agent_config)
self.log = logging.getLogger('%s.%s' % (__name__, name))
self.aggregator = MetricsAggregator(self.hostname,
recent_point_threshold=agent_config.get('recent_point_threshold',
None))
self.events = []
self.instances = instances or []
self.warnings = []
self.library_versions = None
def instance_count(self):
"""Return the number of instances that are configured for this check.
"""
return len(self.instances)
def gauge(self, metric, value, dimensions=None, delegated_tenant=None,
hostname=None, device_name=None, timestamp=None):
"""Record the value of a gauge, with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value of the gauge
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
:param timestamp: (optional) The timestamp for this metric value
"""
self.aggregator.gauge(metric,
value,
self._set_dimensions(dimensions),
delegated_tenant,
hostname,
device_name,
timestamp)
def increment(self, metric, value=1, dimensions=None, delegated_tenant=None, hostname=None, device_name=None):
"""Increment a counter with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value to increment by
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.increment(metric,
value,
self._set_dimensions(dimensions),
delegated_tenant,
hostname,
device_name)
def decrement(self, metric, value=-1, dimensions=None, delegated_tenant=None, hostname=None, device_name=None):
"""Decrement a counter with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value to decrement by
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.decrement(metric,
value,
self._set_dimensions(dimensions),
delegated_tenant,
hostname,
device_name)
def rate(self, metric, value, dimensions=None, delegated_tenant=None, hostname=None, device_name=None):
"""Submit a point for a metric that will be calculated as a rate on flush.
Values will persist across each call to `check` if there is not enough
point to generate a rate on the flush.
:param metric: The name of the metric
:param value: The value of the rate
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.rate(metric,
value,
self._set_dimensions(dimensions),
delegated_tenant,
hostname,
device_name)
def histogram(self, metric, value, dimensions=None, delegated_tenant=None, hostname=None, device_name=None):
"""Sample a histogram value, with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value to sample for the histogram
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.histogram(metric,
value,
self._set_dimensions(dimensions),
delegated_tenant,
hostname,
device_name)
def set(self, metric, value, dimensions=None, delegated_tenant=None, hostname=None, device_name=None):
"""Sample a set value, with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value for the set
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.set(metric,
value,
self._set_dimensions(dimensions),
delegated_tenant,
hostname,
device_name)
def _set_dimensions(self, dimensions):
new_dimensions = {'component': 'monasca-agent', 'service': 'monitoring'}
if dimensions is not None:
new_dimensions.update(dimensions.copy())
return new_dimensions
def event(self, event):
"""Save an event.
:param event: The event payload as a dictionary. Has the following
structure:
{
"timestamp": int, the epoch timestamp for the event,
"event_type": string, the event time name,
"api_key": string, the api key of the account to associate the event with,
"msg_title": string, the title of the event,
"msg_text": string, the text body of the event,
"alert_type": (optional) string, one of ('error', 'warning', 'success', 'info').
Defaults to 'info'.
"source_type_name": (optional) string, the source type name,
"host": (optional) string, the name of the host,
"dimensions": (optional) a dictionary of dimensions to associate with this event
}
"""
if event.get('api_key') is None:
event['api_key'] = self.agent_config['api_key']
self.events.append(event)
def has_events(self):
"""Check whether the check has saved any events
@return whether or not the check has saved any events
@rtype boolean
"""
return len(self.events) > 0
def get_metrics(self, prettyprint=False):
"""Get all metrics, including the ones that are tagged.
@return the list of samples
@rtype list of Measurement objects from monagent.common.metrics
"""
if prettyprint:
metrics = self.aggregator.flush()
for metric in metrics:
print(" Timestamp: {}".format(metric.timestamp))
print(" Name: {}".format(metric.name))
print(" Value: {}".format(metric.value))
if (metric.delegated_tenant):
print(" Delegtd ID: {}".format(metric.delegated_tenant))
print(" Dimensions: ", end='')
line = 0
for name in metric.dimensions:
if line != 0:
print(" " * 13, end='')
print("{0}={1}".format(name, metric.dimensions[name]))
line += 1
print("-" * 24)
return self.aggregator.flush()
def get_events(self):
"""Return a list of the events saved by the check, if any
@return the list of events saved by this check
@rtype list of event dictionaries
"""
events = self.events
self.events = []
return events
def has_warnings(self):
"""Check whether the instance run created any warnings.
"""
return len(self.warnings) > 0
def warning(self, warning_message):
"""Add a warning message that will be printed in the info page
:param warning_message: String. Warning message to be displayed
"""
self.warnings.append(warning_message)
def get_library_info(self):
if self.library_versions is not None:
return self.library_versions
try:
self.library_versions = self.get_library_versions()
except NotImplementedError:
pass
def get_library_versions(self):
"""Should return a string that shows which version
of the needed libraries are used
"""
raise NotImplementedError
def get_warnings(self):
"""Return the list of warnings messages to be displayed in the info page.
"""
warnings = self.warnings
self.warnings = []
return warnings
def run(self):
"""Run all instances.
"""
instance_statuses = []
for i, instance in enumerate(self.instances):
try:
self.check(instance)
if self.has_warnings():
instance_status = monagent.common.check_status.InstanceStatus(i,
monagent.common.check_status.STATUS_WARNING,
warnings=self.get_warnings())
else:
instance_status = monagent.common.check_status.InstanceStatus(i,
monagent.common.check_status.STATUS_OK)
except Exception as e:
self.log.exception("Check '%s' instance #%s failed" % (self.name, i))
instance_status = monagent.common.check_status.InstanceStatus(i,
monagent.common.check_status.STATUS_ERROR,
error=e,
tb=traceback.format_exc())
instance_statuses.append(instance_status)
return instance_statuses
def check(self, instance):
"""Overriden by the check class. This will be called to run the check.
:param instance: A dict with the instance information. This will vary
depending on your config structure.
"""
raise NotImplementedError()
@staticmethod
def stop():
"""To be executed when the agent is being stopped to clean ressources.
"""
pass
@classmethod
def from_yaml(cls, path_to_yaml=None, agentConfig=None, yaml_text=None, check_name=None):
"""A method used for testing your check without running the agent.
"""
if hasattr(yaml, 'CLoader'):
Loader = yaml.CLoader
else:
Loader = yaml.Loader
if path_to_yaml:
check_name = os.path.basename(path_to_yaml).split('.')[0]
try:
f = open(path_to_yaml)
except IOError:
raise Exception('Unable to open yaml config: %s' % path_to_yaml)
yaml_text = f.read()
f.close()
config = yaml.load(yaml_text, Loader=Loader)
check = cls(check_name, config.get('init_config') or {}, agentConfig or {})
return check, config.get('instances', [])
@staticmethod
def normalize(metric, prefix=None):
"""Turn a metric into a well-formed metric name prefix.b.c
:param metric The metric name to normalize
:param prefix A prefix to to add to the normalized name, default None
"""
name = re.sub(r"[,\+\*\-/()\[\]{}]", "_", metric)
# Eliminate multiple _
name = re.sub(r"__+", "_", name)
# Don't start/end with _
name = re.sub(r"^_", "", name)
name = re.sub(r"_$", "", name)
# Drop ._ and _.
name = re.sub(r"\._", ".", name)
name = re.sub(r"_\.", ".", name)
if prefix is not None:
return prefix + "." + name
else:
return name
@staticmethod
def read_config(instance, key, message=None, cast=None, optional=False):
val = instance.get(key)
if val is None:
if optional is False:
message = message or 'Must provide `%s` value in instance config' % key
raise Exception(message)
else:
return val
if cast is None:
return val
else:
return cast(val)
class AgentCheck(object):
def __init__(self, name, init_config, agent_config, instances=None):
"""Initialize a new check.
:param name: The name of the check
:param init_config: The config for initializing the check
:param agent_config: The global configuration for the agent
:param instances: A list of configuration objects for each instance.
"""
self.name = name
self.init_config = init_config
self.agent_config = agent_config
self.hostname = monagent.common.util.get_hostname(agent_config)
self.log = logging.getLogger('%s.%s' % (__name__, name))
self.aggregator = MetricsAggregator(
self.hostname,
recent_point_threshold=agent_config.get('recent_point_threshold',
None))
self.events = []
self.instances = instances or []
self.warnings = []
self.library_versions = None
def instance_count(self):
"""Return the number of instances that are configured for this check.
"""
return len(self.instances)
def gauge(self,
metric,
value,
dimensions=None,
delegated_tenant=None,
hostname=None,
device_name=None,
timestamp=None):
"""Record the value of a gauge, with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value of the gauge
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
:param timestamp: (optional) The timestamp for this metric value
"""
self.aggregator.gauge(metric, value, self._set_dimensions(dimensions),
delegated_tenant, hostname, device_name,
timestamp)
def increment(self,
metric,
value=1,
dimensions=None,
delegated_tenant=None,
hostname=None,
device_name=None):
"""Increment a counter with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value to increment by
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.increment(metric, value,
self._set_dimensions(dimensions),
delegated_tenant, hostname, device_name)
def decrement(self,
metric,
value=-1,
dimensions=None,
delegated_tenant=None,
hostname=None,
device_name=None):
"""Decrement a counter with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value to decrement by
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.decrement(metric, value,
self._set_dimensions(dimensions),
delegated_tenant, hostname, device_name)
def rate(self,
metric,
value,
dimensions=None,
delegated_tenant=None,
hostname=None,
device_name=None):
"""Submit a point for a metric that will be calculated as a rate on flush.
Values will persist across each call to `check` if there is not enough
point to generate a rate on the flush.
:param metric: The name of the metric
:param value: The value of the rate
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.rate(metric, value, self._set_dimensions(dimensions),
delegated_tenant, hostname, device_name)
def histogram(self,
metric,
value,
dimensions=None,
delegated_tenant=None,
hostname=None,
device_name=None):
"""Sample a histogram value, with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value to sample for the histogram
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.histogram(metric, value,
self._set_dimensions(dimensions),
delegated_tenant, hostname, device_name)
def set(self,
metric,
value,
dimensions=None,
delegated_tenant=None,
hostname=None,
device_name=None):
"""Sample a set value, with optional dimensions, hostname and device name.
:param metric: The name of the metric
:param value: The value for the set
:param dimensions: (optional) A dictionary of dimensions for this metric
:param delegated_tenant: (optional) Submit metrics on behalf of this tenant ID.
:param hostname: (optional) A hostname for this metric. Defaults to the current hostname.
:param device_name: (optional) The device name for this metric
"""
self.aggregator.set(metric, value, self._set_dimensions(dimensions),
delegated_tenant, hostname, device_name)
def _set_dimensions(self, dimensions):
new_dimensions = {
'component': 'monasca-agent',
'service': 'monitoring'
}
if dimensions is not None:
new_dimensions.update(dimensions.copy())
return new_dimensions
def event(self, event):
"""Save an event.
:param event: The event payload as a dictionary. Has the following
structure:
{
"timestamp": int, the epoch timestamp for the event,
"event_type": string, the event time name,
"api_key": string, the api key of the account to associate the event with,
"msg_title": string, the title of the event,
"msg_text": string, the text body of the event,
"alert_type": (optional) string, one of ('error', 'warning', 'success', 'info').
Defaults to 'info'.
"source_type_name": (optional) string, the source type name,
"host": (optional) string, the name of the host,
"dimensions": (optional) a dictionary of dimensions to associate with this event
}
"""
if event.get('api_key') is None:
event['api_key'] = self.agent_config['api_key']
self.events.append(event)
def has_events(self):
"""Check whether the check has saved any events
@return whether or not the check has saved any events
@rtype boolean
"""
return len(self.events) > 0
def get_metrics(self, prettyprint=False):
"""Get all metrics, including the ones that are tagged.
@return the list of samples
@rtype list of Measurement objects from monagent.common.metrics
"""
if prettyprint:
metrics = self.aggregator.flush()
for metric in metrics:
print(" Timestamp: {}".format(metric.timestamp))
print(" Name: {}".format(metric.name))
print(" Value: {}".format(metric.value))
if (metric.delegated_tenant):
print(" Delegtd ID: {}".format(metric.delegated_tenant))
print(" Dimensions: ", end='')
line = 0
for name in metric.dimensions:
if line != 0:
print(" " * 13, end='')
print("{0}={1}".format(name, metric.dimensions[name]))
line += 1
print("-" * 24)
return self.aggregator.flush()
def get_events(self):
"""Return a list of the events saved by the check, if any
@return the list of events saved by this check
@rtype list of event dictionaries
"""
events = self.events
self.events = []
return events
def has_warnings(self):
"""Check whether the instance run created any warnings.
"""
return len(self.warnings) > 0
def warning(self, warning_message):
"""Add a warning message that will be printed in the info page
:param warning_message: String. Warning message to be displayed
"""
self.warnings.append(warning_message)
def get_library_info(self):
if self.library_versions is not None:
return self.library_versions
try:
self.library_versions = self.get_library_versions()
except NotImplementedError:
pass
def get_library_versions(self):
"""Should return a string that shows which version
of the needed libraries are used
"""
raise NotImplementedError
def get_warnings(self):
"""Return the list of warnings messages to be displayed in the info page.
"""
warnings = self.warnings
self.warnings = []
return warnings
def run(self):
"""Run all instances.
"""
instance_statuses = []
for i, instance in enumerate(self.instances):
try:
self.check(instance)
if self.has_warnings():
instance_status = monagent.common.check_status.InstanceStatus(
i,
monagent.common.check_status.STATUS_WARNING,
warnings=self.get_warnings())
else:
instance_status = monagent.common.check_status.InstanceStatus(
i, monagent.common.check_status.STATUS_OK)
except Exception as e:
self.log.exception("Check '%s' instance #%s failed" %
(self.name, i))
instance_status = monagent.common.check_status.InstanceStatus(
i,
monagent.common.check_status.STATUS_ERROR,
error=e,
tb=traceback.format_exc())
instance_statuses.append(instance_status)
return instance_statuses
def check(self, instance):
"""Overriden by the check class. This will be called to run the check.
:param instance: A dict with the instance information. This will vary
depending on your config structure.
"""
raise NotImplementedError()
@staticmethod
def stop():
"""To be executed when the agent is being stopped to clean ressources.
"""
pass
@classmethod
def from_yaml(cls,
path_to_yaml=None,
agentConfig=None,
yaml_text=None,
check_name=None):
"""A method used for testing your check without running the agent.
"""
if hasattr(yaml, 'CLoader'):
Loader = yaml.CLoader
else:
Loader = yaml.Loader
if path_to_yaml:
check_name = os.path.basename(path_to_yaml).split('.')[0]
try:
f = open(path_to_yaml)
except IOError:
raise Exception('Unable to open yaml config: %s' %
path_to_yaml)
yaml_text = f.read()
f.close()
config = yaml.load(yaml_text, Loader=Loader)
check = cls(check_name,
config.get('init_config') or {}, agentConfig or {})
return check, config.get('instances', [])
@staticmethod
def normalize(metric, prefix=None):
"""Turn a metric into a well-formed metric name prefix.b.c
:param metric The metric name to normalize
:param prefix A prefix to to add to the normalized name, default None
"""
name = re.sub(r"[,\+\*\-/()\[\]{}]", "_", metric)
# Eliminate multiple _
name = re.sub(r"__+", "_", name)
# Don't start/end with _
name = re.sub(r"^_", "", name)
name = re.sub(r"_$", "", name)
# Drop ._ and _.
name = re.sub(r"\._", ".", name)
name = re.sub(r"_\.", ".", name)
if prefix is not None:
return prefix + "." + name
else:
return name
@staticmethod
def read_config(instance, key, message=None, cast=None, optional=False):
val = instance.get(key)
if val is None:
if optional is False:
message = message or 'Must provide `%s` value in instance config' % key
raise Exception(message)
else:
return val
if cast is None:
return val
else:
return cast(val)