def test_checksd_aggregation_perf(self): ma = MetricsAggregator('my.host') for _ in xrange(self.FLUSH_COUNT): for i in xrange(self.LOOPS_PER_FLUSH): # Counters for j in xrange(self.METRIC_COUNT): ma.increment('counter.%s' % j, i) ma.gauge('gauge.%s' % j, i) ma.histogram('histogram.%s' % j, i) ma.set('set.%s' % j, float(i)) ma.flush()
class AgentCheck(object): OK, WARNING, CRITICAL, UNKNOWN = (0, 1, 2, 3) SOURCE_TYPE_NAME = None DEFAULT_MIN_COLLECTION_INTERVAL = 0 def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self.name = name self.init_config = init_config or {} self.agentConfig = agentConfig self.in_developer_mode = agentConfig.get( 'developer_mode') and psutil is not None self._internal_profiling_stats = None self.hostname = agentConfig.get('checksd_hostname') or get_hostname( agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.aggregator = MetricsAggregator( self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None), histogram_aggregates=agentConfig.get('histogram_aggregates'), histogram_percentiles=agentConfig.get('histogram_percentiles')) self.events = [] self.service_checks = [] self.instances = instances or [] self.warnings = [] self.library_versions = None self.last_collection_time = defaultdict(int) self._instance_metadata = [] self.svc_metadata = [] def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submit a raw count with optional tags, hostname and device name :param metric: The name of the metric :param value: The value :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.submit_count(metric, value, tags, hostname, device_name) def monotonic_count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submits a raw count with optional tags, hostname and device name based on increasing counter values. E.g. 1, 3, 5, 7 will submit 6 on flush. Note that reset counters are skipped. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.count_from_counter(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "agent_key": string, the api key of the account to associate the event with, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ # Events are disabled. return if event.get('agent_key') is None: event['agent_key'] = self.agentConfig['agent_key'] self.events.append(event) def service_check(self, check_name, status, tags=None, timestamp=None, hostname=None, check_run_id=None, message=None): """ Save a service check. :param check_name: string, name of the service check :param status: int, describing the status. 0 for success, 1 for warning, 2 for failure :param tags: (optional) list of strings, a list of tags for this run :param timestamp: (optional) float, unix timestamp for when the run occurred :param hostname: (optional) str, host that generated the service check. Defaults to the host_name of the agent :param check_run_id: (optional) int, id used for logging and tracing purposes. Don't need to be unique. If not specified, one will be generated. """ if hostname is None: hostname = self.hostname if message is not None: message = str(message) self.service_checks.append( create_service_check(check_name, status, tags, timestamp, hostname, check_run_id, message)) def service_metadata(self, meta_name, value): """ Save metadata. :param meta_name: metadata key name :type meta_name: string :param value: metadata value :type value: string """ self._instance_metadata.append((meta_name, str(value))) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def get_service_checks(self): """ Return a list of the service checks saved by the check, if any and clears them out of the instance's service_checks list @return the list of service checks saved by this check @rtype list of service check dicts """ service_checks = self.service_checks self.service_checks = [] return service_checks def _roll_up_instance_metadata(self): """ Concatenate and flush instance metadata. """ self.svc_metadata.append( dict((k, v) for (k, v) in self._instance_metadata)) self._instance_metadata = [] def get_service_metadata(self): """ Return a list of the metadata dictionaries saved by the check -if any- and clears them out of the instance's service_checks list @return the list of metadata saved by this check @rtype list of metadata dicts """ if self._instance_metadata: self._roll_up_instance_metadata() service_metadata = self.svc_metadata self.svc_metadata = [] return service_metadata def has_warnings(self): """ Check whether the instance run created any warnings """ return len(self.warnings) > 0 def warning(self, warning_message): """ Add a warning message that will be printed in the info page :param warning_message: String. Warning message to be displayed """ self.warnings.append(str(warning_message)) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): """ Should return a string that shows which version of the needed libraries are used """ raise NotImplementedError def get_warnings(self): """ Return the list of warnings messages to be displayed in the info page """ warnings = self.warnings self.warnings = [] return warnings @staticmethod def _get_statistic_name_from_method(method_name): return method_name[4:] if method_name.startswith( 'get_') else method_name @staticmethod def _collect_internal_stats(methods=None): current_process = psutil.Process(os.getpid()) methods = methods or DEFAULT_PSUTIL_METHODS filtered_methods = [m for m in methods if hasattr(current_process, m)] stats = {} for method in filtered_methods: # Go from `get_memory_info` -> `memory_info` stat_name = AgentCheck._get_statistic_name_from_method(method) try: raw_stats = getattr(current_process, method)() try: stats[stat_name] = raw_stats._asdict() except AttributeError: if isinstance(raw_stats, numbers.Number): stats[stat_name] = raw_stats else: log.warn( "Could not serialize output of {0} to dict".format( method)) except psutil.AccessDenied: log.warn( "Cannot call psutil method {0} : Access Denied".format( method)) return stats def _set_internal_profiling_stats(self, before, after): self._internal_profiling_stats = {'before': before, 'after': after} def _get_internal_profiling_stats(self): """ If in developer mode, return a dictionary of statistics about the check run """ stats = self._internal_profiling_stats self._internal_profiling_stats = None return stats def run(self): """ Run all instances. """ # Store run statistics if needed before, after = None, None if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: before = AgentCheck._collect_internal_stats() except Exception: # It's fine if we can't collect stats for the run, just log and proceed self.log.debug( "Failed to collect Agent Stats before check {0}".format( self.name)) instance_statuses = [] for i, instance in enumerate(self.instances): try: min_collection_interval = instance.get( 'min_collection_interval', self.init_config.get('min_collection_interval', self.DEFAULT_MIN_COLLECTION_INTERVAL)) now = time.time() if now - self.last_collection_time[i] < min_collection_interval: self.log.debug( "Not running instance #{0} of check {1} as it ran less than {2}s ago" .format(i, self.name, min_collection_interval)) continue self.last_collection_time[i] = now check_start_time = None if self.in_developer_mode: check_start_time = timeit.default_timer() self.check(copy.deepcopy(instance)) instance_check_stats = None if check_start_time is not None: instance_check_stats = { 'run_time': timeit.default_timer() - check_start_time } if self.has_warnings(): instance_status = check_status.InstanceStatus( i, check_status.STATUS_WARNING, warnings=self.get_warnings(), instance_check_stats=instance_check_stats) else: instance_status = check_status.InstanceStatus( i, check_status.STATUS_OK, instance_check_stats=instance_check_stats) except Exception, e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus( i, check_status.STATUS_ERROR, error=str(e), tb=traceback.format_exc()) finally:
class AgentCheck(object): def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self.name = name self.init_config = init_config self.agentConfig = agentConfig self.hostname = gethostname(agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.aggregator = MetricsAggregator(self.hostname, formatter=agent_formatter) self.events = [] self.instances = instances or [] def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "api_key": string, the api key of the account to associate the event with, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ self.events.append(event) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def run(self): """ Run all instances. """ instance_statuses = [] for i, instance in enumerate(self.instances): try: self.check(instance) instance_status = check_status.InstanceStatus(i, check_status.STATUS_OK) except Exception, e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) # Send the traceback (located at sys.exc_info()[2]) into the InstanceStatus otherwise a traceback won't be able to be printed instance_status = check_status.InstanceStatus(i, check_status.STATUS_ERROR, e, sys.exc_info()[2]) instance_statuses.append(instance_status) return instance_statuses
class AgentCheck(object): OK, WARNING, CRITICAL, UNKNOWN = (0, 1, 2, 3) SOURCE_TYPE_NAME = None DEFAULT_EXPIRY_SECONDS = 300 DEFAULT_MIN_COLLECTION_INTERVAL = 0 _enabled_checks = [] @classmethod def is_check_enabled(cls, name): return name in cls._enabled_checks def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self._enabled_checks.append(name) self._enabled_checks = list(set(self._enabled_checks)) self.name = name self.init_config = init_config or {} self.agentConfig = agentConfig self.in_developer_mode = agentConfig.get('developer_mode') and psutil self._internal_profiling_stats = None self.default_integration_http_timeout = float( agentConfig.get('default_integration_http_timeout', 9)) self.hostname = agentConfig.get('checksd_hostname') or get_hostname( agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.min_collection_interval = self.init_config.get( 'min_collection_interval', self.DEFAULT_MIN_COLLECTION_INTERVAL) self.aggregator = MetricsAggregator( self.hostname, expiry_seconds=self.min_collection_interval + self.DEFAULT_EXPIRY_SECONDS, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None), histogram_aggregates=agentConfig.get('histogram_aggregates'), histogram_percentiles=agentConfig.get('histogram_percentiles')) self.events = [] self.service_checks = [] self.instances = instances or [] self.warnings = [] self.library_versions = None self.last_collection_time = defaultdict(int) self._instance_metadata = [] self.svc_metadata = [] self.historate_dict = {} # Set proxy settings self.proxy_settings = get_proxy(self.agentConfig) self._use_proxy = False if init_config is None else init_config.get( "use_agent_proxy", True) self.proxies = { "http": None, "https": None, } if self.proxy_settings and self._use_proxy: uri = "{host}:{port}".format(host=self.proxy_settings['host'], port=self.proxy_settings['port']) if self.proxy_settings['user'] and self.proxy_settings['password']: uri = "{user}:{password}@{uri}".format( user=self.proxy_settings['user'], password=self.proxy_settings['password'], uri=uri) self.proxies['http'] = "http://{uri}".format(uri=uri) self.proxies['https'] = "https://{uri}".format(uri=uri) def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submit a raw count with optional tags, hostname and device name :param metric: The name of the metric :param value: The value :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.submit_count(metric, value, tags, hostname, device_name) def monotonic_count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submits a raw count with optional tags, hostname and device name based on increasing counter values. E.g. 1, 3, 5, 7 will submit 6 on flush. Note that reset counters are skipped. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.count_from_counter(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) @classmethod def generate_historate_func(cls, excluding_tags): def fct(self, metric, value, tags=None, hostname=None, device_name=None): cls.historate(self, metric, value, excluding_tags, tags=tags, hostname=hostname, device_name=device_name) return fct @classmethod def generate_histogram_func(cls, excluding_tags): def fct(self, metric, value, tags=None, hostname=None, device_name=None): tags = list( tags ) # Use a copy of the list to avoid removing tags from originial for tag in list(tags): for exc_tag in excluding_tags: if tag.startswith(exc_tag + ":"): tags.remove(tag) cls.histogram(self, metric, value, tags=tags, hostname=hostname, device_name=device_name) return fct def historate(self, metric, value, excluding_tags, tags=None, hostname=None, device_name=None): """ Function to create a histogram metric for "rate" like metrics. Warning this doesn't use the harmonic mean, beware of what it means when using it. :param metric: The name of the metric :param value: The value to sample for the histogram :param excluding_tags: A list of tags that will be removed when computing the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ tags = list( tags ) # Use a copy of the list to avoid removing tags from originial context = [metric] if tags is not None: context.append("-".join(sorted(tags))) if hostname is not None: context.append("host:" + hostname) if device_name is not None: context.append("device:" + device_name) now = time.time() context = tuple(context) if context in self.historate_dict: if tags is not None: for tag in list(tags): for exc_tag in excluding_tags: if tag.startswith("{0}:".format(exc_tag)): tags.remove(tag) prev_value, prev_ts = self.historate_dict[context] rate = float(value - prev_value) / float(now - prev_ts) self.aggregator.histogram(metric, rate, tags, hostname, device_name) self.historate_dict[context] = (value, now) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ self.events.append(event) def service_check(self, check_name, status, tags=None, timestamp=None, hostname=None, check_run_id=None, message=None): """ Save a service check. :param check_name: string, name of the service check :param status: int, describing the status. 0 for success, 1 for warning, 2 for failure :param tags: (optional) list of strings, a list of tags for this run :param timestamp: (optional) float, unix timestamp for when the run occurred :param hostname: (optional) str, host that generated the service check. Defaults to the host_name of the agent :param check_run_id: (optional) int, id used for logging and tracing purposes. Doesn't need to be unique. If not specified, one will be generated. """ if hostname is None: hostname = self.hostname if message is not None: message = unicode( message) # ascii converts to unicode but not viceversa self.service_checks.append( create_service_check(check_name, status, tags, timestamp, hostname, check_run_id, message)) def service_metadata(self, meta_name, value): """ Save metadata. :param meta_name: metadata key name :type meta_name: string :param value: metadata value :type value: string """ self._instance_metadata.append((meta_name, unicode(value))) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def get_service_checks(self): """ Return a list of the service checks saved by the check, if any and clears them out of the instance's service_checks list @return the list of service checks saved by this check @rtype list of service check dicts """ service_checks = self.service_checks self.service_checks = [] return service_checks def _roll_up_instance_metadata(self): """ Concatenate and flush instance metadata. """ self.svc_metadata.append( dict((k, v) for (k, v) in self._instance_metadata)) self._instance_metadata = [] def get_service_metadata(self): """ Return a list of the metadata dictionaries saved by the check -if any- and clears them out of the instance's service_checks list @return the list of metadata saved by this check @rtype list of metadata dicts """ if self._instance_metadata: self._roll_up_instance_metadata() service_metadata = self.svc_metadata self.svc_metadata = [] return service_metadata def has_warnings(self): """ Check whether the instance run created any warnings """ return len(self.warnings) > 0 def warning(self, warning_message): """ Add a warning message that will be printed in the info page :param warning_message: String. Warning message to be displayed """ warning_message = str(warning_message) self.log.warning(warning_message) self.warnings.append(warning_message) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): """ Should return a string that shows which version of the needed libraries are used """ raise NotImplementedError def get_warnings(self): """ Return the list of warnings messages to be displayed in the info page """ warnings = self.warnings self.warnings = [] return warnings @staticmethod def _get_statistic_name_from_method(method_name): return method_name[4:] if method_name.startswith( 'get_') else method_name @staticmethod def _collect_internal_stats(methods=None): current_process = psutil.Process(os.getpid()) methods = methods or DEFAULT_PSUTIL_METHODS filtered_methods = [m for m in methods if hasattr(current_process, m)] stats = {} for method in filtered_methods: # Go from `get_memory_info` -> `memory_info` stat_name = AgentCheck._get_statistic_name_from_method(method) try: raw_stats = getattr(current_process, method)() try: stats[stat_name] = raw_stats._asdict() except AttributeError: if isinstance(raw_stats, numbers.Number): stats[stat_name] = raw_stats else: log.warn( "Could not serialize output of {0} to dict".format( method)) except psutil.AccessDenied: log.warn("Cannot call psutil method {} : Access Denied".format( method)) return stats def _set_internal_profiling_stats(self, before, after): self._internal_profiling_stats = {'before': before, 'after': after} def _get_internal_profiling_stats(self): """ If in developer mode, return a dictionary of statistics about the check run """ stats = self._internal_profiling_stats self._internal_profiling_stats = None return stats def run(self): """ Run all instances. """ # Store run statistics if needed before, after = None, None if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: before = AgentCheck._collect_internal_stats() except Exception: # It's fine if we can't collect stats for the run, just log and proceed self.log.debug( "Failed to collect Agent Stats before check {0}".format( self.name)) instance_statuses = [] for i, instance in enumerate(self.instances): try: min_collection_interval = instance.get( 'min_collection_interval', self.min_collection_interval) now = time.time() if now - self.last_collection_time[i] < min_collection_interval: self.log.debug( "Not running instance #{0} of check {1} as it ran less than {2}s ago" .format(i, self.name, min_collection_interval)) continue self.last_collection_time[i] = now check_start_time = None if self.in_developer_mode: check_start_time = timeit.default_timer() self.check(copy.deepcopy(instance)) instance_check_stats = None if check_start_time is not None: instance_check_stats = { 'run_time': timeit.default_timer() - check_start_time } if self.has_warnings(): instance_status = check_status.InstanceStatus( i, check_status.STATUS_WARNING, warnings=self.get_warnings(), instance_check_stats=instance_check_stats) else: instance_status = check_status.InstanceStatus( i, check_status.STATUS_OK, instance_check_stats=instance_check_stats) except Exception as e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus( i, check_status.STATUS_ERROR, error=str(e), tb=traceback.format_exc()) finally: self._roll_up_instance_metadata() instance_statuses.append(instance_status) if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: after = AgentCheck._collect_internal_stats() self._set_internal_profiling_stats(before, after) log.info("\n \t %s %s" % (self.name, pretty_statistics(self._internal_profiling_stats))) except Exception: # It's fine if we can't collect stats for the run, just log and proceed self.log.debug( "Failed to collect Agent Stats after check {0}".format( self.name)) return instance_statuses def check(self, instance): """ Overriden by the check class. This will be called to run the check. :param instance: A dict with the instance information. This will vary depending on your config structure. """ raise NotImplementedError() def stop(self): """ To be executed when the agent is being stopped to clean ressources """ pass @classmethod def from_yaml(cls, path_to_yaml=None, agentConfig=None, yaml_text=None, check_name=None): """ A method used for testing your check without running the agent. """ if path_to_yaml: check_name = os.path.basename(path_to_yaml).split('.')[0] try: f = open(path_to_yaml) except IOError: raise Exception('Unable to open yaml config: %s' % path_to_yaml) yaml_text = f.read() f.close() config = yaml.load(yaml_text, Loader=yLoader) try: check = cls(check_name, config.get('init_config') or {}, agentConfig or {}, config.get('instances')) except TypeError: # Compatibility for the check not supporting instances check = cls(check_name, config.get('init_config') or {}, agentConfig or {}) return check, config.get('instances', []) def normalize(self, metric, prefix=None, fix_case=False): """ Turn a metric into a well-formed metric name prefix.b.c :param metric The metric name to normalize :param prefix A prefix to to add to the normalized name, default None :param fix_case A boolean, indicating whether to make sure that the metric name returned is in underscore_case """ if isinstance(metric, unicode): metric_name = unicodedata.normalize('NFKD', metric).encode( 'ascii', 'ignore') else: metric_name = metric if fix_case: name = self.convert_to_underscore_separated(metric_name) if prefix is not None: prefix = self.convert_to_underscore_separated(prefix) else: name = re.sub(r"[,\+\*\-/()\[\]{}\s]", "_", metric_name) # Eliminate multiple _ name = re.sub(r"__+", "_", name) # Don't start/end with _ name = re.sub(r"^_", "", name) name = re.sub(r"_$", "", name) # Drop ._ and _. name = re.sub(r"\._", ".", name) name = re.sub(r"_\.", ".", name) if prefix is not None: return prefix + "." + name else: return name FIRST_CAP_RE = re.compile('(.)([A-Z][a-z]+)') ALL_CAP_RE = re.compile('([a-z0-9])([A-Z])') METRIC_REPLACEMENT = re.compile(r'([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)') DOT_UNDERSCORE_CLEANUP = re.compile(r'_*\._*') def convert_to_underscore_separated(self, name): """ Convert from CamelCase to camel_case And substitute illegal metric characters """ metric_name = self.FIRST_CAP_RE.sub(r'\1_\2', name) metric_name = self.ALL_CAP_RE.sub(r'\1_\2', metric_name).lower() metric_name = self.METRIC_REPLACEMENT.sub('_', metric_name) return self.DOT_UNDERSCORE_CLEANUP.sub('.', metric_name).strip('_') @staticmethod def read_config(instance, key, message=None, cast=None): val = instance.get(key) if val is None: message = message or 'Must provide `%s` value in instance config' % key raise Exception(message) if cast is None: return val else: return cast(val)
class AgentCheck(object): OK, WARNING, CRITICAL, UNKNOWN = (0, 1, 2, 3) SOURCE_TYPE_NAME = None DEFAULT_MIN_COLLECTION_INTERVAL = 0 _enabled_checks = [] @classmethod def is_check_enabled(cls, name): return name in cls._enabled_checks def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self._enabled_checks.append(name) self._enabled_checks = list(set(self._enabled_checks)) self.name = name self.init_config = init_config or {} self.agentConfig = agentConfig self.in_developer_mode = agentConfig.get("developer_mode") and psutil self._internal_profiling_stats = None self.hostname = agentConfig.get("checksd_hostname") or get_hostname(agentConfig) self.log = logging.getLogger("%s.%s" % (__name__, name)) self.aggregator = MetricsAggregator( self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get("recent_point_threshold", None), histogram_aggregates=agentConfig.get("histogram_aggregates"), histogram_percentiles=agentConfig.get("histogram_percentiles"), ) if Platform.is_linux() and psutil is not None: procfs_path = self.agentConfig.get("procfs_path", "/proc").rstrip("/") psutil.PROCFS_PATH = procfs_path self.events = [] self.service_checks = [] self.instances = instances or [] self.warnings = [] self.library_versions = None self.last_collection_time = defaultdict(int) self._instance_metadata = [] self.svc_metadata = [] self.historate_dict = {} def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submit a raw count with optional tags, hostname and device name :param metric: The name of the metric :param value: The value :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.submit_count(metric, value, tags, hostname, device_name) def monotonic_count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submits a raw count with optional tags, hostname and device name based on increasing counter values. E.g. 1, 3, 5, 7 will submit 6 on flush. Note that reset counters are skipped. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.count_from_counter(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) @classmethod def generate_historate_func(cls, excluding_tags): def fct(self, metric, value, tags=None, hostname=None, device_name=None): cls.historate(self, metric, value, excluding_tags, tags=tags, hostname=hostname, device_name=device_name) return fct @classmethod def generate_histogram_func(cls, excluding_tags): def fct(self, metric, value, tags=None, hostname=None, device_name=None): tags = list(tags) # Use a copy of the list to avoid removing tags from originial for tag in list(tags): for exc_tag in excluding_tags: if tag.startswith(exc_tag + ":"): tags.remove(tag) cls.histogram(self, metric, value, tags=tags, hostname=hostname, device_name=device_name) return fct def historate(self, metric, value, excluding_tags, tags=None, hostname=None, device_name=None): """ Function to create a histogram metric for "rate" like metrics. Warning this doesn't use the harmonic mean, beware of what it means when using it. :param metric: The name of the metric :param value: The value to sample for the histogram :param excluding_tags: A list of tags that will be removed when computing the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ tags = list(tags) # Use a copy of the list to avoid removing tags from originial context = [metric] if tags is not None: context.append("-".join(sorted(tags))) if hostname is not None: context.append("host:" + hostname) if device_name is not None: context.append("device:" + device_name) now = time.time() context = tuple(context) if context in self.historate_dict: if tags is not None: for tag in list(tags): for exc_tag in excluding_tags: if tag.startswith("{0}:".format(exc_tag)): tags.remove(tag) prev_value, prev_ts = self.historate_dict[context] rate = float(value - prev_value) / float(now - prev_ts) self.aggregator.histogram(metric, rate, tags, hostname, device_name) self.historate_dict[context] = (value, now) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ self.events.append(event) def service_check( self, check_name, status, tags=None, timestamp=None, hostname=None, check_run_id=None, message=None ): """ Save a service check. :param check_name: string, name of the service check :param status: int, describing the status. 0 for success, 1 for warning, 2 for failure :param tags: (optional) list of strings, a list of tags for this run :param timestamp: (optional) float, unix timestamp for when the run occurred :param hostname: (optional) str, host that generated the service check. Defaults to the host_name of the agent :param check_run_id: (optional) int, id used for logging and tracing purposes. Doesn't need to be unique. If not specified, one will be generated. """ if hostname is None: hostname = self.hostname if message is not None: message = unicode(message) # ascii converts to unicode but not viceversa self.service_checks.append( create_service_check(check_name, status, tags, timestamp, hostname, check_run_id, message) ) def service_metadata(self, meta_name, value): """ Save metadata. :param meta_name: metadata key name :type meta_name: string :param value: metadata value :type value: string """ self._instance_metadata.append((meta_name, unicode(value))) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def get_service_checks(self): """ Return a list of the service checks saved by the check, if any and clears them out of the instance's service_checks list @return the list of service checks saved by this check @rtype list of service check dicts """ service_checks = self.service_checks self.service_checks = [] return service_checks def _roll_up_instance_metadata(self): """ Concatenate and flush instance metadata. """ self.svc_metadata.append(dict((k, v) for (k, v) in self._instance_metadata)) self._instance_metadata = [] def get_service_metadata(self): """ Return a list of the metadata dictionaries saved by the check -if any- and clears them out of the instance's service_checks list @return the list of metadata saved by this check @rtype list of metadata dicts """ if self._instance_metadata: self._roll_up_instance_metadata() service_metadata = self.svc_metadata self.svc_metadata = [] return service_metadata def has_warnings(self): """ Check whether the instance run created any warnings """ return len(self.warnings) > 0 def warning(self, warning_message): """ Add a warning message that will be printed in the info page :param warning_message: String. Warning message to be displayed """ warning_message = str(warning_message) self.log.warning(warning_message) self.warnings.append(warning_message) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): """ Should return a string that shows which version of the needed libraries are used """ raise NotImplementedError def get_warnings(self): """ Return the list of warnings messages to be displayed in the info page """ warnings = self.warnings self.warnings = [] return warnings @staticmethod def _get_statistic_name_from_method(method_name): return method_name[4:] if method_name.startswith("get_") else method_name @staticmethod def _collect_internal_stats(methods=None): current_process = psutil.Process(os.getpid()) methods = methods or DEFAULT_PSUTIL_METHODS filtered_methods = [m for m in methods if hasattr(current_process, m)] stats = {} for method in filtered_methods: # Go from `get_memory_info` -> `memory_info` stat_name = AgentCheck._get_statistic_name_from_method(method) try: raw_stats = getattr(current_process, method)() try: stats[stat_name] = raw_stats._asdict() except AttributeError: if isinstance(raw_stats, numbers.Number): stats[stat_name] = raw_stats else: log.warn("Could not serialize output of {0} to dict".format(method)) except psutil.AccessDenied: log.warn("Cannot call psutil method {} : Access Denied".format(method)) return stats def _set_internal_profiling_stats(self, before, after): self._internal_profiling_stats = {"before": before, "after": after} def _get_internal_profiling_stats(self): """ If in developer mode, return a dictionary of statistics about the check run """ stats = self._internal_profiling_stats self._internal_profiling_stats = None return stats def run(self): """ Run all instances. """ # Store run statistics if needed before, after = None, None if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: before = AgentCheck._collect_internal_stats() except Exception: # It's fine if we can't collect stats for the run, just log and proceed self.log.debug("Failed to collect Agent Stats before check {0}".format(self.name)) instance_statuses = [] for i, instance in enumerate(self.instances): try: min_collection_interval = instance.get( "min_collection_interval", self.init_config.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL), ) now = time.time() if now - self.last_collection_time[i] < min_collection_interval: self.log.debug( "Not running instance #{0} of check {1} as it ran less than {2}s ago".format( i, self.name, min_collection_interval ) ) continue self.last_collection_time[i] = now check_start_time = None if self.in_developer_mode: check_start_time = timeit.default_timer() self.check(copy.deepcopy(instance)) instance_check_stats = None if check_start_time is not None: instance_check_stats = {"run_time": timeit.default_timer() - check_start_time} if self.has_warnings(): instance_status = check_status.InstanceStatus( i, check_status.STATUS_WARNING, warnings=self.get_warnings(), instance_check_stats=instance_check_stats, ) else: instance_status = check_status.InstanceStatus( i, check_status.STATUS_OK, instance_check_stats=instance_check_stats ) except Exception as e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus( i, check_status.STATUS_ERROR, error=str(e), tb=traceback.format_exc() ) finally: self._roll_up_instance_metadata() instance_statuses.append(instance_status) if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: after = AgentCheck._collect_internal_stats() self._set_internal_profiling_stats(before, after) log.info("\n \t %s %s" % (self.name, pretty_statistics(self._internal_profiling_stats))) except Exception: # It's fine if we can't collect stats for the run, just log and proceed self.log.debug("Failed to collect Agent Stats after check {0}".format(self.name)) return instance_statuses def check(self, instance): """ Overriden by the check class. This will be called to run the check. :param instance: A dict with the instance information. This will vary depending on your config structure. """ raise NotImplementedError() def stop(self): """ To be executed when the agent is being stopped to clean ressources """ pass @classmethod def from_yaml(cls, path_to_yaml=None, agentConfig=None, yaml_text=None, check_name=None): """ A method used for testing your check without running the agent. """ if path_to_yaml: check_name = os.path.basename(path_to_yaml).split(".")[0] try: f = open(path_to_yaml) except IOError: raise Exception("Unable to open yaml config: %s" % path_to_yaml) yaml_text = f.read() f.close() config = yaml.load(yaml_text, Loader=yLoader) try: check = cls(check_name, config.get("init_config") or {}, agentConfig or {}, config.get("instances")) except TypeError: # Compatibility for the check not supporting instances check = cls(check_name, config.get("init_config") or {}, agentConfig or {}) return check, config.get("instances", []) def normalize(self, metric, prefix=None, fix_case=False): """ Turn a metric into a well-formed metric name prefix.b.c :param metric The metric name to normalize :param prefix A prefix to to add to the normalized name, default None :param fix_case A boolean, indicating whether to make sure that the metric name returned is in underscore_case """ if isinstance(metric, unicode): metric_name = unicodedata.normalize("NFKD", metric).encode("ascii", "ignore") else: metric_name = metric if fix_case: name = self.convert_to_underscore_separated(metric_name) if prefix is not None: prefix = self.convert_to_underscore_separated(prefix) else: name = re.sub(r"[,\+\*\-/()\[\]{}\s]", "_", metric_name) # Eliminate multiple _ name = re.sub(r"__+", "_", name) # Don't start/end with _ name = re.sub(r"^_", "", name) name = re.sub(r"_$", "", name) # Drop ._ and _. name = re.sub(r"\._", ".", name) name = re.sub(r"_\.", ".", name) if prefix is not None: return prefix + "." + name else: return name FIRST_CAP_RE = re.compile("(.)([A-Z][a-z]+)") ALL_CAP_RE = re.compile("([a-z0-9])([A-Z])") METRIC_REPLACEMENT = re.compile(r"([^a-zA-Z0-9_.]+)|(^[^a-zA-Z]+)") DOT_UNDERSCORE_CLEANUP = re.compile(r"_*\._*") def convert_to_underscore_separated(self, name): """ Convert from CamelCase to camel_case And substitute illegal metric characters """ metric_name = self.FIRST_CAP_RE.sub(r"\1_\2", name) metric_name = self.ALL_CAP_RE.sub(r"\1_\2", metric_name).lower() metric_name = self.METRIC_REPLACEMENT.sub("_", metric_name) return self.DOT_UNDERSCORE_CLEANUP.sub(".", metric_name).strip("_") @staticmethod def read_config(instance, key, message=None, cast=None): val = instance.get(key) if val is None: message = message or "Must provide `%s` value in instance config" % key raise Exception(message) if cast is None: return val else: return cast(val)
class AgentCheck(object): OK, WARNING, CRITICAL, UNKNOWN = (0, 1, 2, 3) SOURCE_TYPE_NAME = None DEFAULT_MIN_COLLECTION_INTERVAL = 0 _enabled_checks = [] @classmethod def is_check_enabled(cls, name): return name in cls._enabled_checks def __init__(self, name, init_config, agentConfig, instances=None): from aggregator import MetricsAggregator self._enabled_checks.append(name) self._enabled_checks = list(set(self._enabled_checks)) self.name = name self.init_config = init_config or {} self.agentConfig = agentConfig self.in_developer_mode = agentConfig.get('developer_mode') and psutil self._internal_profiling_stats = None self.hostname = agentConfig.get('checksd_hostname') or get_hostname( agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.aggregator = MetricsAggregator( self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None), histogram_aggregates=agentConfig.get('histogram_aggregates'), histogram_percentiles=agentConfig.get('histogram_percentiles')) self.events = [] self.service_checks = [] if instances: jsoned_instances = json.dumps(instances) encrypted_passwd_list = re.findall('>>>.*?<<<', jsoned_instances) if encrypted_passwd_list: for encrypted_passwd in encrypted_passwd_list: decrypted_passwd = decrypted(encrypted_passwd) jsoned_instances = jsoned_instances.replace( encrypted_passwd, decrypted_passwd) self.instances = convert_to_str( json.loads(jsoned_instances, encoding='utf-8')) else: self.instances = instances else: self.instances = [] self.warnings = [] self.library_versions = None self.last_collection_time = defaultdict(int) self._instance_metadata = [] self.svc_metadata = [] self.historate_dict = {} def instance_count(self): return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): self.aggregator.decrement(metric, value, tags, hostname, device_name) def count(self, metric, value=0, tags=None, hostname=None, device_name=None): self.aggregator.submit_count(metric, value, tags, hostname, device_name) def monotonic_count(self, metric, value=0, tags=None, hostname=None, device_name=None): self.aggregator.count_from_counter(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): self.aggregator.histogram(metric, value, tags, hostname, device_name) @classmethod def generate_historate_func(cls, excluding_tags): def fct(self, metric, value, tags=None, hostname=None, device_name=None): cls.historate(self, metric, value, excluding_tags, tags=tags, hostname=hostname, device_name=device_name) return fct @classmethod def generate_histogram_func(cls, excluding_tags): def fct(self, metric, value, tags=None, hostname=None, device_name=None): tags = list(tags) for tag in list(tags): for exc_tag in excluding_tags: if tag.startswith(exc_tag + ":"): tags.remove(tag) cls.histogram(self, metric, value, tags=tags, hostname=hostname, device_name=device_name) return fct def historate(self, metric, value, excluding_tags, tags=None, hostname=None, device_name=None): tags = list(tags) context = [metric] if tags is not None: context.append("-".join(sorted(tags))) if hostname is not None: context.append("host:" + hostname) if device_name is not None: context.append("device:" + device_name) now = time.time() context = tuple(context) if context in self.historate_dict: if tags is not None: for tag in list(tags): for exc_tag in excluding_tags: if tag.startswith("{0}:".format(exc_tag)): tags.remove(tag) prev_value, prev_ts = self.historate_dict[context] rate = float(value - prev_value) / float(now - prev_ts) self.aggregator.histogram(metric, rate, tags, hostname, device_name) self.historate_dict[context] = (value, now) def set(self, metric, value, tags=None, hostname=None, device_name=None): self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): if event.get('api_key') is None: event['api_key'] = self.agentConfig['api_key'] self.events.append(event) def service_check(self, check_name, status, tags=None, timestamp=None, hostname=None, check_run_id=None, message=None): if hostname is None: hostname = self.hostname if message is not None: message = unicode(message) self.service_checks.append( create_service_check(check_name, status, tags, timestamp, hostname, check_run_id, message)) def service_metadata(self, meta_name, value): self._instance_metadata.append((meta_name, unicode(value))) def has_events(self): return len(self.events) > 0 def get_metrics(self): return self.aggregator.flush() def get_events(self): events = self.events self.events = [] return events def get_service_checks(self): service_checks = self.service_checks self.service_checks = [] return service_checks def _roll_up_instance_metadata(self): self.svc_metadata.append( dict((k, v) for (k, v) in self._instance_metadata)) self._instance_metadata = [] def get_service_metadata(self): if self._instance_metadata: self._roll_up_instance_metadata() service_metadata = self.svc_metadata self.svc_metadata = [] return service_metadata def has_warnings(self): return len(self.warnings) > 0 def warning(self, warning_message): warning_message = str(warning_message) self.log.warning(warning_message) self.warnings.append(warning_message) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): raise NotImplementedError def get_warnings(self): warnings = self.warnings self.warnings = [] return warnings @staticmethod def _get_statistic_name_from_method(method_name): return method_name[4:] if method_name.startswith( 'get_') else method_name @staticmethod def _collect_internal_stats(methods=None): current_process = psutil.Process(os.getpid()) methods = methods or DEFAULT_PSUTIL_METHODS filtered_methods = [m for m in methods if hasattr(current_process, m)] stats = {} for method in filtered_methods: stat_name = AgentCheck._get_statistic_name_from_method(method) try: raw_stats = getattr(current_process, method)() try: stats[stat_name] = raw_stats._asdict() except AttributeError: if isinstance(raw_stats, numbers.Number): stats[stat_name] = raw_stats else: log.warn( "Could not serialize output of {0} to dict".format( method)) except psutil.AccessDenied: log.warn("Cannot call psutil method {} : Access Denied".format( method)) return stats def _set_internal_profiling_stats(self, before, after): self._internal_profiling_stats = {'before': before, 'after': after} def _get_internal_profiling_stats(self): stats = self._internal_profiling_stats self._internal_profiling_stats = None return stats def run(self): before, after = None, None if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: before = AgentCheck._collect_internal_stats() except Exception: self.log.debug( "Failed to collect Agent Stats before check {0}".format( self.name)) instance_statuses = [] for i, instance in enumerate(self.instances): try: min_collection_interval = instance.get( 'min_collection_interval', self.init_config.get('min_collection_interval', self.DEFAULT_MIN_COLLECTION_INTERVAL)) now = time.time() if now - self.last_collection_time[i] < min_collection_interval: self.log.debug( "Not running instance #{0} of check {1} as it ran less than {2}s ago" .format(i, self.name, min_collection_interval)) continue self.last_collection_time[i] = now check_start_time = None if self.in_developer_mode: check_start_time = timeit.default_timer() self.check(copy.deepcopy(instance)) instance_check_stats = None if check_start_time is not None: instance_check_stats = { 'run_time': timeit.default_timer() - check_start_time } if self.has_warnings(): instance_status = check_status.InstanceStatus( i, check_status.STATUS_WARNING, warnings=self.get_warnings(), instance_check_stats=instance_check_stats) else: instance_status = check_status.InstanceStatus( i, check_status.STATUS_OK, instance_check_stats=instance_check_stats) except Exception, e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus( i, check_status.STATUS_ERROR, error=str(e), tb=str(e)) finally:
class AgentCheck(object): def __init__(self, name, init_config, agentConfig): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent """ from aggregator import MetricsAggregator self.name = name self.init_config = init_config self.agentConfig = agentConfig self.hostname = gethostname(agentConfig) self.log = logging.getLogger('checks.%s' % name) self.aggregator = MetricsAggregator(self.hostname, formatter=agent_formatter) self.events = [] def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "api_key": string, the api key of the account to associate the event with, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ self.events.append(event) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def check(self, instance): """ Overriden by the check class. This will be called to run the check. :param instance: A dict with the instance information. This will vary depending on your config structure. """ raise NotImplementedError() @classmethod def from_yaml(cls, path_to_yaml=None, agentConfig=None, yaml_text=None, check_name=None): """ A method used for testing your check without running the agent. """ from util import yaml, yLoader if path_to_yaml: check_name = os.path.basename(path_to_yaml).split('.')[0] try: f = open(path_to_yaml) except IOError: raise Exception('Unable to open yaml config: %s' % path_to_yaml) yaml_text = f.read() f.close() config = yaml.load(yaml_text, Loader=yLoader) check = cls(check_name, config.get('init_config') or {}, agentConfig or {}) return check, config.get('instances', []) def normalize(self, metric, prefix=None): """ Turn a metric into a well-formed metric name prefix.b.c :param metric The metric name to normalize :param prefix A prefix to to add to the normalized name, default None """ name = re.sub(r"[,\+\*\-/()\[\]{}]", "_", metric) # Eliminate multiple _ name = re.sub(r"__+", "_", name) # Don't start/end with _ name = re.sub(r"^_", "", name) name = re.sub(r"_$", "", name) # Drop ._ and _. name = re.sub(r"\._", ".", name) name = re.sub(r"_\.", ".", name) if prefix is not None: return prefix + "." + name else: return name
class AgentCheck(object): OK, WARNING, CRITICAL, UNKNOWN = (0, 1, 2, 3) SOURCE_TYPE_NAME = None DEFAULT_MIN_COLLECTION_INTERVAL = 0 def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self.name = name self.init_config = init_config or {} self.agentConfig = agentConfig self.in_developer_mode = agentConfig.get('developer_mode') and psutil is not None self._internal_profiling_stats = None self.hostname = agentConfig.get('checksd_hostname') or get_hostname(agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.aggregator = MetricsAggregator( self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None), histogram_aggregates=agentConfig.get('histogram_aggregates'), histogram_percentiles=agentConfig.get('histogram_percentiles') ) self.events = [] self.service_checks = [] self.instances = instances or [] self.warnings = [] self.library_versions = None self.last_collection_time = defaultdict(int) self._instance_metadata = [] self.svc_metadata = [] def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submit a raw count with optional tags, hostname and device name :param metric: The name of the metric :param value: The value :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.submit_count(metric, value, tags, hostname, device_name) def monotonic_count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submits a raw count with optional tags, hostname and device name based on increasing counter values. E.g. 1, 3, 5, 7 will submit 6 on flush. Note that reset counters are skipped. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.count_from_counter(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "api_key": string, the api key of the account to associate the event with, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ if event.get('api_key') is None: event['api_key'] = self.agentConfig['api_key'] self.events.append(event) def service_check(self, check_name, status, tags=None, timestamp=None, hostname=None, check_run_id=None, message=None): """ Save a service check. :param check_name: string, name of the service check :param status: int, describing the status. 0 for success, 1 for warning, 2 for failure :param tags: (optional) list of strings, a list of tags for this run :param timestamp: (optional) float, unix timestamp for when the run occurred :param hostname: (optional) str, host that generated the service check. Defaults to the host_name of the agent :param check_run_id: (optional) int, id used for logging and tracing purposes. Don't need to be unique. If not specified, one will be generated. """ if hostname is None: hostname = self.hostname if message is not None: message = str(message) self.service_checks.append( create_service_check(check_name, status, tags, timestamp, hostname, check_run_id, message) ) def service_metadata(self, meta_name, value): """ Save metadata. :param meta_name: metadata key name :type meta_name: string :param value: metadata value :type value: string """ self._instance_metadata.append((meta_name, str(value))) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def get_service_checks(self): """ Return a list of the service checks saved by the check, if any and clears them out of the instance's service_checks list @return the list of service checks saved by this check @rtype list of service check dicts """ service_checks = self.service_checks self.service_checks = [] return service_checks def _roll_up_instance_metadata(self): """ Concatenate and flush instance metadata. """ self.svc_metadata.append(dict((k, v) for (k, v) in self._instance_metadata)) self._instance_metadata = [] def get_service_metadata(self): """ Return a list of the metadata dictionaries saved by the check -if any- and clears them out of the instance's service_checks list @return the list of metadata saved by this check @rtype list of metadata dicts """ if self._instance_metadata: self._roll_up_instance_metadata() service_metadata = self.svc_metadata self.svc_metadata = [] return service_metadata def has_warnings(self): """ Check whether the instance run created any warnings """ return len(self.warnings) > 0 def warning(self, warning_message): """ Add a warning message that will be printed in the info page :param warning_message: String. Warning message to be displayed """ self.warnings.append(str(warning_message)) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): """ Should return a string that shows which version of the needed libraries are used """ raise NotImplementedError def get_warnings(self): """ Return the list of warnings messages to be displayed in the info page """ warnings = self.warnings self.warnings = [] return warnings @staticmethod def _get_statistic_name_from_method(method_name): return method_name[4:] if method_name.startswith('get_') else method_name @staticmethod def _collect_internal_stats(methods=None): current_process = psutil.Process(os.getpid()) methods = methods or DEFAULT_PSUTIL_METHODS filtered_methods = [m for m in methods if hasattr(current_process, m)] stats = {} for method in filtered_methods: # Go from `get_memory_info` -> `memory_info` stat_name = AgentCheck._get_statistic_name_from_method(method) try: raw_stats = getattr(current_process, method)() try: stats[stat_name] = raw_stats._asdict() except AttributeError: if isinstance(raw_stats, numbers.Number): stats[stat_name] = raw_stats else: log.warn("Could not serialize output of {0} to dict".format(method)) except psutil.AccessDenied: log.warn("Cannot call psutil method {} : Access Denied".format(method)) return stats def _set_internal_profiling_stats(self, before, after): self._internal_profiling_stats = {'before': before, 'after': after} def _get_internal_profiling_stats(self): """ If in developer mode, return a dictionary of statistics about the check run """ stats = self._internal_profiling_stats self._internal_profiling_stats = None return stats def run(self): """ Run all instances. """ # Store run statistics if needed before, after = None, None if self.in_developer_mode and self.name != AGENT_METRICS_CHECK_NAME: try: before = AgentCheck._collect_internal_stats() except Exception: # It's fine if we can't collect stats for the run, just log and proceed self.log.debug("Failed to collect Agent Stats before check {0}".format(self.name)) instance_statuses = [] for i, instance in enumerate(self.instances): try: min_collection_interval = instance.get( 'min_collection_interval', self.init_config.get( 'min_collection_interval', self.DEFAULT_MIN_COLLECTION_INTERVAL ) ) now = time.time() if now - self.last_collection_time[i] < min_collection_interval: self.log.debug("Not running instance #{0} of check {1} as it ran less than {2}s ago".format(i, self.name, min_collection_interval)) continue self.last_collection_time[i] = now check_start_time = None if self.in_developer_mode: check_start_time = timeit.default_timer() self.check(copy.deepcopy(instance)) instance_check_stats = None if check_start_time is not None: instance_check_stats = {'run_time': timeit.default_timer() - check_start_time} if self.has_warnings(): instance_status = check_status.InstanceStatus( i, check_status.STATUS_WARNING, warnings=self.get_warnings(), instance_check_stats=instance_check_stats ) else: instance_status = check_status.InstanceStatus( i, check_status.STATUS_OK, instance_check_stats=instance_check_stats ) except Exception, e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus( i, check_status.STATUS_ERROR, error=str(e), tb=traceback.format_exc() ) finally:
class AgentCheck(object): OK, WARNING, CRITICAL, UNKNOWN = (0, 1, 2, 3) SOURCE_TYPE_NAME = None DEFAULT_MIN_COLLECTION_INTERVAL = 0 def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self.name = name self.init_config = init_config or {} self.agentConfig = agentConfig self.hostname = agentConfig.get('checksd_hostname') or get_hostname(agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.aggregator = MetricsAggregator( self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None), histogram_aggregates=agentConfig.get('histogram_aggregates'), histogram_percentiles=agentConfig.get('histogram_percentiles') ) self.events = [] self.service_checks = [] self.instances = instances or [] self.warnings = [] self.library_versions = None self.last_collection_time = defaultdict(int) def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submit a raw count with optional tags, hostname and device name :param metric: The name of the metric :param value: The value :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.submit_count(metric, value, tags, hostname, device_name) def monotonic_count(self, metric, value=0, tags=None, hostname=None, device_name=None): """ Submits a raw count with optional tags, hostname and device name based on increasing counter values. E.g. 1, 3, 5, 7 will submit 6 on flush. Note that reset counters are skipped. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.count_from_counter(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "api_key": string, the api key of the account to associate the event with, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ if event.get('api_key') is None: event['api_key'] = self.agentConfig['api_key'] self.events.append(event) def service_check(self, check_name, status, tags=None, timestamp=None, hostname=None, check_run_id=None, message=None): """ Save a service check. :param check_name: string, name of the service check :param status: int, describing the status. 0 for success, 1 for warning, 2 for failure :param tags: (optional) list of strings, a list of tags for this run :param timestamp: (optional) float, unix timestamp for when the run occurred :param hostname: (optional) str, host that generated the service check. Defaults to the host_name of the agent :param check_run_id: (optional) int, id used for logging and tracing purposes. Don't need to be unique. If not specified, one will be generated. """ if hostname is None: hostname = self.hostname self.service_checks.append(create_service_check(check_name, status, tags, timestamp, hostname, check_run_id, message)) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def get_service_checks(self): """ Return a list of the service checks saved by the check, if any and clears them out of the instance's service_checks list @return the list of service checks saved by this check @rtype list of service check dicts """ service_checks = self.service_checks self.service_checks = [] return service_checks def has_warnings(self): """ Check whether the instance run created any warnings """ return len(self.warnings) > 0 def warning(self, warning_message): """ Add a warning message that will be printed in the info page :param warning_message: String. Warning message to be displayed """ self.warnings.append(warning_message) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): """ Should return a string that shows which version of the needed libraries are used """ raise NotImplementedError def get_warnings(self): """ Return the list of warnings messages to be displayed in the info page """ warnings = self.warnings self.warnings = [] return warnings def run(self): """ Run all instances. """ instance_statuses = [] for i, instance in enumerate(self.instances): try: min_collection_interval = instance.get('min_collection_interval', self.init_config.get('min_collection_interval', self.DEFAULT_MIN_COLLECTION_INTERVAL)) now = time.time() if now - self.last_collection_time[i] < min_collection_interval: self.log.debug("Not running instance #{0} of check {1} as it ran less than {2}s ago".format(i, self.name, min_collection_interval)) continue self.last_collection_time[i] = now self.check(copy.deepcopy(instance)) if self.has_warnings(): instance_status = check_status.InstanceStatus(i, check_status.STATUS_WARNING, warnings=self.get_warnings() ) else: instance_status = check_status.InstanceStatus(i, check_status.STATUS_OK) except Exception, e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus(i, check_status.STATUS_ERROR, error=e, tb=traceback.format_exc() ) instance_statuses.append(instance_status) return instance_statuses
class AgentCheck(object): def __init__(self, name, init_config, agentConfig, instances=None): """ Initialize a new check. :param name: The name of the check :param init_config: The config for initializing the check :param agentConfig: The global configuration for the agent :param instances: A list of configuration objects for each instance. """ from aggregator import MetricsAggregator self.name = name self.init_config = init_config self.agentConfig = agentConfig self.hostname = get_hostname(agentConfig) self.log = logging.getLogger('%s.%s' % (__name__, name)) self.aggregator = MetricsAggregator( self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None)) self.events = [] self.instances = instances or [] self.warnings = [] self.library_versions = None def instance_count(self): """ Return the number of instances that are configured for this check. """ return len(self.instances) def gauge(self, metric, value, tags=None, hostname=None, device_name=None, timestamp=None): """ Record the value of a gauge, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value of the gauge :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric :param timestamp: (optional) The timestamp for this metric value """ self.aggregator.gauge(metric, value, tags, hostname, device_name, timestamp) def increment(self, metric, value=1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to increment by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.increment(metric, value, tags, hostname, device_name) def decrement(self, metric, value=-1, tags=None, hostname=None, device_name=None): """ Increment a counter with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to decrement by :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.decrement(metric, value, tags, hostname, device_name) def rate(self, metric, value, tags=None, hostname=None, device_name=None): """ Submit a point for a metric that will be calculated as a rate on flush. Values will persist across each call to `check` if there is not enough point to generate a rate on the flush. :param metric: The name of the metric :param value: The value of the rate :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.rate(metric, value, tags, hostname, device_name) def histogram(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a histogram value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value to sample for the histogram :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.histogram(metric, value, tags, hostname, device_name) def set(self, metric, value, tags=None, hostname=None, device_name=None): """ Sample a set value, with optional tags, hostname and device name. :param metric: The name of the metric :param value: The value for the set :param tags: (optional) A list of tags for this metric :param hostname: (optional) A hostname for this metric. Defaults to the current hostname. :param device_name: (optional) The device name for this metric """ self.aggregator.set(metric, value, tags, hostname, device_name) def event(self, event): """ Save an event. :param event: The event payload as a dictionary. Has the following structure: { "timestamp": int, the epoch timestamp for the event, "event_type": string, the event time name, "api_key": string, the api key of the account to associate the event with, "msg_title": string, the title of the event, "msg_text": string, the text body of the event, "alert_type": (optional) string, one of ('error', 'warning', 'success', 'info'). Defaults to 'info'. "source_type_name": (optional) string, the source type name, "host": (optional) string, the name of the host, "tags": (optional) list, a list of tags to associate with this event } """ self.events.append(event) def has_events(self): """ Check whether the check has saved any events @return whether or not the check has saved any events @rtype boolean """ return len(self.events) > 0 def get_metrics(self): """ Get all metrics, including the ones that are tagged. @return the list of samples @rtype [(metric_name, timestamp, value, {"tags": ["tag1", "tag2"]}), ...] """ return self.aggregator.flush() def get_events(self): """ Return a list of the events saved by the check, if any @return the list of events saved by this check @rtype list of event dictionaries """ events = self.events self.events = [] return events def has_warnings(self): """ Check whether the instance run created any warnings """ return len(self.warnings) > 0 def warning(self, warning_message): """ Add a warning message that will be printed in the info page :param warning_message: String. Warning message to be displayed """ self.warnings.append(warning_message) def get_library_info(self): if self.library_versions is not None: return self.library_versions try: self.library_versions = self.get_library_versions() except NotImplementedError: pass def get_library_versions(self): """ Should return a string that shows which version of the needed libraries are used """ raise NotImplementedError def get_warnings(self): """ Return the list of warnings messages to be displayed in the info page """ warnings = self.warnings self.warnings = [] return warnings def run(self): """ Run all instances. """ instance_statuses = [] for i, instance in enumerate(self.instances): try: self.check(instance) if self.has_warnings(): instance_status = check_status.InstanceStatus( i, check_status.STATUS_WARNING, warnings=self.get_warnings()) else: instance_status = check_status.InstanceStatus( i, check_status.STATUS_OK) except Exception, e: self.log.exception("Check '%s' instance #%s failed" % (self.name, i)) instance_status = check_status.InstanceStatus( i, check_status.STATUS_ERROR, error=e, tb=traceback.format_exc()) instance_statuses.append(instance_status) return instance_statuses