def setUp(self):
self.reporter = GangliaReporter("Group Name", "localhost", 8649)
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(5)
Metrology.utilization_timer('utimer').update(5)
def setUp(self):
self.reporter = LibratoReporter("<email>", "<token>")
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(1.5)
Metrology.utilization_timer('utimer').update(1.5)
def setUp(self):
self.reporter = GraphiteReporter('localhost', 3333)
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(1.5)
Metrology.utilization_timer('utimer').update(1.5)
def setUp(self):
self.output = StringIO()
logging.basicConfig(stream=self.output, level=logging.INFO)
self.reporter = LoggerReporter()
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(1.5)
Metrology.utilization_timer('utimer').update(1.5)
def test_send_batch(self, mock):
self.reporter = GraphiteReporter("localhost", 3334, pickle=True, batch_size=2)
Metrology.meter("meter").mark()
Metrology.counter("counter").increment()
Metrology.timer("timer").update(5)
Metrology.utilization_timer("utimer").update(5)
Metrology.histogram("histogram").update(5)
self.reporter.write()
self.assertTrue(mock.sendall.assert_called())
self.assertEqual(25, len(mock.sendall.call_args_list))
self.reporter.stop()
def test_send_batch(self, mock):
self.reporter = GraphiteReporter('localhost', 3333, batch_size=2)
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(5)
Metrology.utilization_timer('utimer').update(5)
Metrology.histogram('histogram').update(5)
self.reporter.write()
self.assertTrue(mock.send.assert_called())
self.assertEqual(25, len(mock.sendall.call_args_list))
self.reporter.stop()
def test_udp_send_batch(self, mock):
self.reporter = StatsDReporter('localhost', 3333,
batch_size=2, conn_type='udp')
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(5)
Metrology.utilization_timer('utimer').update(5)
Metrology.histogram('histogram').update(5)
self.reporter.write()
self.assertTrue(mock.sendto.called)
self.assertEqual(3, len(mock.sendto.call_args_list))
self.reporter.stop()
def __init__(self, zenhub, service):
self.zenhub = zenhub
self.service = service
self._serviceCalls = Metrology.meter("zenhub.serviceCalls")
self.log = logging.getLogger('zen.zenhub.WorkerInterceptor')
self._admTimer = Metrology.timer('zenhub.applyDataMap')
self._eventsSent = Metrology.meter("zenhub.eventsSent")
self.meters = {
'sendEvent': self.mark_send_event_timer,
'sendEvents': self.mark_send_events_timer,
'applyDataMaps': self.mark_apply_datamaps_timer,
}
def test_udp_send_batch(self, mock):
self.reporter = StatsDReporter('localhost',
3333,
batch_size=2,
conn_type='udp')
Metrology.meter('meter').mark()
Metrology.counter('counter').increment()
Metrology.timer('timer').update(5)
Metrology.utilization_timer('utimer').update(5)
Metrology.histogram('histogram').update(5)
self.reporter.write()
self.assertTrue(mock.sendto.called)
self.assertEqual(3, len(mock.sendto.call_args_list))
self.reporter.stop()
def __init__(self, application, reporter=None, **kwargs):
self.application = application
self.request = Metrology.meter('request')
self.request_time = Metrology.timer('request_time')
# Start reporter
if reporter:
reporter.start()
def __init__(self):
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
ZCmdBase.__init__(self)
if self.options.profiling:
self.profiler = ContinuousProfiler('zenhubworker', log=self.log)
self.profiler.start()
self.current = IDLE
self.currentStart = 0
self.numCalls = Metrology.meter("zenhub.workerCalls")
try:
self.log.debug("establishing SIGUSR1 signal handler")
signal.signal(signal.SIGUSR1, self.sighandler_USR1)
self.log.debug("establishing SIGUSR2 signal handler")
signal.signal(signal.SIGUSR2, self.sighandler_USR2)
except ValueError:
# If we get called multiple times, this will generate an exception:
# ValueError: signal only works in main thread
# Ignore it as we've already set up the signal handler.
pass
self.zem = self.dmd.ZenEventManager
loadPlugins(self.dmd)
self.pid = os.getpid()
self.services = {}
factory = ReconnectingPBClientFactory(pingPerspective=False)
self.log.debug("Connecting to %s:%d", self.options.hubhost,
self.options.hubport)
reactor.connectTCP(self.options.hubhost, self.options.hubport, factory)
self.log.debug("Logging in as %s", self.options.username)
c = credentials.UsernamePassword(self.options.username,
self.options.password)
factory.gotPerspective = self.gotPerspective
def stop(*args):
reactor.callLater(0, reactor.stop)
factory.clientConnectionLost = stop
factory.setCredentials(c)
self.log.debug("Creating async MetricReporter")
daemonTags = {
'zenoss_daemon': 'zenhub_worker_%s' % self.options.workernum,
'zenoss_monitor': self.options.monitor,
'internal': True
}
def stopReporter():
if self.metricreporter:
return self.metricreporter.stop()
# Order of the shutdown triggers matter. Want to stop reporter first, calling metricWriter() below
# registers shutdown triggers for the actual metric http and redis publishers.
reactor.addSystemEventTrigger('before', 'shutdown', stopReporter)
self.metricreporter = TwistedMetricReporter(
metricWriter=metricWriter(), tags=daemonTags)
self.metricreporter.start()
def __init__(self, single=False):
"""
Initalizer
@param single: collect from a single device?
@type single: boolean
"""
PBDaemon.__init__(self)
# FIXME: cleanup --force option #2660
self.options.force = True
self.start = None
self.startat = None
self.rrdStats = DaemonStats()
self.single = single
if self.options.device:
self.single = True
self.modelerCycleInterval = self.options.cycletime
# get the minutes and convert to fraction of a day
self.collage = float(self.options.collage) / 1440.0
self.pendingNewClients = False
self.clients = []
self.finished = []
self.devicegen = None
self.counters = collections.Counter()
self.configFilter = None
self.configLoaded = False
# Make sendEvent() available to plugins
zope.component.provideUtility(self, IEventService)
# Delay start for between 10 and 60 seconds when run as a daemon.
self.started = False
self.startDelay = 0
self.immediate = 1
if self.options.daemon or self.options.cycle:
if self.options.now:
self.log.debug('option "now" specified, starting immediately.')
else:
# self.startDelay = randint(10, 60) * 60
self.startDelay = randint(10, 60) * 1
self.immediate = 0
self.log.info(
'option "now" not specified, waiting %s seconds to start.'
% self.startDelay)
else:
self.log.debug("Run in foreground, starting immediately.")
# ZEN-26637
self.collectorLoopIteration = 0
self.mainLoopGotDeviceList = False
self.isMainScheduled = False
self._modeledDevicesMetric = Metrology.meter(
"zenmodeler.modeledDevices")
self._failuresMetric = Metrology.counter("zenmodeler.failures")
def __init__(self, options, log):
self.options = options
self.transformers = _load_utilities(ICollectorEventTransformer)
self.log = log
self.discarded_events = 0
# TODO: Do we want to limit the size of the clear event dictionary?
self.clear_events_count = {}
self._initQueues()
self._eventsSent = Metrology.meter("collectordaemon.eventsSent")
self._discardedEvents = Metrology.meter("collectordaemon.discardedEvent")
self._eventTimer = Metrology.timer('collectordaemon.eventTimer')
metricNames = {x[0] for x in registry}
if 'collectordaemon.eventQueue' not in metricNames:
queue = self
class EventQueueGauge(Gauge):
@property
def value(self):
return queue.event_queue_length
Metrology.gauge('collectordaemon.eventQueue', EventQueueGauge())
def __init__(self, single=False ):
"""
Initalizer
@param single: collect from a single device?
@type single: boolean
"""
PBDaemon.__init__(self)
# FIXME: cleanup --force option #2660
self.options.force = True
self.start = None
self.startat = None
self.rrdStats = DaemonStats()
self.single = single
if self.options.device:
self.single = True
self.modelerCycleInterval = self.options.cycletime
# get the minutes and convert to fraction of a day
self.collage = float( self.options.collage ) / 1440.0
self.pendingNewClients = False
self.clients = []
self.finished = []
self.devicegen = None
self.counters = collections.Counter()
self.configFilter = None
self.configLoaded = False
# Make sendEvent() available to plugins
zope.component.provideUtility(self, IEventService)
# Delay start for between 10 and 60 seconds when run as a daemon.
self.started = False
self.startDelay = 0
self.immediate = 1
if self.options.daemon or self.options.cycle:
if self.options.now:
self.log.debug('option "now" specified, starting immediately.')
else:
# self.startDelay = randint(10, 60) * 60
self.startDelay = randint(10, 60) * 1
self.immediate = 0
self.log.info('option "now" not specified, waiting %s seconds to start.' %
self.startDelay)
else:
self.log.debug("Run in foreground, starting immediately.")
# ZEN-26637
self.collectorLoopIteration = 0
self.mainLoopGotDeviceList = False
self._modeledDevicesMetric = Metrology.meter("zenmodeler.modeledDevices")
self._failuresMetric = Metrology.counter("zenmodeler.failures")
def __init__(self, options, log):
self.options = options
self.transformers = _load_utilities(ICollectorEventTransformer)
self.log = log
self.discarded_events = 0
# TODO: Do we want to limit the size of the clear event dictionary?
self.clear_events_count = {}
self._initQueues()
self._eventsSent = Metrology.meter("collectordaemon.eventsSent")
self._discardedEvents = Metrology.meter(
"collectordaemon.discardedEvent")
self._eventTimer = Metrology.timer('collectordaemon.eventTimer')
metricNames = {x[0] for x in registry}
if 'collectordaemon.eventQueue' not in metricNames:
queue = self
class EventQueueGauge(Gauge):
@property
def value(self):
return queue.event_queue_length
Metrology.gauge('collectordaemon.eventQueue', EventQueueGauge())
def __init__(self, reactor):
"""Initialize a ZenHubWorker instance."""
ZCmdBase.__init__(self)
self.__reactor = reactor
if self.options.profiling:
self.profiler = ContinuousProfiler('ZenHubWorker', log=self.log)
self.profiler.start()
reactor.addSystemEventTrigger(
'before', 'shutdown', self.profiler.stop,
)
self.instanceId = self.options.workerid
self.current = IDLE
self.currentStart = 0
self.numCalls = Metrology.meter("zenhub.workerCalls")
self.zem = self.dmd.ZenEventManager
loadPlugins(self.dmd)
serviceFactory = ServiceReferenceFactory(self)
self.__registry = HubServiceRegistry(self.dmd, serviceFactory)
# Configure/initialize the ZenHub client
creds = UsernamePassword(
self.options.hubusername, self.options.hubpassword,
)
endpointDescriptor = "tcp:{host}:{port}".format(
host=self.options.hubhost, port=self.options.hubport,
)
endpoint = clientFromString(reactor, endpointDescriptor)
self.__client = ZenHubClient(reactor, endpoint, creds, self, 10.0)
# Setup Metric Reporting
self.log.debug("Creating async MetricReporter")
self._metric_manager = MetricManager(
daemon_tags={
'zenoss_daemon': 'zenhub_worker_%s' % self.options.workerid,
'zenoss_monitor': self.options.monitor,
'internal': True,
},
)
def register_legacy_worklist_metrics():
"""Create the Metrology counters for tracking worklist statistics."""
config = getUtility(IHubServerConfig)
global _legacy_worklist_counters
for metricName, priorityName in config.legacy_metric_priority_map.items():
gauge = registry.metrics.get(metricName)
priority = ServiceCallPriority[priorityName]
if not gauge:
gauge = WorkListGauge(_legacy_worklist_counters, priority)
Metrology.gauge(metricName, gauge)
_legacy_worklist_counters[priority] = 0
gauge = registry.metrics.get(_legacy_metric_worklist_total.metric)
if not gauge:
gauge = WorkListGauge(
_legacy_worklist_counters,
_legacy_metric_worklist_total.name,
)
Metrology.gauge(_legacy_metric_worklist_total.metric, gauge)
_legacy_worklist_counters["total"] = 0
global _legacy_events_meter
_legacy_events_meter = Metrology.meter("zenhub.eventsSent")
def test_meter(self):
self.assertTrue(Metrology.meter('test') is not None)
try:
from networkx import shortest_simple_paths
except ImportError:
# The networkx version shipped with Zenoss 4 (1.3) doesn't have
# shortest_simple_paths. So we have a compatible copy of the algorithm
# locally.
from .nx.simple_paths import shortest_simple_paths
from . import connections
import logging
LOG = logging.getLogger("zen.Layer2")
from metrology import Metrology
s_meter = Metrology.meter("events-suppressed")
# Default exports.
__all__ = [
"get_suppressor",
]
# Make status checks clearer.
UP, DOWN = True, False
# Make toggle checks clearer.
ENABLED, DISABLED = True, False
# Singleton to keep state for callers who can't keep their own state.
SUPPRESSOR = None
from metrology.reporter import LoggerReporter
from producer_conf import *
def generate_data(fake):
data = {
"username": fake.user_name(),
"ip": fake.ipv4_public(),
"uri": fake.uri(),
"user_agent": fake.user_agent(),
"method": fake.random_element(elements=["POST", "GET", "HEAD", "PUT"]),
"status_code": fake.random_element(elements=[200,201, 202, 301, 303, 400, 401, 403, 404, 500]),
"ts": datetime.now().isoformat()
}
return data
meter = Metrology.meter("messages")
successful = Metrology.counter("success")
errors = Metrology.counter("errors")
logging.basicConfig(level=logging.DEBUG)
reporter = LoggerReporter(level=logging.INFO, interval=10)
def delivery_report(err, msg):
if err is not None:
errors.increment()
else:
successful.increment()
meter.mark()
producer_config = {"bootstrap.servers": BROKER_HOST}
if len(KAFKA_USERNAME):
import logging, configargparse, pyodbc, time, pdb, re, json, sys
from threading import Event
from confluent_kafka import Producer
from metrology import Metrology
from metrology.reporter import LoggerReporter
meter = Metrology.meter("messages")
successful = Metrology.counter("success")
errors = Metrology.meter("errors")
stop_event = Event()
def get_kafka_parameters(options):
producer_config = {
"bootstrap.servers": options.brokers,
"message.timeout.ms": 1000
}
match = re.findall("://([^/]+)/", options.brokers)
if len(match) == 1:
producer_config["bootstrap.servers"] = match[0] + ":9093"
producer_config.update({
'sasl.mechanisms': 'PLAIN',
'security.protocol': 'SASL_SSL',
"sasl.username": "******",
"sasl.password": options.brokers
})
logging.debug("Using Kafka config: {}".format(json.dumps(producer_config)))
return producer_config
def get_badge(options):
def __init__(self, preferences, taskSplitter,
configurationListener=DUMMY_LISTENER,
initializationCallback=None,
stoppingCallback=None):
"""
Constructs a new instance of the CollectorDaemon framework. Normally
only a singleton instance of a CollectorDaemon should exist within a
process, but this is not enforced.
@param preferences: the collector configuration
@type preferences: ICollectorPreferences
@param taskSplitter: the task splitter to use for this collector
@type taskSplitter: ITaskSplitter
@param initializationCallback: a callable that will be executed after
connection to the hub but before
retrieving configuration information
@type initializationCallback: any callable
@param stoppingCallback: a callable that will be executed first during
the stopping process. Exceptions will be
logged but otherwise ignored.
@type stoppingCallback: any callable
"""
# create the configuration first, so we have the collector name
# available before activating the rest of the Daemon class hierarchy.
if not ICollectorPreferences.providedBy(preferences):
raise TypeError("configuration must provide ICollectorPreferences")
else:
self._prefs = ObservableProxy(preferences)
self._prefs.attachAttributeObserver('configCycleInterval', self._rescheduleConfig)
if not ITaskSplitter.providedBy(taskSplitter):
raise TypeError("taskSplitter must provide ITaskSplitter")
else:
self._taskSplitter = taskSplitter
if not IConfigurationListener.providedBy(configurationListener):
raise TypeError(
"configurationListener must provide IConfigurationListener")
self._configListener = ConfigListenerNotifier()
self._configListener.addListener(configurationListener)
self._configListener.addListener(DeviceGuidListener(self))
self._initializationCallback = initializationCallback
self._stoppingCallback = stoppingCallback
# register the various interfaces we provide the rest of the system so
# that collector implementors can easily retrieve a reference back here
# if needed
zope.component.provideUtility(self, ICollector)
zope.component.provideUtility(self, IEventService)
zope.component.provideUtility(self, IDataService)
# register the collector's own preferences object so it may be easily
# retrieved by factories, tasks, etc.
zope.component.provideUtility(self.preferences,
ICollectorPreferences,
self.preferences.collectorName)
super(CollectorDaemon, self).__init__(name=self.preferences.collectorName)
self._statService = StatisticsService()
zope.component.provideUtility(self._statService, IStatisticsService)
if self.options.cycle:
# setup daemon statistics (deprecated names)
self._statService.addStatistic("devices", "GAUGE")
self._statService.addStatistic("dataPoints", "DERIVE")
self._statService.addStatistic("runningTasks", "GAUGE")
self._statService.addStatistic("taskCount", "GAUGE")
self._statService.addStatistic("queuedTasks", "GAUGE")
self._statService.addStatistic("missedRuns", "GAUGE")
# namespace these a bit so they can be used in ZP monitoring.
# prefer these stat names and metrology in future refs
self._dataPointsMetric = Metrology.meter("collectordaemon.dataPoints")
daemon = self
class DeviceGauge(Gauge):
@property
def value(self):
return len(daemon._devices)
Metrology.gauge('collectordaemon.devices', DeviceGauge())
# Scheduler statistics
class RunningTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.running
Metrology.gauge('collectordaemon.runningTasks', RunningTasks())
class TaskCount(Gauge):
@property
def value(self):
return daemon._scheduler.taskCount
Metrology.gauge('collectordaemon.taskCount', TaskCount())
class QueuedTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.queued
Metrology.gauge('collectordaemon.queuedTasks', QueuedTasks())
class MissedRuns(Gauge):
@property
def value(self):
return daemon._scheduler.missedRuns
Metrology.gauge('collectordaemon.missedRuns', MissedRuns())
self._deviceGuids = {}
self._devices = set()
self._unresponsiveDevices = set()
self._rrd = None
self._metric_writer = None
self._derivative_tracker = None
self.reconfigureTimeout = None
# keep track of pending tasks if we're doing a single run, and not a
# continuous cycle
if not self.options.cycle:
self._completedTasks = 0
self._pendingTasks = []
frameworkFactory = zope.component.queryUtility(IFrameworkFactory, self._frameworkFactoryName)
self._configProxy = frameworkFactory.getConfigurationProxy()
self._scheduler = frameworkFactory.getScheduler()
self._scheduler.maxTasks = self.options.maxTasks
self._ConfigurationLoaderTask = frameworkFactory.getConfigurationLoaderTask()
# OLD - set the initialServices attribute so that the PBDaemon class
# will load all of the remote services we need.
self.initialServices = PBDaemon.initialServices +\
[self.preferences.configurationService]
# trap SIGUSR2 so that we can display detailed statistics
signal.signal(signal.SIGUSR2, self._signalHandler)
# let the configuration do any additional startup it might need
self.preferences.postStartup()
self.addedPostStartupTasks = False
# Variables used by enterprise collector in resmgr
#
# flag that indicates we have finished loading the configs for the first time after a restart
self.firstConfigLoadDone = False
# flag that indicates the daemon has received the encryption key from zenhub
self.encryptionKeyInitialized = False
# flag that indicates the daemon is loading the cached configs
self.loadingCachedConfigs = False
def store_request(self, request, response):
"""Store events in the cache, update statistics.
Allow for separate samples/events/URI cache data.
"""
# Create a unique client_id that has: host/uri
client_ip = request.getClientIP()
client_id = (client_ip, request.uri)
# Add the client's message to the cache
self.cache[client_id].append({
'timestamp': datetime.datetime.now(),
'response_code': request.code,
'response_code_message': request.code_message,
'request_body': request.content.getvalue(),
'response_body': response,
'zenoss_actions': request._zaction
})
# --------------------------------------------------------------------
# Use Metrology to record the event
# --------------------------------------------------------------------
v = dict(
dotted_uri=request.uri.replace('/', '.').strip('.'),
client_ip=client_ip,
method=request.method.lower()
)
Metrology.meter('http.requests').mark()
Metrology.meter('http.{method}.requests'.format(**v)).mark()
Metrology.meter("http.{method}.{dotted_uri}.requests".format(**v)).mark()
Metrology.meter("http.{method}.{dotted_uri}.{client_ip}.requests".format(**v)).mark()
if (request.code >= 400):
Metrology.meter("http.{method}.{dotted_uri}.{client_ip}.errors".format(**v)).mark()
for _ in request._zaction['maps']:
Metrology.meter("zenopenstack.{dotted_uri}.{client_ip}.datamaps".format(**v)).mark()
for _ in request._zaction['events']:
Metrology.meter("zenopenstack.{dotted_uri}.{client_ip}.events".format(**v)).mark()
for _ in request._zaction['metrics']:
Metrology.meter("zenopenstack.{dotted_uri}.{client_ip}.metrics".format(**v)).mark()
import collections
import time
import types
from Products.ZenEvents.ZenEventClasses import Status_Ping
from zenoss.protocols.protobufs.zep_pb2 import (
STATUS_SUPPRESSED,
SEVERITY_CLEAR,
SEVERITY_CRITICAL,
)
from . import connections
from metrology import Metrology
s_meter = Metrology.meter("events-suppressed")
# Default exports.
__all__ = [
"get_suppressor",
]
# Make status checks clearer.
UP, DOWN = True, False
# Make toggle checks clearer.
ENABLED, DISABLED = True, False
# Singleton to keep state for callers who can't keep their own state.
SUPPRESSOR = None
def test_request(self):
self.server.get('/')
self.assertEqual(1, Metrology.meter('request').count)
self.assertEqual(1, Metrology.timer('request_time').count)
def __init__(self,
preferences,
taskSplitter,
configurationListener=DUMMY_LISTENER,
initializationCallback=None,
stoppingCallback=None):
"""
Constructs a new instance of the CollectorDaemon framework. Normally
only a singleton instance of a CollectorDaemon should exist within a
process, but this is not enforced.
@param preferences: the collector configuration
@type preferences: ICollectorPreferences
@param taskSplitter: the task splitter to use for this collector
@type taskSplitter: ITaskSplitter
@param initializationCallback: a callable that will be executed after
connection to the hub but before
retrieving configuration information
@type initializationCallback: any callable
@param stoppingCallback: a callable that will be executed first during
the stopping process. Exceptions will be
logged but otherwise ignored.
@type stoppingCallback: any callable
"""
# create the configuration first, so we have the collector name
# available before activating the rest of the Daemon class hierarchy.
if not ICollectorPreferences.providedBy(preferences):
raise TypeError("configuration must provide ICollectorPreferences")
else:
self._prefs = ObservableProxy(preferences)
self._prefs.attachAttributeObserver('configCycleInterval',
self._rescheduleConfig)
if not ITaskSplitter.providedBy(taskSplitter):
raise TypeError("taskSplitter must provide ITaskSplitter")
else:
self._taskSplitter = taskSplitter
if not IConfigurationListener.providedBy(configurationListener):
raise TypeError(
"configurationListener must provide IConfigurationListener")
self._configListener = ConfigListenerNotifier()
self._configListener.addListener(configurationListener)
self._configListener.addListener(DeviceGuidListener(self))
self._initializationCallback = initializationCallback
self._stoppingCallback = stoppingCallback
# register the various interfaces we provide the rest of the system so
# that collector implementors can easily retrieve a reference back here
# if needed
zope.component.provideUtility(self, ICollector)
zope.component.provideUtility(self, IEventService)
zope.component.provideUtility(self, IDataService)
# register the collector's own preferences object so it may be easily
# retrieved by factories, tasks, etc.
zope.component.provideUtility(self.preferences, ICollectorPreferences,
self.preferences.collectorName)
super(CollectorDaemon,
self).__init__(name=self.preferences.collectorName)
self._statService = StatisticsService()
zope.component.provideUtility(self._statService, IStatisticsService)
if self.options.cycle:
# setup daemon statistics (deprecated names)
self._statService.addStatistic("devices", "GAUGE")
self._statService.addStatistic("dataPoints", "DERIVE")
self._statService.addStatistic("runningTasks", "GAUGE")
self._statService.addStatistic("taskCount", "GAUGE")
self._statService.addStatistic("queuedTasks", "GAUGE")
self._statService.addStatistic("missedRuns", "GAUGE")
# namespace these a bit so they can be used in ZP monitoring.
# prefer these stat names and metrology in future refs
self._dataPointsMetric = Metrology.meter(
"collectordaemon.dataPoints")
daemon = self
class DeviceGauge(Gauge):
@property
def value(self):
return len(daemon._devices)
Metrology.gauge('collectordaemon.devices', DeviceGauge())
# Scheduler statistics
class RunningTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.running
Metrology.gauge('collectordaemon.runningTasks', RunningTasks())
class TaskCount(Gauge):
@property
def value(self):
return daemon._scheduler.taskCount
Metrology.gauge('collectordaemon.taskCount', TaskCount())
class QueuedTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.queued
Metrology.gauge('collectordaemon.queuedTasks', QueuedTasks())
class MissedRuns(Gauge):
@property
def value(self):
return daemon._scheduler.missedRuns
Metrology.gauge('collectordaemon.missedRuns', MissedRuns())
self._deviceGuids = {}
self._devices = set()
self._unresponsiveDevices = set()
self._rrd = None
self._metric_writer = None
self._derivative_tracker = None
self.reconfigureTimeout = None
# keep track of pending tasks if we're doing a single run, and not a
# continuous cycle
if not self.options.cycle:
self._completedTasks = 0
self._pendingTasks = []
frameworkFactory = zope.component.queryUtility(
IFrameworkFactory, self._frameworkFactoryName)
self._configProxy = frameworkFactory.getConfigurationProxy()
self._scheduler = frameworkFactory.getScheduler()
self._scheduler.maxTasks = self.options.maxTasks
self._ConfigurationLoaderTask = frameworkFactory.getConfigurationLoaderTask(
)
# OLD - set the initialServices attribute so that the PBDaemon class
# will load all of the remote services we need.
self.initialServices = PBDaemon.initialServices +\
[self.preferences.configurationService]
# trap SIGUSR2 so that we can display detailed statistics
signal.signal(signal.SIGUSR2, self._signalHandler)
# let the configuration do any additional startup it might need
self.preferences.postStartup()
self.addedPostStartupTasks = False
# Variables used by enterprise collector in resmgr
#
# flag that indicates we have finished loading the configs for the first time after a restart
self.firstConfigLoadDone = False
# flag that indicates the daemon has received the encryption key from zenhub
self.encryptionKeyInitialized = False
# flag that indicates the daemon is loading the cached configs
self.loadingCachedConfigs = False
class Health(Resource):
""" /health: expose health of zenopenstack daemon (metrics, logs) """
isLeaf = True
def render_GET(self, request):
request.setResponseCode(200)
request.setHeader(b"content-type", b"text/html")
if len(request.postpath):
if len(request.postpath) == 1 and request.postpath[0] == "metrics":
body = "<html><body>"
body += "<table border=\"1\">"
body += "<tr>"
body += " <th>Metric</th>"
body += " <th>Count</th>"
body += " <th>1M Rate</th>"
body += " <th>5M Rate</th>"
body += " <th>15M Rate</th>"
body += " <th>Mean Rate</th>"
body += "</tr>"
for name, metric in sorted(metrology_registry):
if isinstance(metric, Meter):
body += "<tr>"
body += " <td>%s</td>" % name
body += " <td>%d</td>" % metric.count
body += " <td>%f</td>" % metric.one_minute_rate
body += " <td>%f</td>" % metric.five_minute_rate
body += " <td>%f</td>" % metric.fifteen_minute_rate
body += " <td>%f</td>" % metric.mean_rate
body += "</tr>"
else:
log.debug("Ignoring unhandled metric type: %s", metric)
body += "</body></html>"
return body
if len(request.postpath) == 1 and request.postpath[0] == "queue":
body = "<html><body>"
body = "The following devices have received model updates:<ul>"
for device_id in MAP_QUEUE:
body += '<li><a href="/health/queue/%s">%s</a></li>' % (device_id, device_id)
body += "</ul></body></html>"
return body
if len(request.postpath) == 2 and request.postpath[0] == "queue":
device_id = request.postpath[1]
if device_id not in MAP_QUEUE:
return NoResource().render(request)
body = "<html><body>"
body += "<b>Currently-Held Object Maps</b><p>"
for component_id, objmap in MAP_QUEUE[device_id].held_objmaps.iteritems():
body += "<hr>"
body += component_id + ":<br>"
body += "<pre>" + cgi.escape(pformat(objmap[1])) + "</pre>"
body += "<b>Most Recently Released Object Maps</b>"
for timestamp, objmap in reversed(MAP_QUEUE[device_id].released_objmaps):
body += "<hr>%s (%s ago)" % (
timestamp.isoformat(),
(datetime.datetime.now() - timestamp)
)
body += "<pre>" + cgi.escape(pformat(objmap)) + "</pre>"
body += "</body></html>"
return body
if len(request.postpath) < 3 or request.postpath[0] != "logs":
return NoResource().render(request)
ip = request.postpath[1]
uri = "/".join(request.postpath[2:])
body = "<html><body>"
records = self.site.request_buffer.get_requests((ip, '/' + uri))
body += "<b>Most recent %d requests from /%s to %s</b> (of %d max)<p>" % (
len(records),
ip,
uri,
self.site.request_buffer.cache_size
)
for record in reversed(records):
try:
request_body = record['request_body']
try:
# if the payload is JSON data, reindent it for readability
request_body = json.dumps(json.loads(request_body), indent=5)
except Exception:
pass
response_body = cgi.escape(record['response_body'])
zenoss_actions = cgi.escape(pformat(record['zenoss_actions']))
body += "<hr>%s (%s ago)" % (
record['timestamp'].isoformat(),
(datetime.datetime.now() - record['timestamp'])
)
body += '<table border="1" style="font-size: 80%"><tr><th>Request</th><th>Response</th><th>Zenoss Action</th></tr>'
body += "<tr valign=\"top\"><td><pre>" + request_body + "</pre></td>"
body += "<td><b>%d %s</b><pre>%s</pre></td>" % (
record['response_code'],
record['response_code_message'],
response_body)
body += "<td><pre>" + zenoss_actions + "</pre></td>"
body += "</tr></table>"
except Exception, e:
body += "<hr>Error displaying log record: %s<hr>" % str(e)
body += "</body></html>"
return body
body = """
<html>
<body>
<table border="1">
<tr>
<th rowspan="2">URI</th>
<th rowspan="2">Client IP</th>
<th colspan="3">POST</th>
<th colspan="3">GET</th>
<th colspan="3">POST Errors</th>
<th colspan="3">GET Errors</th>
<th colspan="3">Datamaps</th>
<th colspan="3">Events</th>
<th colspan="3">Metrics</th>
<th rowspan="2">Last Request</th>
<th rowspan="2">Details</th>
</tr>
<tr>
<th>count</th>
<th>15m</th>
<th>mean</th>
<th>count</th>
<th>15m</th>
<th>mean</th>
<th>count</th>
<th>15m</th>
<th>mean</th>
<th>count</th>
<th>15m</th>
<th>mean</th>
<th>count</th>
<th>15m</th>
<th>mean</th>
<th>count</th>
<th>15m</th>
<th>mean</th>
<th>count</th>
<th>15m</th>
<th>mean</th>
</tr>
"""
for client_ip, uri in sorted(self.site.request_buffer.get_client_keys()):
v = dict(
dotted_uri=uri.replace('/', '.').strip('.'),
client_ip=client_ip
)
posts = Metrology.meter("http.post.{dotted_uri}.{client_ip}.requests".format(**v))
gets = Metrology.meter("http.get.{dotted_uri}.{client_ip}.requests".format(**v))
post_errors = Metrology.meter("http.post.{dotted_uri}.{client_ip}.errors".format(**v))
get_errors = Metrology.meter("http.get.{dotted_uri}.{client_ip}.errors".format(**v))
datamaps = Metrology.meter("zenopenstack.{dotted_uri}.{client_ip}.datamaps".format(**v))
events = Metrology.meter("zenopenstack.{dotted_uri}.{client_ip}.events".format(**v))
metrics = Metrology.meter("zenopenstack.{dotted_uri}.{client_ip}.metrics".format(**v))
most_recent_request = self.site.request_buffer.get_most_recent_request((client_ip, uri))
body += "<tr>"
body += " <td>%s</td>" % uri
body += " <td>%s</td>" % client_ip
body += " <td>%d</td>" % posts.count
body += " <td>%.2f</td>" % posts.fifteen_minute_rate
body += " <td>%.2f</td>" % posts.mean_rate
body += " <td>%d</td>" % gets.count
body += " <td>%.2f</td>" % gets.fifteen_minute_rate
body += " <td>%.2f</td>" % gets.mean_rate
body += " <td>%d</td>" % post_errors.count
body += " <td>%.2f</td>" % post_errors.fifteen_minute_rate
body += " <td>%.2f</td>" % post_errors.mean_rate
body += " <td>%d</td>" % get_errors.count
body += " <td>%.2f</td>" % get_errors.fifteen_minute_rate
body += " <td>%.2f</td>" % get_errors.mean_rate
body += " <td>%d</td>" % datamaps.count
body += " <td>%.2f</td>" % datamaps.fifteen_minute_rate
body += " <td>%.2f</td>" % datamaps.mean_rate
body += " <td>%d</td>" % events.count
body += " <td>%.2f</td>" % events.fifteen_minute_rate
body += " <td>%.2f</td>" % events.mean_rate
body += " <td>%d</td>" % metrics.count
body += " <td>%.2f</td>" % metrics.fifteen_minute_rate
body += " <td>%.2f</td>" % metrics.mean_rate
if most_recent_request:
body += " <td>%s ago (%d)</td>" % (
(datetime.datetime.now() - most_recent_request['timestamp']),
most_recent_request['response_code'])
else:
body += " <td>None</td>"
body += " <td><a href=\"/health/logs/%s%s\">request log</a></td>" % (client_ip, uri)
body += "</tr>"
return body + """
def __init__(self, reactor):
"""Initialize a ZenHubWorker instance."""
ZCmdBase.__init__(self)
self.__reactor = reactor
if self.options.profiling:
self.profiler = ContinuousProfiler('ZenHubWorker', log=self.log)
self.profiler.start()
reactor.addSystemEventTrigger(
'before',
'shutdown',
self.profiler.stop,
)
self.current = IDLE
self.currentStart = 0
self.numCalls = Metrology.meter("zenhub.workerCalls")
self.zem = self.dmd.ZenEventManager
loadPlugins(self.dmd)
self.__registry = ServiceRegistry()
loader = ServiceLoader()
factory = ServiceReferenceFactory(self)
self.__manager = ServiceManager(self.__registry, loader, factory)
# Configure/initialize the ZenHub client
creds = UsernamePassword(
self.options.hubusername,
self.options.hubpassword,
)
endpointDescriptor = "tcp:{host}:{port}".format(
host=self.options.hubhost,
port=self.options.hubport,
)
endpoint = clientFromString(reactor, endpointDescriptor)
self.__client = ZenHubClient(
reactor,
endpoint,
creds,
self,
10.0,
self.worklistId,
)
# Setup Metric Reporting
self.log.debug("Creating async MetricReporter")
self._metric_manager = MetricManager(daemon_tags={
'zenoss_daemon':
'zenhub_worker_%s' % self.instanceId,
'zenoss_monitor':
self.options.monitor,
'internal':
True,
}, )
# Make the metric manager available via zope.component.getUtility
getGlobalSiteManager().registerUtility(
self._metric_manager,
IMetricManager,
name='zenhub_worker_metricmanager',
)
