def register_metrics_on_worklist(worklist):
metricNames = {x[0] for x in registry}
for metricName, priority in _gauge_priority_map.iteritems():
if metricName not in metricNames:
gauge = PriorityListLengthGauge(worklist, priority)
Metrology.gauge(metricName, gauge)
# Original metric name
if "zenhub.workList" not in metricNames:
gauge = WorklistLengthGauge(worklist)
Metrology.gauge("zenhub.workList", gauge)
def __init__(self, options, log):
self.options = options
self.transformers = _load_utilities(ICollectorEventTransformer)
self.log = log
self.discarded_events = 0
# TODO: Do we want to limit the size of the clear event dictionary?
self.clear_events_count = {}
self._initQueues()
self._eventsSent = Metrology.meter("collectordaemon.eventsSent")
self._discardedEvents = Metrology.meter("collectordaemon.discardedEvent")
self._eventTimer = Metrology.timer('collectordaemon.eventTimer')
metricNames = {x[0] for x in registry}
if 'collectordaemon.eventQueue' not in metricNames:
queue = self
class EventQueueGauge(Gauge):
@property
def value(self):
return queue.event_queue_length
Metrology.gauge('collectordaemon.eventQueue', EventQueueGauge())
def __init__(self, options, log):
self.options = options
self.transformers = _load_utilities(ICollectorEventTransformer)
self.log = log
self.discarded_events = 0
# TODO: Do we want to limit the size of the clear event dictionary?
self.clear_events_count = {}
self._initQueues()
self._eventsSent = Metrology.meter("collectordaemon.eventsSent")
self._discardedEvents = Metrology.meter(
"collectordaemon.discardedEvent")
self._eventTimer = Metrology.timer('collectordaemon.eventTimer')
metricNames = {x[0] for x in registry}
if 'collectordaemon.eventQueue' not in metricNames:
queue = self
class EventQueueGauge(Gauge):
@property
def value(self):
return queue.event_queue_length
Metrology.gauge('collectordaemon.eventQueue', EventQueueGauge())
def register_legacy_worklist_metrics():
"""Create the Metrology counters for tracking worklist statistics."""
config = getUtility(IHubServerConfig)
global _legacy_worklist_counters
for metricName, priorityName in config.legacy_metric_priority_map.items():
gauge = registry.metrics.get(metricName)
priority = ServiceCallPriority[priorityName]
if not gauge:
gauge = WorkListGauge(_legacy_worklist_counters, priority)
Metrology.gauge(metricName, gauge)
_legacy_worklist_counters[priority] = 0
gauge = registry.metrics.get(_legacy_metric_worklist_total.metric)
if not gauge:
gauge = WorkListGauge(
_legacy_worklist_counters,
_legacy_metric_worklist_total.name,
)
Metrology.gauge(_legacy_metric_worklist_total.metric, gauge)
_legacy_worklist_counters["total"] = 0
global _legacy_events_meter
_legacy_events_meter = Metrology.meter("zenhub.eventsSent")
def test_gauge(self):
self.assertTrue(Metrology.gauge('test', Gauge) is not None)
def __init__(self,
preferences,
taskSplitter,
configurationListener=DUMMY_LISTENER,
initializationCallback=None,
stoppingCallback=None):
"""
Constructs a new instance of the CollectorDaemon framework. Normally
only a singleton instance of a CollectorDaemon should exist within a
process, but this is not enforced.
@param preferences: the collector configuration
@type preferences: ICollectorPreferences
@param taskSplitter: the task splitter to use for this collector
@type taskSplitter: ITaskSplitter
@param initializationCallback: a callable that will be executed after
connection to the hub but before
retrieving configuration information
@type initializationCallback: any callable
@param stoppingCallback: a callable that will be executed first during
the stopping process. Exceptions will be
logged but otherwise ignored.
@type stoppingCallback: any callable
"""
# create the configuration first, so we have the collector name
# available before activating the rest of the Daemon class hierarchy.
if not ICollectorPreferences.providedBy(preferences):
raise TypeError("configuration must provide ICollectorPreferences")
else:
self._prefs = ObservableProxy(preferences)
self._prefs.attachAttributeObserver('configCycleInterval',
self._rescheduleConfig)
if not ITaskSplitter.providedBy(taskSplitter):
raise TypeError("taskSplitter must provide ITaskSplitter")
else:
self._taskSplitter = taskSplitter
if not IConfigurationListener.providedBy(configurationListener):
raise TypeError(
"configurationListener must provide IConfigurationListener")
self._configListener = ConfigListenerNotifier()
self._configListener.addListener(configurationListener)
self._configListener.addListener(DeviceGuidListener(self))
self._initializationCallback = initializationCallback
self._stoppingCallback = stoppingCallback
# register the various interfaces we provide the rest of the system so
# that collector implementors can easily retrieve a reference back here
# if needed
zope.component.provideUtility(self, ICollector)
zope.component.provideUtility(self, IEventService)
zope.component.provideUtility(self, IDataService)
# register the collector's own preferences object so it may be easily
# retrieved by factories, tasks, etc.
zope.component.provideUtility(self.preferences, ICollectorPreferences,
self.preferences.collectorName)
super(CollectorDaemon,
self).__init__(name=self.preferences.collectorName)
self._statService = StatisticsService()
zope.component.provideUtility(self._statService, IStatisticsService)
if self.options.cycle:
# setup daemon statistics (deprecated names)
self._statService.addStatistic("devices", "GAUGE")
self._statService.addStatistic("dataPoints", "DERIVE")
self._statService.addStatistic("runningTasks", "GAUGE")
self._statService.addStatistic("taskCount", "GAUGE")
self._statService.addStatistic("queuedTasks", "GAUGE")
self._statService.addStatistic("missedRuns", "GAUGE")
# namespace these a bit so they can be used in ZP monitoring.
# prefer these stat names and metrology in future refs
self._dataPointsMetric = Metrology.meter(
"collectordaemon.dataPoints")
daemon = self
class DeviceGauge(Gauge):
@property
def value(self):
return len(daemon._devices)
Metrology.gauge('collectordaemon.devices', DeviceGauge())
# Scheduler statistics
class RunningTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.running
Metrology.gauge('collectordaemon.runningTasks', RunningTasks())
class TaskCount(Gauge):
@property
def value(self):
return daemon._scheduler.taskCount
Metrology.gauge('collectordaemon.taskCount', TaskCount())
class QueuedTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.queued
Metrology.gauge('collectordaemon.queuedTasks', QueuedTasks())
class MissedRuns(Gauge):
@property
def value(self):
return daemon._scheduler.missedRuns
Metrology.gauge('collectordaemon.missedRuns', MissedRuns())
self._deviceGuids = {}
self._devices = set()
self._unresponsiveDevices = set()
self._rrd = None
self._metric_writer = None
self._derivative_tracker = None
self.reconfigureTimeout = None
# keep track of pending tasks if we're doing a single run, and not a
# continuous cycle
if not self.options.cycle:
self._completedTasks = 0
self._pendingTasks = []
frameworkFactory = zope.component.queryUtility(
IFrameworkFactory, self._frameworkFactoryName)
self._configProxy = frameworkFactory.getConfigurationProxy()
self._scheduler = frameworkFactory.getScheduler()
self._scheduler.maxTasks = self.options.maxTasks
self._ConfigurationLoaderTask = frameworkFactory.getConfigurationLoaderTask(
)
# OLD - set the initialServices attribute so that the PBDaemon class
# will load all of the remote services we need.
self.initialServices = PBDaemon.initialServices +\
[self.preferences.configurationService]
# trap SIGUSR2 so that we can display detailed statistics
signal.signal(signal.SIGUSR2, self._signalHandler)
# let the configuration do any additional startup it might need
self.preferences.postStartup()
self.addedPostStartupTasks = False
# Variables used by enterprise collector in resmgr
#
# flag that indicates we have finished loading the configs for the first time after a restart
self.firstConfigLoadDone = False
# flag that indicates the daemon has received the encryption key from zenhub
self.encryptionKeyInitialized = False
# flag that indicates the daemon is loading the cached configs
self.loadingCachedConfigs = False
def __init__(self, preferences, taskSplitter,
configurationListener=DUMMY_LISTENER,
initializationCallback=None,
stoppingCallback=None):
"""
Constructs a new instance of the CollectorDaemon framework. Normally
only a singleton instance of a CollectorDaemon should exist within a
process, but this is not enforced.
@param preferences: the collector configuration
@type preferences: ICollectorPreferences
@param taskSplitter: the task splitter to use for this collector
@type taskSplitter: ITaskSplitter
@param initializationCallback: a callable that will be executed after
connection to the hub but before
retrieving configuration information
@type initializationCallback: any callable
@param stoppingCallback: a callable that will be executed first during
the stopping process. Exceptions will be
logged but otherwise ignored.
@type stoppingCallback: any callable
"""
# create the configuration first, so we have the collector name
# available before activating the rest of the Daemon class hierarchy.
if not ICollectorPreferences.providedBy(preferences):
raise TypeError("configuration must provide ICollectorPreferences")
else:
self._prefs = ObservableProxy(preferences)
self._prefs.attachAttributeObserver('configCycleInterval', self._rescheduleConfig)
if not ITaskSplitter.providedBy(taskSplitter):
raise TypeError("taskSplitter must provide ITaskSplitter")
else:
self._taskSplitter = taskSplitter
if not IConfigurationListener.providedBy(configurationListener):
raise TypeError(
"configurationListener must provide IConfigurationListener")
self._configListener = ConfigListenerNotifier()
self._configListener.addListener(configurationListener)
self._configListener.addListener(DeviceGuidListener(self))
self._initializationCallback = initializationCallback
self._stoppingCallback = stoppingCallback
# register the various interfaces we provide the rest of the system so
# that collector implementors can easily retrieve a reference back here
# if needed
zope.component.provideUtility(self, ICollector)
zope.component.provideUtility(self, IEventService)
zope.component.provideUtility(self, IDataService)
# register the collector's own preferences object so it may be easily
# retrieved by factories, tasks, etc.
zope.component.provideUtility(self.preferences,
ICollectorPreferences,
self.preferences.collectorName)
super(CollectorDaemon, self).__init__(name=self.preferences.collectorName)
self._statService = StatisticsService()
zope.component.provideUtility(self._statService, IStatisticsService)
if self.options.cycle:
# setup daemon statistics (deprecated names)
self._statService.addStatistic("devices", "GAUGE")
self._statService.addStatistic("dataPoints", "DERIVE")
self._statService.addStatistic("runningTasks", "GAUGE")
self._statService.addStatistic("taskCount", "GAUGE")
self._statService.addStatistic("queuedTasks", "GAUGE")
self._statService.addStatistic("missedRuns", "GAUGE")
# namespace these a bit so they can be used in ZP monitoring.
# prefer these stat names and metrology in future refs
self._dataPointsMetric = Metrology.meter("collectordaemon.dataPoints")
daemon = self
class DeviceGauge(Gauge):
@property
def value(self):
return len(daemon._devices)
Metrology.gauge('collectordaemon.devices', DeviceGauge())
# Scheduler statistics
class RunningTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.running
Metrology.gauge('collectordaemon.runningTasks', RunningTasks())
class TaskCount(Gauge):
@property
def value(self):
return daemon._scheduler.taskCount
Metrology.gauge('collectordaemon.taskCount', TaskCount())
class QueuedTasks(Gauge):
@property
def value(self):
return daemon._scheduler._executor.queued
Metrology.gauge('collectordaemon.queuedTasks', QueuedTasks())
class MissedRuns(Gauge):
@property
def value(self):
return daemon._scheduler.missedRuns
Metrology.gauge('collectordaemon.missedRuns', MissedRuns())
self._deviceGuids = {}
self._devices = set()
self._unresponsiveDevices = set()
self._rrd = None
self._metric_writer = None
self._derivative_tracker = None
self.reconfigureTimeout = None
# keep track of pending tasks if we're doing a single run, and not a
# continuous cycle
if not self.options.cycle:
self._completedTasks = 0
self._pendingTasks = []
frameworkFactory = zope.component.queryUtility(IFrameworkFactory, self._frameworkFactoryName)
self._configProxy = frameworkFactory.getConfigurationProxy()
self._scheduler = frameworkFactory.getScheduler()
self._scheduler.maxTasks = self.options.maxTasks
self._ConfigurationLoaderTask = frameworkFactory.getConfigurationLoaderTask()
# OLD - set the initialServices attribute so that the PBDaemon class
# will load all of the remote services we need.
self.initialServices = PBDaemon.initialServices +\
[self.preferences.configurationService]
# trap SIGUSR2 so that we can display detailed statistics
signal.signal(signal.SIGUSR2, self._signalHandler)
# let the configuration do any additional startup it might need
self.preferences.postStartup()
self.addedPostStartupTasks = False
# Variables used by enterprise collector in resmgr
#
# flag that indicates we have finished loading the configs for the first time after a restart
self.firstConfigLoadDone = False
# flag that indicates the daemon has received the encryption key from zenhub
self.encryptionKeyInitialized = False
# flag that indicates the daemon is loading the cached configs
self.loadingCachedConfigs = False
def __init__(self):
"""
Hook ourselves up to the Zeo database and wait for collectors
to connect.
"""
# list of remote worker references
self.workers = []
self.workTracker = {}
# zenhub execution stats:
# [count, idle_total, running_total, last_called_time]
self.executionTimer = collections.defaultdict(lambda: [0, 0.0, 0.0, 0])
self.workList = _ZenHubWorklist()
# set of worker processes
self.worker_processes = set()
# map of worker pids -> worker processes
self.workerprocessmap = {}
self.shutdown = False
self.counters = collections.Counter()
self._invalidations_paused = False
wl = self.workList
metricNames = {x[0] for x in registry}
class EventWorkList(Gauge):
@property
def value(self):
return len(wl.eventworklist)
if 'zenhub.eventWorkList' not in metricNames:
Metrology.gauge('zenhub.eventWorkList', EventWorkList())
class ADMWorkList(Gauge):
@property
def value(self):
return len(wl.applyworklist)
if 'zenhub.admWorkList' not in metricNames:
Metrology.gauge('zenhub.admWorkList', ADMWorkList())
class OtherWorkList(Gauge):
@property
def value(self):
return len(wl.otherworklist)
if 'zenhub.otherWorkList' not in metricNames:
Metrology.gauge('zenhub.otherWorkList', OtherWorkList())
class WorkListTotal(Gauge):
@property
def value(self):
return len(wl)
if 'zenhub.workList' not in metricNames:
Metrology.gauge('zenhub.workList', WorkListTotal())
ZCmdBase.__init__(self)
import Products.ZenHub
load_config("hub.zcml", Products.ZenHub)
notify(HubWillBeCreatedEvent(self))
if self.options.profiling:
self.profiler = ContinuousProfiler('zenhub', log=self.log)
self.profiler.start()
# Worker selection handler
self.workerselector = WorkerSelector(self.options)
self.workList.log = self.log
# make sure we don't reserve more than n-1 workers for events
maxReservedEventsWorkers = 0
if self.options.workers:
maxReservedEventsWorkers = self.options.workers - 1
if self.options.workersReservedForEvents > maxReservedEventsWorkers:
self.options.workersReservedForEvents = maxReservedEventsWorkers
self.log.info(
"reduced number of workers reserved for sending events to %d",
self.options.workersReservedForEvents)
self.zem = self.dmd.ZenEventManager
loadPlugins(self.dmd)
self.services = {}
er = HubRealm(self)
checker = self.loadChecker()
pt = portal.Portal(er, [checker])
interface = '::' if ipv6_available() else ''
pbport = reactor.listenTCP(self.options.pbport,
pb.PBServerFactory(pt),
interface=interface)
self.setKeepAlive(pbport.socket)
xmlsvc = AuthXmlRpcService(self.dmd, checker)
reactor.listenTCP(self.options.xmlrpcport,
server.Site(xmlsvc),
interface=interface)
# responsible for sending messages to the queues
import Products.ZenMessaging.queuemessaging
load_config_override('twistedpublisher.zcml',
Products.ZenMessaging.queuemessaging)
notify(HubCreatedEvent(self))
self.sendEvent(eventClass=App_Start,
summary="%s started" % self.name,
severity=0)
self._initialize_invalidation_filters()
reactor.callLater(self.options.invalidation_poll_interval,
self.processQueue)
self._metric_writer = metricWriter()
self.rrdStats = self.getRRDStats()
if self.options.workers:
self.workerconfig = zenPath(
'var', 'zenhub', '{}_worker.conf'.format(self._getConf().id))
self._createWorkerConf()
for i in range(self.options.workers):
self.createWorker(i)
# start cyclic call to giveWorkToWorkers
reactor.callLater(2, self.giveWorkToWorkers, True)
# set up SIGUSR2 handling
try:
signal.signal(signal.SIGUSR2, self.sighandler_USR2)
except ValueError:
# If we get called multiple times, this will generate an exception:
# ValueError: signal only works in main thread
# Ignore it as we've already set up the signal handler.
pass
# ZEN-26671 Wait at least this duration in secs
# before signaling a worker process
self.SIGUSR_TIMEOUT = 5
