def register_metrics_on_worklist(worklist): metricNames = {x[0] for x in registry} for metricName, priority in _gauge_priority_map.iteritems(): if metricName not in metricNames: gauge = PriorityListLengthGauge(worklist, priority) Metrology.gauge(metricName, gauge) # Original metric name if "zenhub.workList" not in metricNames: gauge = WorklistLengthGauge(worklist) Metrology.gauge("zenhub.workList", gauge)
def __init__(self, options, log): self.options = options self.transformers = _load_utilities(ICollectorEventTransformer) self.log = log self.discarded_events = 0 # TODO: Do we want to limit the size of the clear event dictionary? self.clear_events_count = {} self._initQueues() self._eventsSent = Metrology.meter("collectordaemon.eventsSent") self._discardedEvents = Metrology.meter("collectordaemon.discardedEvent") self._eventTimer = Metrology.timer('collectordaemon.eventTimer') metricNames = {x[0] for x in registry} if 'collectordaemon.eventQueue' not in metricNames: queue = self class EventQueueGauge(Gauge): @property def value(self): return queue.event_queue_length Metrology.gauge('collectordaemon.eventQueue', EventQueueGauge())
def __init__(self, options, log): self.options = options self.transformers = _load_utilities(ICollectorEventTransformer) self.log = log self.discarded_events = 0 # TODO: Do we want to limit the size of the clear event dictionary? self.clear_events_count = {} self._initQueues() self._eventsSent = Metrology.meter("collectordaemon.eventsSent") self._discardedEvents = Metrology.meter( "collectordaemon.discardedEvent") self._eventTimer = Metrology.timer('collectordaemon.eventTimer') metricNames = {x[0] for x in registry} if 'collectordaemon.eventQueue' not in metricNames: queue = self class EventQueueGauge(Gauge): @property def value(self): return queue.event_queue_length Metrology.gauge('collectordaemon.eventQueue', EventQueueGauge())
def register_legacy_worklist_metrics(): """Create the Metrology counters for tracking worklist statistics.""" config = getUtility(IHubServerConfig) global _legacy_worklist_counters for metricName, priorityName in config.legacy_metric_priority_map.items(): gauge = registry.metrics.get(metricName) priority = ServiceCallPriority[priorityName] if not gauge: gauge = WorkListGauge(_legacy_worklist_counters, priority) Metrology.gauge(metricName, gauge) _legacy_worklist_counters[priority] = 0 gauge = registry.metrics.get(_legacy_metric_worklist_total.metric) if not gauge: gauge = WorkListGauge( _legacy_worklist_counters, _legacy_metric_worklist_total.name, ) Metrology.gauge(_legacy_metric_worklist_total.metric, gauge) _legacy_worklist_counters["total"] = 0 global _legacy_events_meter _legacy_events_meter = Metrology.meter("zenhub.eventsSent")
def test_gauge(self): self.assertTrue(Metrology.gauge('test', Gauge) is not None)
def __init__(self, preferences, taskSplitter, configurationListener=DUMMY_LISTENER, initializationCallback=None, stoppingCallback=None): """ Constructs a new instance of the CollectorDaemon framework. Normally only a singleton instance of a CollectorDaemon should exist within a process, but this is not enforced. @param preferences: the collector configuration @type preferences: ICollectorPreferences @param taskSplitter: the task splitter to use for this collector @type taskSplitter: ITaskSplitter @param initializationCallback: a callable that will be executed after connection to the hub but before retrieving configuration information @type initializationCallback: any callable @param stoppingCallback: a callable that will be executed first during the stopping process. Exceptions will be logged but otherwise ignored. @type stoppingCallback: any callable """ # create the configuration first, so we have the collector name # available before activating the rest of the Daemon class hierarchy. if not ICollectorPreferences.providedBy(preferences): raise TypeError("configuration must provide ICollectorPreferences") else: self._prefs = ObservableProxy(preferences) self._prefs.attachAttributeObserver('configCycleInterval', self._rescheduleConfig) if not ITaskSplitter.providedBy(taskSplitter): raise TypeError("taskSplitter must provide ITaskSplitter") else: self._taskSplitter = taskSplitter if not IConfigurationListener.providedBy(configurationListener): raise TypeError( "configurationListener must provide IConfigurationListener") self._configListener = ConfigListenerNotifier() self._configListener.addListener(configurationListener) self._configListener.addListener(DeviceGuidListener(self)) self._initializationCallback = initializationCallback self._stoppingCallback = stoppingCallback # register the various interfaces we provide the rest of the system so # that collector implementors can easily retrieve a reference back here # if needed zope.component.provideUtility(self, ICollector) zope.component.provideUtility(self, IEventService) zope.component.provideUtility(self, IDataService) # register the collector's own preferences object so it may be easily # retrieved by factories, tasks, etc. zope.component.provideUtility(self.preferences, ICollectorPreferences, self.preferences.collectorName) super(CollectorDaemon, self).__init__(name=self.preferences.collectorName) self._statService = StatisticsService() zope.component.provideUtility(self._statService, IStatisticsService) if self.options.cycle: # setup daemon statistics (deprecated names) self._statService.addStatistic("devices", "GAUGE") self._statService.addStatistic("dataPoints", "DERIVE") self._statService.addStatistic("runningTasks", "GAUGE") self._statService.addStatistic("taskCount", "GAUGE") self._statService.addStatistic("queuedTasks", "GAUGE") self._statService.addStatistic("missedRuns", "GAUGE") # namespace these a bit so they can be used in ZP monitoring. # prefer these stat names and metrology in future refs self._dataPointsMetric = Metrology.meter( "collectordaemon.dataPoints") daemon = self class DeviceGauge(Gauge): @property def value(self): return len(daemon._devices) Metrology.gauge('collectordaemon.devices', DeviceGauge()) # Scheduler statistics class RunningTasks(Gauge): @property def value(self): return daemon._scheduler._executor.running Metrology.gauge('collectordaemon.runningTasks', RunningTasks()) class TaskCount(Gauge): @property def value(self): return daemon._scheduler.taskCount Metrology.gauge('collectordaemon.taskCount', TaskCount()) class QueuedTasks(Gauge): @property def value(self): return daemon._scheduler._executor.queued Metrology.gauge('collectordaemon.queuedTasks', QueuedTasks()) class MissedRuns(Gauge): @property def value(self): return daemon._scheduler.missedRuns Metrology.gauge('collectordaemon.missedRuns', MissedRuns()) self._deviceGuids = {} self._devices = set() self._unresponsiveDevices = set() self._rrd = None self._metric_writer = None self._derivative_tracker = None self.reconfigureTimeout = None # keep track of pending tasks if we're doing a single run, and not a # continuous cycle if not self.options.cycle: self._completedTasks = 0 self._pendingTasks = [] frameworkFactory = zope.component.queryUtility( IFrameworkFactory, self._frameworkFactoryName) self._configProxy = frameworkFactory.getConfigurationProxy() self._scheduler = frameworkFactory.getScheduler() self._scheduler.maxTasks = self.options.maxTasks self._ConfigurationLoaderTask = frameworkFactory.getConfigurationLoaderTask( ) # OLD - set the initialServices attribute so that the PBDaemon class # will load all of the remote services we need. self.initialServices = PBDaemon.initialServices +\ [self.preferences.configurationService] # trap SIGUSR2 so that we can display detailed statistics signal.signal(signal.SIGUSR2, self._signalHandler) # let the configuration do any additional startup it might need self.preferences.postStartup() self.addedPostStartupTasks = False # Variables used by enterprise collector in resmgr # # flag that indicates we have finished loading the configs for the first time after a restart self.firstConfigLoadDone = False # flag that indicates the daemon has received the encryption key from zenhub self.encryptionKeyInitialized = False # flag that indicates the daemon is loading the cached configs self.loadingCachedConfigs = False
def __init__(self, preferences, taskSplitter, configurationListener=DUMMY_LISTENER, initializationCallback=None, stoppingCallback=None): """ Constructs a new instance of the CollectorDaemon framework. Normally only a singleton instance of a CollectorDaemon should exist within a process, but this is not enforced. @param preferences: the collector configuration @type preferences: ICollectorPreferences @param taskSplitter: the task splitter to use for this collector @type taskSplitter: ITaskSplitter @param initializationCallback: a callable that will be executed after connection to the hub but before retrieving configuration information @type initializationCallback: any callable @param stoppingCallback: a callable that will be executed first during the stopping process. Exceptions will be logged but otherwise ignored. @type stoppingCallback: any callable """ # create the configuration first, so we have the collector name # available before activating the rest of the Daemon class hierarchy. if not ICollectorPreferences.providedBy(preferences): raise TypeError("configuration must provide ICollectorPreferences") else: self._prefs = ObservableProxy(preferences) self._prefs.attachAttributeObserver('configCycleInterval', self._rescheduleConfig) if not ITaskSplitter.providedBy(taskSplitter): raise TypeError("taskSplitter must provide ITaskSplitter") else: self._taskSplitter = taskSplitter if not IConfigurationListener.providedBy(configurationListener): raise TypeError( "configurationListener must provide IConfigurationListener") self._configListener = ConfigListenerNotifier() self._configListener.addListener(configurationListener) self._configListener.addListener(DeviceGuidListener(self)) self._initializationCallback = initializationCallback self._stoppingCallback = stoppingCallback # register the various interfaces we provide the rest of the system so # that collector implementors can easily retrieve a reference back here # if needed zope.component.provideUtility(self, ICollector) zope.component.provideUtility(self, IEventService) zope.component.provideUtility(self, IDataService) # register the collector's own preferences object so it may be easily # retrieved by factories, tasks, etc. zope.component.provideUtility(self.preferences, ICollectorPreferences, self.preferences.collectorName) super(CollectorDaemon, self).__init__(name=self.preferences.collectorName) self._statService = StatisticsService() zope.component.provideUtility(self._statService, IStatisticsService) if self.options.cycle: # setup daemon statistics (deprecated names) self._statService.addStatistic("devices", "GAUGE") self._statService.addStatistic("dataPoints", "DERIVE") self._statService.addStatistic("runningTasks", "GAUGE") self._statService.addStatistic("taskCount", "GAUGE") self._statService.addStatistic("queuedTasks", "GAUGE") self._statService.addStatistic("missedRuns", "GAUGE") # namespace these a bit so they can be used in ZP monitoring. # prefer these stat names and metrology in future refs self._dataPointsMetric = Metrology.meter("collectordaemon.dataPoints") daemon = self class DeviceGauge(Gauge): @property def value(self): return len(daemon._devices) Metrology.gauge('collectordaemon.devices', DeviceGauge()) # Scheduler statistics class RunningTasks(Gauge): @property def value(self): return daemon._scheduler._executor.running Metrology.gauge('collectordaemon.runningTasks', RunningTasks()) class TaskCount(Gauge): @property def value(self): return daemon._scheduler.taskCount Metrology.gauge('collectordaemon.taskCount', TaskCount()) class QueuedTasks(Gauge): @property def value(self): return daemon._scheduler._executor.queued Metrology.gauge('collectordaemon.queuedTasks', QueuedTasks()) class MissedRuns(Gauge): @property def value(self): return daemon._scheduler.missedRuns Metrology.gauge('collectordaemon.missedRuns', MissedRuns()) self._deviceGuids = {} self._devices = set() self._unresponsiveDevices = set() self._rrd = None self._metric_writer = None self._derivative_tracker = None self.reconfigureTimeout = None # keep track of pending tasks if we're doing a single run, and not a # continuous cycle if not self.options.cycle: self._completedTasks = 0 self._pendingTasks = [] frameworkFactory = zope.component.queryUtility(IFrameworkFactory, self._frameworkFactoryName) self._configProxy = frameworkFactory.getConfigurationProxy() self._scheduler = frameworkFactory.getScheduler() self._scheduler.maxTasks = self.options.maxTasks self._ConfigurationLoaderTask = frameworkFactory.getConfigurationLoaderTask() # OLD - set the initialServices attribute so that the PBDaemon class # will load all of the remote services we need. self.initialServices = PBDaemon.initialServices +\ [self.preferences.configurationService] # trap SIGUSR2 so that we can display detailed statistics signal.signal(signal.SIGUSR2, self._signalHandler) # let the configuration do any additional startup it might need self.preferences.postStartup() self.addedPostStartupTasks = False # Variables used by enterprise collector in resmgr # # flag that indicates we have finished loading the configs for the first time after a restart self.firstConfigLoadDone = False # flag that indicates the daemon has received the encryption key from zenhub self.encryptionKeyInitialized = False # flag that indicates the daemon is loading the cached configs self.loadingCachedConfigs = False
def __init__(self): """ Hook ourselves up to the Zeo database and wait for collectors to connect. """ # list of remote worker references self.workers = [] self.workTracker = {} # zenhub execution stats: # [count, idle_total, running_total, last_called_time] self.executionTimer = collections.defaultdict(lambda: [0, 0.0, 0.0, 0]) self.workList = _ZenHubWorklist() # set of worker processes self.worker_processes = set() # map of worker pids -> worker processes self.workerprocessmap = {} self.shutdown = False self.counters = collections.Counter() self._invalidations_paused = False wl = self.workList metricNames = {x[0] for x in registry} class EventWorkList(Gauge): @property def value(self): return len(wl.eventworklist) if 'zenhub.eventWorkList' not in metricNames: Metrology.gauge('zenhub.eventWorkList', EventWorkList()) class ADMWorkList(Gauge): @property def value(self): return len(wl.applyworklist) if 'zenhub.admWorkList' not in metricNames: Metrology.gauge('zenhub.admWorkList', ADMWorkList()) class OtherWorkList(Gauge): @property def value(self): return len(wl.otherworklist) if 'zenhub.otherWorkList' not in metricNames: Metrology.gauge('zenhub.otherWorkList', OtherWorkList()) class WorkListTotal(Gauge): @property def value(self): return len(wl) if 'zenhub.workList' not in metricNames: Metrology.gauge('zenhub.workList', WorkListTotal()) ZCmdBase.__init__(self) import Products.ZenHub load_config("hub.zcml", Products.ZenHub) notify(HubWillBeCreatedEvent(self)) if self.options.profiling: self.profiler = ContinuousProfiler('zenhub', log=self.log) self.profiler.start() # Worker selection handler self.workerselector = WorkerSelector(self.options) self.workList.log = self.log # make sure we don't reserve more than n-1 workers for events maxReservedEventsWorkers = 0 if self.options.workers: maxReservedEventsWorkers = self.options.workers - 1 if self.options.workersReservedForEvents > maxReservedEventsWorkers: self.options.workersReservedForEvents = maxReservedEventsWorkers self.log.info( "reduced number of workers reserved for sending events to %d", self.options.workersReservedForEvents) self.zem = self.dmd.ZenEventManager loadPlugins(self.dmd) self.services = {} er = HubRealm(self) checker = self.loadChecker() pt = portal.Portal(er, [checker]) interface = '::' if ipv6_available() else '' pbport = reactor.listenTCP(self.options.pbport, pb.PBServerFactory(pt), interface=interface) self.setKeepAlive(pbport.socket) xmlsvc = AuthXmlRpcService(self.dmd, checker) reactor.listenTCP(self.options.xmlrpcport, server.Site(xmlsvc), interface=interface) # responsible for sending messages to the queues import Products.ZenMessaging.queuemessaging load_config_override('twistedpublisher.zcml', Products.ZenMessaging.queuemessaging) notify(HubCreatedEvent(self)) self.sendEvent(eventClass=App_Start, summary="%s started" % self.name, severity=0) self._initialize_invalidation_filters() reactor.callLater(self.options.invalidation_poll_interval, self.processQueue) self._metric_writer = metricWriter() self.rrdStats = self.getRRDStats() if self.options.workers: self.workerconfig = zenPath( 'var', 'zenhub', '{}_worker.conf'.format(self._getConf().id)) self._createWorkerConf() for i in range(self.options.workers): self.createWorker(i) # start cyclic call to giveWorkToWorkers reactor.callLater(2, self.giveWorkToWorkers, True) # set up SIGUSR2 handling try: signal.signal(signal.SIGUSR2, self.sighandler_USR2) except ValueError: # If we get called multiple times, this will generate an exception: # ValueError: signal only works in main thread # Ignore it as we've already set up the signal handler. pass # ZEN-26671 Wait at least this duration in secs # before signaling a worker process self.SIGUSR_TIMEOUT = 5