def __init__(self, noopts=0, keeproot=False, name=None):
# if we were provided our collector name via the constructor instead of
# via code, be sure to store it correctly.
if name is not None:
self.name = name
self.mname = name
try:
ZenDaemon.__init__(self, noopts, keeproot)
except IOError:
import traceback
self.log.critical(traceback.format_exc(0))
sys.exit(1)
self.rrdStats = DaemonStats()
self.lastStats = 0
self.perspective = None
self.services = {}
self.eventQueueManager = EventQueueManager(self.options, self.log)
self.startEvent = startEvent.copy()
self.stopEvent = stopEvent.copy()
details = dict(component=self.name, device=self.options.monitor)
for evt in self.startEvent, self.stopEvent, self.heartbeatEvent:
evt.update(details)
self.initialConnect = defer.Deferred()
self.stopped = False
self.counters = collections.Counter()
self.loadCounters()
self._pingedZenhub = None
self._connectionTimeout = None
# Add a shutdown trigger to send a stop event and flush the event queue
reactor.addSystemEventTrigger('before', 'shutdown', self._stopPbDaemon)
def __init__(self, noopts=0, keeproot=False, name=None):
# if we were provided our collector name via the constructor instead of
# via code, be sure to store it correctly.
if name is not None:
self.name = name
self.mname = name
try:
ZenDaemon.__init__(self, noopts, keeproot)
except IOError:
import traceback
self.log.critical( traceback.format_exc( 0 ) )
sys.exit(1)
self.rrdStats = DaemonStats()
self.lastStats = 0
self.perspective = None
self.services = {}
self.eventQueueManager = EventQueueManager(self.options, self.log)
self.startEvent = startEvent.copy()
self.stopEvent = stopEvent.copy()
details = dict(component=self.name, device=self.options.monitor)
for evt in self.startEvent, self.stopEvent, self.heartbeatEvent:
evt.update(details)
self.initialConnect = defer.Deferred()
self.stopped = False
self.counters = collections.Counter()
self.loadCounters()
self._pingedZenhub = None
self._connectionTimeout = None
# Add a shutdown trigger to send a stop event and flush the event queue
reactor.addSystemEventTrigger('before', 'shutdown', self._stopPbDaemon)
def buildOptions(self):
"""basic options setup sub classes can add more options here"""
ZenDaemon.buildOptions(self)
self.parser.add_option("--createcatalog",
action="store_true",
default=False,
help="Create global catalog and populate it")
self.parser.add_option(
"--forceindex",
action="store_true",
default=False,
help="works with --createcatalog to re-create index, "
"if catalog exists it will be dropped first")
self.parser.add_option("--reindex",
action="store_true",
default=False,
help="reindex existing catalog")
self.parser.add_option("--permissionsOnly",
action="store_true",
default=False,
help="Only works with --reindex, only update "\
" the permissions catalog")
self.parser.add_option(
"--clean",
action="store_true",
default=False,
help="Cleans up unresolvable paths in the catalog")
self.parser.add_option("--resume",
action="store_true",
default=False,
help="continue indexing after an interruption")
self.parser.add_option(
"--workers",
type="int",
default=4,
help="Number of processes working simultaneously")
self.parser.add_option(
"--buffersize",
type="int",
default=200,
help="Number of indexed documents to batch up in one commit")
self.parser.add_option(
"--inputqueuesize",
type="int",
default=300,
help="Maximum number of objects to have backlogged to process")
self.parser.add_option(
"--processedqueuesize",
type="int",
default=300,
help="Maximum number of objects to have backlogged to commit")
def buildOptions(self):
"""basic options setup sub classes can add more options here"""
ZenDaemon.buildOptions(self)
self.parser.add_option("--createcatalog",
action="store_true",
default=False,
help="Create global catalog and populate it")
self.parser.add_option("--forceindex",
action="store_true",
default=False,
help="works with --createcatalog to re-create index, "
"if catalog exists it will be dropped first")
self.parser.add_option("--reindex",
action="store_true",
default=False,
help="reindex existing catalog")
self.parser.add_option("--permissionsOnly",
action="store_true",
default=False,
help="Only works with --reindex, only update " \
" permissions")
self.parser.add_option("--resume",
action="store_true",
default=False,
help="DEPRECATED")
self.parser.add_option("--clearmemcached",
action="store_true",
default=False,
help="clears memcached after processing")
self.parser.add_option('--zodb-cacheservers',
dest='zodb_cacheservers',
default="",
help='memcached servers (used for --clearmemcached option')
self.parser.add_option("--workers",
type="int",
default=8,
help="Number of processes working simultaneously")
self.parser.add_option("--buffersize",
type="int",
default=200,
help="DEPRECATED")
self.parser.add_option("--inputqueuesize",
type="int",
default=300,
help="DEPRECATED")
self.parser.add_option("--processedqueuesize",
type="int",
default=300,
help="DEPRECATED")
def buildOptions(self):
"""
Adds our command line options to ZCmdBase command line options.
"""
ZenDaemon.buildOptions(self)
self.parser.add_option('--job-log-path', dest='job_log_path',
default=zenPath("log", "jobs"),
help='Directory in which to store individual job log files')
self.parser.add_option('--max-jobs-per-worker',
dest='max_jobs_per_worker', type='int', default=1,
help='Number of jobs a worker process runs before it shuts down')
self.parser.add_option('--concurrent-jobs',
dest='num_workers', type='int', default=2,
help='Number of jobs to process concurrently')
connectionFactory = getUtility(IZodbFactoryLookup).get()
connectionFactory.buildOptions(self.parser)
def buildOptions(self):
"""basic options setup sub classes can add more options here"""
ZenDaemon.buildOptions(self)
self.parser.add_option("--createcatalog",
action="store_true",
default=False,
help="Create global catalog and populate it")
self.parser.add_option("--forceindex",
action="store_true",
default=False,
help="works with --createcatalog to re-create index, "
"if catalog exists it will be dropped first")
self.parser.add_option("--reindex",
action="store_true",
default=False,
help="reindex existing catalog")
self.parser.add_option("--permissionsOnly",
action="store_true",
default=False,
help="Only works with --reindex, only update "\
" the permissions catalog")
self.parser.add_option("--clean",
action="store_true",
default=False,
help="Cleans up unresolvable paths in the catalog")
self.parser.add_option("--resume",
action="store_true",
default=False,
help="continue indexing after an interruption")
self.parser.add_option("--workers",
type="int",
default=4,
help="Number of processes working simultaneously")
self.parser.add_option("--buffersize",
type="int",
default=200,
help="Number of indexed documents to batch up in one commit")
self.parser.add_option("--inputqueuesize",
type="int",
default=300,
help="Maximum number of objects to have backlogged to process")
self.parser.add_option("--processedqueuesize",
type="int",
default=300,
help="Maximum number of objects to have backlogged to commit")
def buildOptions(self):
ZenDaemon.buildOptions(self)
self.parser.add_option('--hubhost',
dest='hubhost',
default=DEFAULT_HUB_HOST,
help='Host of zenhub daemon.'
' Default is %s.' % DEFAULT_HUB_HOST)
self.parser.add_option('--hubport',
dest='hubport',
type='int',
default=DEFAULT_HUB_PORT,
help='Port zenhub listens on.'
'Default is %s.' % DEFAULT_HUB_PORT)
self.parser.add_option('--hubusername',
dest='hubusername',
default=DEFAULT_HUB_USERNAME,
help='Username for zenhub login.'
' Default is %s.' % DEFAULT_HUB_USERNAME)
self.parser.add_option('--hubpassword',
dest='hubpassword',
default=DEFAULT_HUB_PASSWORD,
help='Password for zenhub login.'
' Default is %s.' % DEFAULT_HUB_PASSWORD)
self.parser.add_option('--monitor',
dest='monitor',
default=DEFAULT_HUB_MONITOR,
help='Name of monitor instance to use for'
' configuration. Default is %s.'
% DEFAULT_HUB_MONITOR)
self.parser.add_option('--initialHubTimeout',
dest='hubtimeout',
type='int',
default=30,
help='Initial time to wait for a ZenHub '
'connection')
self.parser.add_option('--allowduplicateclears',
dest='allowduplicateclears',
default=False,
action='store_true',
help='Send clear events even when the most '
'recent event was also a clear event.')
self.parser.add_option('--duplicateclearinterval',
dest='duplicateclearinterval',
default=0,
type='int',
help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
'events.')
)
self.parser.add_option('--eventflushseconds',
dest='eventflushseconds',
default=5.,
type='float',
help='Seconds between attempts to flush '
'events to ZenHub.')
self.parser.add_option('--eventflushchunksize',
dest='eventflushchunksize',
default=50,
type='int',
help='Number of events to send to ZenHub'
'at one time')
self.parser.add_option('--maxqueuelen',
dest='maxqueuelen',
default=5000,
type='int',
help='Maximum number of events to queue')
self.parser.add_option('--queuehighwatermark',
dest='queueHighWaterMark',
default=0.75,
type='float',
help='The size, in percent, of the event queue when event pushback starts')
self.parser.add_option('--zenhubpinginterval',
dest='zhPingInterval',
default=120,
type='int',
help='How often to ping zenhub')
self.parser.add_option('--disable-event-deduplication',
dest='deduplicate_events',
default=True,
action='store_false',
help='Disable event de-duplication')
self.parser.add_option('--redis-url',
dest='redisUrl',
type='string',
default='redis://localhost:{default}/0'.format(default=publisher.defaultRedisPort),
help='redis connection string: redis://[hostname]:[port]/[db], default: %default')
self.parser.add_option('--metricBufferSize',
dest='metricBufferSize',
type='int',
default=publisher.defaultMetricBufferSize,
help='Number of metrics to buffer if redis goes down')
self.parser.add_option('--metricsChannel',
dest='metricsChannel',
type='string',
default=publisher.defaultMetricsChannel,
help='redis channel to which metrics are published')
self.parser.add_option('--maxOutstandingMetrics',
dest='maxOutstandingMetrics',
type='int',
default=publisher.defaultMaxOutstandingMetrics,
help='Max Number of metrics to allow in redis')
self.parser.add_option('--disable-ping-perspective',
dest='pingPerspective',
help="Enable or disable ping perspective",
default=True,
action='store_false')
self.parser.add_option('--writeStatistics',
dest='writeStatistics',
type='int',
default=30,
help='How often to write internal statistics value in seconds')
def __init__(self, *args, **kwargs):
ZenDaemon.__init__(self, *args, **kwargs)
self.setup_celery()
signal.signal(signal.SIGTERM, self.sigTerm)
signal.signal(signal.SIGINT, self.sigTerm)
def buildOptions(self):
self.parser.add_option('--hubhost',
dest='hubhost',
default=DEFAULT_HUB_HOST,
help='Host of zenhub daemon.'
' Default is %s.' % DEFAULT_HUB_HOST)
self.parser.add_option('--hubport',
dest='hubport',
type='int',
default=DEFAULT_HUB_PORT,
help='Port zenhub listens on.'
'Default is %s.' % DEFAULT_HUB_PORT)
self.parser.add_option('--hubusername',
dest='hubusername',
default=DEFAULT_HUB_USERNAME,
help='Username for zenhub login.'
' Default is %s.' % DEFAULT_HUB_USERNAME)
self.parser.add_option('--hubpassword',
dest='hubpassword',
default=DEFAULT_HUB_PASSWORD,
help='Password for zenhub login.'
' Default is %s.' % DEFAULT_HUB_PASSWORD)
self.parser.add_option('--monitor',
dest='monitor',
default=DEFAULT_HUB_MONITOR,
help='Name of monitor instance to use for'
' configuration. Default is %s.' %
DEFAULT_HUB_MONITOR)
self.parser.add_option('--initialHubTimeout',
dest='hubtimeout',
type='int',
default=30,
help='Initial time to wait for a ZenHub '
'connection')
self.parser.add_option('--allowduplicateclears',
dest='allowduplicateclears',
default=False,
action='store_true',
help='Send clear events even when the most '
'recent event was also a clear event.')
self.parser.add_option(
'--duplicateclearinterval',
dest='duplicateclearinterval',
default=0,
type='int',
help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
'events.'))
self.parser.add_option('--eventflushseconds',
dest='eventflushseconds',
default=5.,
type='float',
help='Seconds between attempts to flush '
'events to ZenHub.')
self.parser.add_option('--eventflushchunksize',
dest='eventflushchunksize',
default=50,
type='int',
help='Number of events to send to ZenHub'
'at one time')
self.parser.add_option('--maxqueuelen',
dest='maxqueuelen',
default=5000,
type='int',
help='Maximum number of events to queue')
self.parser.add_option(
'--queuehighwatermark',
dest='queueHighWaterMark',
default=0.75,
type='float',
help=
'The size, in percent, of the event queue when event pushback starts'
)
self.parser.add_option('--zenhubpinginterval',
dest='zhPingInterval',
default=120,
type='int',
help='How often to ping zenhub')
self.parser.add_option('--disable-event-deduplication',
dest='deduplicate_events',
default=True,
action='store_false',
help='Disable event de-duplication')
self.parser.add_option(
'--redis-url',
dest='redisUrl',
type='string',
default='redis://localhost:{default}/0'.format(
default=publisher.defaultRedisPort),
help=
'redis connection string: redis://[hostname]:[port]/[db], default: %default'
)
self.parser.add_option(
'--metricBufferSize',
dest='metricBufferSize',
type='int',
default=publisher.defaultMetricBufferSize,
help='Number of metrics to buffer if redis goes down')
self.parser.add_option(
'--metricsChannel',
dest='metricsChannel',
type='string',
default=publisher.defaultMetricsChannel,
help='redis channel to which metrics are published')
self.parser.add_option('--maxOutstandingMetrics',
dest='maxOutstandingMetrics',
type='int',
default=publisher.defaultMaxOutstandingMetrics,
help='Max Number of metrics to allow in redis')
self.parser.add_option('--disable-ping-perspective',
dest='pingPerspective',
help="Enable or disable ping perspective",
default=True,
action='store_false')
self.parser.add_option(
'--writeStatistics',
dest='writeStatistics',
type='int',
default=30,
help='How often to write internal statistics value in seconds')
ZenDaemon.buildOptions(self)
def buildOptions(self):
self.parser.add_option('--hubhost',
dest='hubhost',
default=DEFAULT_HUB_HOST,
help='Host of zenhub daemon.'
' Default is %s.' % DEFAULT_HUB_HOST)
self.parser.add_option('--hubport',
dest='hubport',
type='int',
default=DEFAULT_HUB_PORT,
help='Port zenhub listens on.'
'Default is %s.' % DEFAULT_HUB_PORT)
self.parser.add_option('--hubusername',
dest='hubusername',
default=DEFAULT_HUB_USERNAME,
help='Username for zenhub login.'
' Default is %s.' % DEFAULT_HUB_USERNAME)
self.parser.add_option('--hubpassword',
dest='hubpassword',
default=DEFAULT_HUB_PASSWORD,
help='Password for zenhub login.'
' Default is %s.' % DEFAULT_HUB_PASSWORD)
self.parser.add_option('--monitor',
dest='monitor',
default=DEFAULT_HUB_MONITOR,
help='Name of monitor instance to use for'
' configuration. Default is %s.' %
DEFAULT_HUB_MONITOR)
self.parser.add_option('--initialHubTimeout',
dest='hubtimeout',
type='int',
default=30,
help='Initial time to wait for a ZenHub '
'connection')
self.parser.add_option('--allowduplicateclears',
dest='allowduplicateclears',
default=False,
action='store_true',
help='Send clear events even when the most '
'recent event was also a clear event.')
self.parser.add_option(
'--duplicateclearinterval',
dest='duplicateclearinterval',
default=0,
type='int',
help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
'events.'))
self.parser.add_option('--eventflushseconds',
dest='eventflushseconds',
default=5.,
type='float',
help='Seconds between attempts to flush '
'events to ZenHub.')
self.parser.add_option('--eventflushchunksize',
dest='eventflushchunksize',
default=50,
type='int',
help='Number of events to send to ZenHub'
'at one time')
self.parser.add_option('--maxqueuelen',
dest='maxqueuelen',
default=5000,
type='int',
help='Maximum number of events to queue')
self.parser.add_option('--zenhubpinginterval',
dest='zhPingInterval',
default=30,
type='int',
help='How often to ping zenhub')
self.parser.add_option('--disable-event-deduplication',
dest='deduplicate_events',
default=True,
action='store_false',
help='Disable event de-duplication')
ZenDaemon.buildOptions(self)
def __init__(self, *args, **kwargs):
ZenDaemon.__init__(self, *args, **kwargs)
self.setup_celery()
def buildOptions(self):
self.parser.add_option('--hubhost',
dest='hubhost',
default=DEFAULT_HUB_HOST,
help='Host of zenhub daemon.'
' Default is %s.' % DEFAULT_HUB_HOST)
self.parser.add_option('--hubport',
dest='hubport',
type='int',
default=DEFAULT_HUB_PORT,
help='Port zenhub listens on.'
'Default is %s.' % DEFAULT_HUB_PORT)
self.parser.add_option('--hubusername',
dest='hubusername',
default=DEFAULT_HUB_USERNAME,
help='Username for zenhub login.'
' Default is %s.' % DEFAULT_HUB_USERNAME)
self.parser.add_option('--hubpassword',
dest='hubpassword',
default=DEFAULT_HUB_PASSWORD,
help='Password for zenhub login.'
' Default is %s.' % DEFAULT_HUB_PASSWORD)
self.parser.add_option('--monitor',
dest='monitor',
default=DEFAULT_HUB_MONITOR,
help='Name of monitor instance to use for'
' configuration. Default is %s.'
% DEFAULT_HUB_MONITOR)
self.parser.add_option('--initialHubTimeout',
dest='hubtimeout',
type='int',
default=30,
help='Initial time to wait for a ZenHub '
'connection')
self.parser.add_option('--allowduplicateclears',
dest='allowduplicateclears',
default=False,
action='store_true',
help='Send clear events even when the most '
'recent event was also a clear event.')
self.parser.add_option('--duplicateclearinterval',
dest='duplicateclearinterval',
default=0,
type='int',
help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
'events.')
)
self.parser.add_option('--eventflushseconds',
dest='eventflushseconds',
default=5.,
type='float',
help='Seconds between attempts to flush '
'events to ZenHub.')
self.parser.add_option('--eventflushchunksize',
dest='eventflushchunksize',
default=50,
type='int',
help='Number of events to send to ZenHub'
'at one time')
self.parser.add_option('--maxqueuelen',
dest='maxqueuelen',
default=5000,
type='int',
help='Maximum number of events to queue')
self.parser.add_option('--zenhubpinginterval',
dest='zhPingInterval',
default=30,
type='int',
help='How often to ping zenhub')
self.parser.add_option('--disable-event-deduplication',
dest='deduplicate_events',
default=True,
action='store_false',
help='Disable event de-duplication')
ZenDaemon.buildOptions(self)
def __init__(self, *args, **kwargs):
ZenDaemon.__init__(self, *args, **kwargs)
self.setup_celery()