Exemplo n.º 1
0
 def buildOptions(self):
     """basic options setup sub classes can add more options here"""
     ZenDaemon.buildOptions(self)
     self.parser.add_option("--createcatalog",
                            action="store_true",
                            default=False,
                            help="Create global catalog and populate it")
     self.parser.add_option(
         "--forceindex",
         action="store_true",
         default=False,
         help="works with --createcatalog to re-create index, "
         "if catalog exists it will be dropped first")
     self.parser.add_option("--reindex",
                            action="store_true",
                            default=False,
                            help="reindex existing catalog")
     self.parser.add_option("--permissionsOnly",
         action="store_true",
         default=False,
         help="Only works with --reindex, only update "\
              " the permissions catalog")
     self.parser.add_option(
         "--clean",
         action="store_true",
         default=False,
         help="Cleans up unresolvable paths in the catalog")
     self.parser.add_option("--resume",
                            action="store_true",
                            default=False,
                            help="continue indexing after an interruption")
     self.parser.add_option(
         "--workers",
         type="int",
         default=4,
         help="Number of processes working simultaneously")
     self.parser.add_option(
         "--buffersize",
         type="int",
         default=200,
         help="Number of indexed documents to batch up in one commit")
     self.parser.add_option(
         "--inputqueuesize",
         type="int",
         default=300,
         help="Maximum number of objects to have backlogged to process")
     self.parser.add_option(
         "--processedqueuesize",
         type="int",
         default=300,
         help="Maximum number of objects to have backlogged to commit")
Exemplo n.º 2
0
 def buildOptions(self):
     """basic options setup sub classes can add more options here"""
     ZenDaemon.buildOptions(self)
     self.parser.add_option("--createcatalog",
                            action="store_true",
                            default=False,
                            help="Create global catalog and populate it")
     self.parser.add_option("--forceindex",
                            action="store_true",
                            default=False,
                            help="works with --createcatalog to re-create index, "
                                 "if catalog exists it will be dropped first")
     self.parser.add_option("--reindex",
                            action="store_true",
                            default=False,
                            help="reindex existing catalog")
     self.parser.add_option("--permissionsOnly",
                            action="store_true",
                            default=False,
                            help="Only works with --reindex, only update " \
                                 " permissions")
     self.parser.add_option("--resume",
                            action="store_true",
                            default=False,
                            help="DEPRECATED")
     self.parser.add_option("--clearmemcached",
                            action="store_true",
                            default=False,
                            help="clears memcached after processing")
     self.parser.add_option('--zodb-cacheservers',
                            dest='zodb_cacheservers',
                            default="",
                            help='memcached servers (used for --clearmemcached option')
     self.parser.add_option("--workers",
                            type="int",
                            default=8,
                            help="Number of processes working simultaneously")
     self.parser.add_option("--buffersize",
                            type="int",
                            default=200,
                            help="DEPRECATED")
     self.parser.add_option("--inputqueuesize",
                            type="int",
                            default=300,
                            help="DEPRECATED")
     self.parser.add_option("--processedqueuesize",
                            type="int",
                            default=300,
                            help="DEPRECATED")
Exemplo n.º 3
0
 def buildOptions(self):
     """
     Adds our command line options to ZCmdBase command line options.
     """
     ZenDaemon.buildOptions(self)
     self.parser.add_option('--job-log-path', dest='job_log_path',
         default=zenPath("log", "jobs"),
         help='Directory in which to store individual job log files')
     self.parser.add_option('--max-jobs-per-worker',
         dest='max_jobs_per_worker', type='int', default=1,
         help='Number of jobs a worker process runs before it shuts down')
     self.parser.add_option('--concurrent-jobs',
         dest='num_workers', type='int', default=2,
         help='Number of jobs to process concurrently')
     connectionFactory = getUtility(IZodbFactoryLookup).get()
     connectionFactory.buildOptions(self.parser)
Exemplo n.º 4
0
 def buildOptions(self):
     """basic options setup sub classes can add more options here"""
     ZenDaemon.buildOptions(self)
     self.parser.add_option("--createcatalog",
         action="store_true",
         default=False,
         help="Create global catalog and populate it")
     self.parser.add_option("--forceindex",
         action="store_true",
         default=False,
         help="works with --createcatalog to re-create index, "
              "if catalog exists it will be dropped first")
     self.parser.add_option("--reindex",
         action="store_true",
         default=False,
         help="reindex existing catalog")
     self.parser.add_option("--permissionsOnly",
         action="store_true",
         default=False,
         help="Only works with --reindex, only update "\
              " the permissions catalog")
     self.parser.add_option("--clean",
         action="store_true",
         default=False,
         help="Cleans up unresolvable paths in the catalog")
     self.parser.add_option("--resume",
         action="store_true",
         default=False,
         help="continue indexing after an interruption")
     self.parser.add_option("--workers",
         type="int",
         default=4,
         help="Number of processes working simultaneously")
     self.parser.add_option("--buffersize",
         type="int",
         default=200,
         help="Number of indexed documents to batch up in one commit")
     self.parser.add_option("--inputqueuesize",
         type="int",
         default=300,
         help="Maximum number of objects to have backlogged to process")
     self.parser.add_option("--processedqueuesize",
         type="int",
         default=300,
         help="Maximum number of objects to have backlogged to commit")
Exemplo n.º 5
0
    def buildOptions(self):
        ZenDaemon.buildOptions(self)

        self.parser.add_option('--hubhost',
                                dest='hubhost',
                                default=DEFAULT_HUB_HOST,
                                help='Host of zenhub daemon.'
                                ' Default is %s.' % DEFAULT_HUB_HOST)
        self.parser.add_option('--hubport',
                                dest='hubport',
                                type='int',
                                default=DEFAULT_HUB_PORT,
                                help='Port zenhub listens on.'
                                    'Default is %s.' % DEFAULT_HUB_PORT)
        self.parser.add_option('--hubusername',
                                dest='hubusername',
                                default=DEFAULT_HUB_USERNAME,
                                help='Username for zenhub login.'
                                    ' Default is %s.' % DEFAULT_HUB_USERNAME)
        self.parser.add_option('--hubpassword',
                                dest='hubpassword',
                                default=DEFAULT_HUB_PASSWORD,
                                help='Password for zenhub login.'
                                    ' Default is %s.' % DEFAULT_HUB_PASSWORD)
        self.parser.add_option('--monitor',
                                dest='monitor',
                                default=DEFAULT_HUB_MONITOR,
                                help='Name of monitor instance to use for'
                                    ' configuration.  Default is %s.'
                                    % DEFAULT_HUB_MONITOR)
        self.parser.add_option('--initialHubTimeout',
                               dest='hubtimeout',
                               type='int',
                               default=30,
                               help='Initial time to wait for a ZenHub '
                                    'connection')
        self.parser.add_option('--allowduplicateclears',
                               dest='allowduplicateclears',
                               default=False,
                               action='store_true',
                               help='Send clear events even when the most '
                               'recent event was also a clear event.')

        self.parser.add_option('--duplicateclearinterval',
                               dest='duplicateclearinterval',
                               default=0,
                               type='int',
                               help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
                                     'events.')
        )

        self.parser.add_option('--eventflushseconds',
                               dest='eventflushseconds',
                               default=5.,
                               type='float',
                               help='Seconds between attempts to flush '
                               'events to ZenHub.')

        self.parser.add_option('--eventflushchunksize',
                               dest='eventflushchunksize',
                               default=50,
                               type='int',
                               help='Number of events to send to ZenHub'
                               'at one time')

        self.parser.add_option('--maxqueuelen',
                               dest='maxqueuelen',
                               default=5000,
                               type='int',
                               help='Maximum number of events to queue')

        self.parser.add_option('--queuehighwatermark',
                               dest='queueHighWaterMark',
                               default=0.75,
                               type='float',
                               help='The size, in percent, of the event queue when event pushback starts')
        self.parser.add_option('--zenhubpinginterval',
                               dest='zhPingInterval',
                               default=120,
                               type='int',
                               help='How often to ping zenhub')

        self.parser.add_option('--disable-event-deduplication',
                               dest='deduplicate_events',
                               default=True,
                               action='store_false',
                               help='Disable event de-duplication')

        self.parser.add_option('--redis-url',
                               dest='redisUrl',
                               type='string',
                               default='redis://localhost:{default}/0'.format(default=publisher.defaultRedisPort),
                               help='redis connection string: redis://[hostname]:[port]/[db], default: %default')

        self.parser.add_option('--metricBufferSize',
                               dest='metricBufferSize',
                               type='int',
                               default=publisher.defaultMetricBufferSize,
                               help='Number of metrics to buffer if redis goes down')
        self.parser.add_option('--metricsChannel',
                               dest='metricsChannel',
                               type='string',
                               default=publisher.defaultMetricsChannel,
                               help='redis channel to which metrics are published')
        self.parser.add_option('--maxOutstandingMetrics',
                               dest='maxOutstandingMetrics',
                               type='int',
                               default=publisher.defaultMaxOutstandingMetrics,
                               help='Max Number of metrics to allow in redis')
        self.parser.add_option('--disable-ping-perspective',
                               dest='pingPerspective',
                               help="Enable or disable ping perspective",
                               default=True,
                               action='store_false')
        self.parser.add_option('--writeStatistics',
                               dest='writeStatistics',
                               type='int',
                               default=30,
                               help='How often to write internal statistics value in seconds')
Exemplo n.º 6
0
    def buildOptions(self):
        self.parser.add_option('--hubhost',
                               dest='hubhost',
                               default=DEFAULT_HUB_HOST,
                               help='Host of zenhub daemon.'
                               ' Default is %s.' % DEFAULT_HUB_HOST)
        self.parser.add_option('--hubport',
                               dest='hubport',
                               type='int',
                               default=DEFAULT_HUB_PORT,
                               help='Port zenhub listens on.'
                               'Default is %s.' % DEFAULT_HUB_PORT)
        self.parser.add_option('--hubusername',
                               dest='hubusername',
                               default=DEFAULT_HUB_USERNAME,
                               help='Username for zenhub login.'
                               ' Default is %s.' % DEFAULT_HUB_USERNAME)
        self.parser.add_option('--hubpassword',
                               dest='hubpassword',
                               default=DEFAULT_HUB_PASSWORD,
                               help='Password for zenhub login.'
                               ' Default is %s.' % DEFAULT_HUB_PASSWORD)
        self.parser.add_option('--monitor',
                               dest='monitor',
                               default=DEFAULT_HUB_MONITOR,
                               help='Name of monitor instance to use for'
                               ' configuration.  Default is %s.' %
                               DEFAULT_HUB_MONITOR)
        self.parser.add_option('--initialHubTimeout',
                               dest='hubtimeout',
                               type='int',
                               default=30,
                               help='Initial time to wait for a ZenHub '
                               'connection')
        self.parser.add_option('--allowduplicateclears',
                               dest='allowduplicateclears',
                               default=False,
                               action='store_true',
                               help='Send clear events even when the most '
                               'recent event was also a clear event.')

        self.parser.add_option(
            '--duplicateclearinterval',
            dest='duplicateclearinterval',
            default=0,
            type='int',
            help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
                  'events.'))

        self.parser.add_option('--eventflushseconds',
                               dest='eventflushseconds',
                               default=5.,
                               type='float',
                               help='Seconds between attempts to flush '
                               'events to ZenHub.')

        self.parser.add_option('--eventflushchunksize',
                               dest='eventflushchunksize',
                               default=50,
                               type='int',
                               help='Number of events to send to ZenHub'
                               'at one time')

        self.parser.add_option('--maxqueuelen',
                               dest='maxqueuelen',
                               default=5000,
                               type='int',
                               help='Maximum number of events to queue')

        self.parser.add_option(
            '--queuehighwatermark',
            dest='queueHighWaterMark',
            default=0.75,
            type='float',
            help=
            'The size, in percent, of the event queue when event pushback starts'
        )
        self.parser.add_option('--zenhubpinginterval',
                               dest='zhPingInterval',
                               default=120,
                               type='int',
                               help='How often to ping zenhub')

        self.parser.add_option('--disable-event-deduplication',
                               dest='deduplicate_events',
                               default=True,
                               action='store_false',
                               help='Disable event de-duplication')

        self.parser.add_option(
            '--redis-url',
            dest='redisUrl',
            type='string',
            default='redis://localhost:{default}/0'.format(
                default=publisher.defaultRedisPort),
            help=
            'redis connection string: redis://[hostname]:[port]/[db], default: %default'
        )

        self.parser.add_option(
            '--metricBufferSize',
            dest='metricBufferSize',
            type='int',
            default=publisher.defaultMetricBufferSize,
            help='Number of metrics to buffer if redis goes down')
        self.parser.add_option(
            '--metricsChannel',
            dest='metricsChannel',
            type='string',
            default=publisher.defaultMetricsChannel,
            help='redis channel to which metrics are published')
        self.parser.add_option('--maxOutstandingMetrics',
                               dest='maxOutstandingMetrics',
                               type='int',
                               default=publisher.defaultMaxOutstandingMetrics,
                               help='Max Number of metrics to allow in redis')
        self.parser.add_option('--disable-ping-perspective',
                               dest='pingPerspective',
                               help="Enable or disable ping perspective",
                               default=True,
                               action='store_false')
        self.parser.add_option(
            '--writeStatistics',
            dest='writeStatistics',
            type='int',
            default=30,
            help='How often to write internal statistics value in seconds')

        ZenDaemon.buildOptions(self)
Exemplo n.º 7
0
    def buildOptions(self):
        self.parser.add_option('--hubhost',
                               dest='hubhost',
                               default=DEFAULT_HUB_HOST,
                               help='Host of zenhub daemon.'
                               ' Default is %s.' % DEFAULT_HUB_HOST)
        self.parser.add_option('--hubport',
                               dest='hubport',
                               type='int',
                               default=DEFAULT_HUB_PORT,
                               help='Port zenhub listens on.'
                               'Default is %s.' % DEFAULT_HUB_PORT)
        self.parser.add_option('--hubusername',
                               dest='hubusername',
                               default=DEFAULT_HUB_USERNAME,
                               help='Username for zenhub login.'
                               ' Default is %s.' % DEFAULT_HUB_USERNAME)
        self.parser.add_option('--hubpassword',
                               dest='hubpassword',
                               default=DEFAULT_HUB_PASSWORD,
                               help='Password for zenhub login.'
                               ' Default is %s.' % DEFAULT_HUB_PASSWORD)
        self.parser.add_option('--monitor',
                               dest='monitor',
                               default=DEFAULT_HUB_MONITOR,
                               help='Name of monitor instance to use for'
                               ' configuration.  Default is %s.' %
                               DEFAULT_HUB_MONITOR)
        self.parser.add_option('--initialHubTimeout',
                               dest='hubtimeout',
                               type='int',
                               default=30,
                               help='Initial time to wait for a ZenHub '
                               'connection')
        self.parser.add_option('--allowduplicateclears',
                               dest='allowduplicateclears',
                               default=False,
                               action='store_true',
                               help='Send clear events even when the most '
                               'recent event was also a clear event.')

        self.parser.add_option(
            '--duplicateclearinterval',
            dest='duplicateclearinterval',
            default=0,
            type='int',
            help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
                  'events.'))

        self.parser.add_option('--eventflushseconds',
                               dest='eventflushseconds',
                               default=5.,
                               type='float',
                               help='Seconds between attempts to flush '
                               'events to ZenHub.')

        self.parser.add_option('--eventflushchunksize',
                               dest='eventflushchunksize',
                               default=50,
                               type='int',
                               help='Number of events to send to ZenHub'
                               'at one time')

        self.parser.add_option('--maxqueuelen',
                               dest='maxqueuelen',
                               default=5000,
                               type='int',
                               help='Maximum number of events to queue')

        self.parser.add_option('--zenhubpinginterval',
                               dest='zhPingInterval',
                               default=30,
                               type='int',
                               help='How often to ping zenhub')

        self.parser.add_option('--disable-event-deduplication',
                               dest='deduplicate_events',
                               default=True,
                               action='store_false',
                               help='Disable event de-duplication')

        ZenDaemon.buildOptions(self)
Exemplo n.º 8
0
    def buildOptions(self):
        self.parser.add_option('--hubhost',
                                dest='hubhost',
                                default=DEFAULT_HUB_HOST,
                                help='Host of zenhub daemon.'
                                ' Default is %s.' % DEFAULT_HUB_HOST)
        self.parser.add_option('--hubport',
                                dest='hubport',
                                type='int',
                                default=DEFAULT_HUB_PORT,
                                help='Port zenhub listens on.'
                                    'Default is %s.' % DEFAULT_HUB_PORT)
        self.parser.add_option('--hubusername',
                                dest='hubusername',
                                default=DEFAULT_HUB_USERNAME,
                                help='Username for zenhub login.'
                                    ' Default is %s.' % DEFAULT_HUB_USERNAME)
        self.parser.add_option('--hubpassword',
                                dest='hubpassword',
                                default=DEFAULT_HUB_PASSWORD,
                                help='Password for zenhub login.'
                                    ' Default is %s.' % DEFAULT_HUB_PASSWORD)
        self.parser.add_option('--monitor', 
                                dest='monitor',
                                default=DEFAULT_HUB_MONITOR,
                                help='Name of monitor instance to use for'
                                    ' configuration.  Default is %s.'
                                    % DEFAULT_HUB_MONITOR)
        self.parser.add_option('--initialHubTimeout',
                               dest='hubtimeout',
                               type='int',
                               default=30,
                               help='Initial time to wait for a ZenHub '
                                    'connection')
        self.parser.add_option('--allowduplicateclears',
                               dest='allowduplicateclears',
                               default=False,
                               action='store_true',
                               help='Send clear events even when the most '
                               'recent event was also a clear event.')

        self.parser.add_option('--duplicateclearinterval',
                               dest='duplicateclearinterval',
                               default=0,
                               type='int',
                               help=('Send a clear event every [DUPLICATECLEARINTEVAL] '
                                     'events.')
        )

        self.parser.add_option('--eventflushseconds',
                               dest='eventflushseconds',
                               default=5.,
                               type='float',
                               help='Seconds between attempts to flush '
                               'events to ZenHub.')

        self.parser.add_option('--eventflushchunksize',
                               dest='eventflushchunksize',
                               default=50,
                               type='int',
                               help='Number of events to send to ZenHub'
                               'at one time')

        self.parser.add_option('--maxqueuelen',
                               dest='maxqueuelen',
                               default=5000,
                               type='int',
                               help='Maximum number of events to queue')

        self.parser.add_option('--zenhubpinginterval',
                               dest='zhPingInterval',
                               default=30,
                               type='int',
                               help='How often to ping zenhub')

        self.parser.add_option('--disable-event-deduplication',
                               dest='deduplicate_events',
                               default=True,
                               action='store_false',
                               help='Disable event de-duplication')

        ZenDaemon.buildOptions(self)