def pyres_manager(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) #parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port",dest="port",type="int", default=6379) parser.add_option("-i", '--interval', dest='interval', default=None, help='the default time interval to sleep between runs') parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.') parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.") parser.add_option('-f', dest='logfile', help='If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.') parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') (options,args) = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') #logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s") setup_pidfile(options.pidfile) interval = options.interval if interval is not None: interval = float(interval) queues = args[0].split(',') server = '%s:%s' % (options.host,options.port) Khan.run(pool_size=options.pool_size, queues=queues, server=server, logging_level=log_level, log_file=options.logfile)
def pyres_scheduler(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) #parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port", dest="port", type="int", default=6379) parser.add_option( '-l', '--log-level', dest='log_level', default='info', help= 'log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.' ) parser.add_option( '-f', dest='logfile', help= 'If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.' ) parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') (options, args) = parser.parse_args() log_level = getattr(logging, options.log_level.upper(), 'INFO') #logging.basicConfig(level=log_level, format="%(module)s: %(asctime)s: %(levelname)s: %(message)s") setup_logging(procname="pyres_scheduler", log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) server = '%s:%s' % (options.host, options.port) Scheduler.run(server)
def pyres_worker(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port",dest="port",type="int", default=6379) parser.add_option("-i", '--interval', dest='interval', default=None, help='the default time interval to sleep between runs') parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.') parser.add_option('-f', dest='logfile', help='If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.') parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') (options,args) = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') setup_logging(procname="pyres_worker", log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) interval = options.interval if interval is not None: interval = int(interval) queues = args[0].split(',') server = '%s:%s' % (options.host,options.port) Worker.run(queues, server, interval)
def pyres_scheduler(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) # parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port", dest="port", type="int", default=6379) parser.add_option( "-l", "--log-level", dest="log_level", default="info", help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.', ) parser.add_option( "-f", dest="logfile", help='If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.', ) parser.add_option("-p", dest="pidfile", help="If present, a pidfile will be used.") (options, args) = parser.parse_args() log_level = getattr(logging, options.log_level.upper(), "INFO") # logging.basicConfig(level=log_level, format="%(module)s: %(asctime)s: %(levelname)s: %(message)s") setup_logging(procname="pyres_scheduler", log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) server = "%s:%s" % (options.host, options.port) Scheduler.run(server)
def pyres_worker(): """Worker CLI, lightly modified from pyres. """ usage = 'usage: %prog [options] arg1' parser = OptionParser(usage=usage) parser.add_option('--host', dest='host', default=settings['host']) parser.add_option('--port', dest='port', type='int', default=settings['port']) parser.add_option('--password', dest='password', default=settings['password']) parser.add_option('-i', '--interval', dest='interval', default=None) parser.add_option('-l', '--log-level', dest='log_level', default='info') parser.add_option('-f', dest='logfile') parser.add_option('-p', dest='pidfile') parser.add_option('-t', '--timeout', dest='timeout') options, args = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') setup_logging(procname="pyres_worker", log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) interval = options.interval if interval is not None: interval = int(interval) timeout = options.timeout and int(options.timeout) queues = args[0].split(',') server = '{0}:{1}'.format(options.host, options.port) Worker.run(queues, server, options.password, interval, timeout=timeout)
def pyres_worker(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port", dest="port", type="int", default=6379) parser.add_option("--password", dest="password", default=None) parser.add_option("-i", '--interval', dest='interval', default=None, help='the default time interval to sleep between runs') parser.add_option( '-l', '--log-level', dest='log_level', default='info', help= 'log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.' ) parser.add_option( '-f', dest='logfile', help= 'If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.' ) parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') parser.add_option("-t", '--timeout', dest='timeout', default=None, help='the timeout in seconds for this worker') (options, args) = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') setup_logging(procname="pyres_worker", log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) interval = options.interval if interval is not None: interval = int(interval) timeout = options.timeout and int(options.timeout) queues = args[0].split(',') server = '%s:%s' % (options.host, options.port) password = options.password Worker.run(queues, server, password, interval, timeout=timeout)
def pyres_manager(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) #parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port", dest="port", type="int", default=6379) parser.add_option("-i", '--interval', dest='interval', default=None, help='the default time interval to sleep between runs') parser.add_option( '-l', '--log-level', dest='log_level', default='info', help= 'log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.' ) parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.") parser.add_option( '-f', dest='logfile', help= 'If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.' ) parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') (options, args) = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') #logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s") setup_pidfile(options.pidfile) interval = options.interval if interval is not None: interval = float(interval) queues = args[0].split(',') server = '%s:%s' % (options.host, options.port) Khan.run(pool_size=options.pool_size, queues=queues, server=server, logging_level=log_level, log_file=options.logfile)
def handle(self, queue_list, **options): queues = queue_list.split(',') log_level = getattr(logging, options['log_level'].upper(), 'INFO') pool_size = options.get('pool_size') log_file = options.get('logfile') if pool_size == 0: pool_size = settings.PYRES_MANAGER_POOL_SIZE setup_pidfile(settings.PYRES_WORKER_PIDFILE) worker = Khan(pool_size=pool_size, queues=queues, server=settings.PYRES_HOST, password=settings.PYRES_PASSWORD, logging_level=log_level, log_file=log_file) worker.work()
def pyres_scheduler(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) #parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port",dest="port",type="int", default=6379) parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.') parser.add_option('-f', dest='logfile', help='If present, a logfile will be used.') parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') (options,args) = parser.parse_args() log_level = getattr(logging, options.log_level.upper(),'INFO') #logging.basicConfig(level=log_level, format="%(module)s: %(asctime)s: %(levelname)s: %(message)s") setup_logging(log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) server = '%s:%s' % (options.host, options.port) Scheduler.run(server)
def handle(self, queue_list, **options): queues = queue_list.split(',') log_level = getattr(logging, options['log_level'].upper(), 'INFO') setup_logging(procname="pyres_worker", log_level=log_level, filename=None) setup_pidfile(settings.PYRES_WORKER_PIDFILE) interval = settings.PYRES_WORKER_INTERVAL worker = Worker( queues=queues, server=settings.PYRES_HOST, password=settings.PYRES_PASSWORD, timeout=settings.PYRES_WORKER_TIMEOUT ) if interval is not None: worker.work(interval) else: worker.work()
def handle(self, queue_list, **options): queues = queue_list.split(',') log_level = getattr(logging, options['log_level'].upper(), 'INFO') setup_logging(procname="pyres_worker", log_level=log_level, filename=None) setup_pidfile(settings.PYRES_WORKER_PIDFILE) interval = settings.PYRES_WORKER_INTERVAL worker = Worker(queues=queues, server=settings.PYRES_HOST, password=settings.PYRES_PASSWORD, timeout=settings.PYRES_WORKER_TIMEOUT) if interval is not None: worker.work(interval) else: worker.work()
def handle(self, queue_list, **options): queues = queue_list.split(",") log_level = getattr(logging, options["log_level"].upper(), "INFO") pool_size = options.get("pool_size") log_file = options.get("logfile") if pool_size == 0: pool_size = settings.PYRES_MANAGER_POOL_SIZE setup_pidfile(settings.PYRES_WORKER_PIDFILE) worker = Khan( pool_size=pool_size, queues=queues, server=settings.PYRES_HOST, password=settings.PYRES_PASSWORD, logging_level=log_level, log_file=log_file, ) worker.work()
def pyres_manager(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) #parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port", dest="port",type="int", default=6379) parser.add_option("--password", dest="password", default=None) parser.add_option("-i", '--interval', dest='manager_interval', default=None, help='the default time interval to sleep between runs - manager') parser.add_option("--minions_interval", dest='minions_interval', default=None, help='the default time interval to sleep between runs - minions') parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.') parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.") parser.add_option("-j", "--process_max_jobs", dest="max_jobs", type=int, default=0, help='how many jobs should be processed on worker run.') parser.add_option('-f', dest='logfile', help='If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.') parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') parser.add_option("--concat_minions_logs", action="store_true", dest="concat_minions_logs", help='Concat all minions logs on same file.') (options,args) = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') #logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s") concat_minions_logs = options.concat_minions_logs setup_pidfile(options.pidfile) manager_interval = options.manager_interval if manager_interval is not None: manager_interval = float(manager_interval) minions_interval = options.minions_interval if minions_interval is not None: minions_interval = float(minions_interval) queues = args[0].split(',') server = '%s:%s' % (options.host,options.port) password = options.password Khan.run(pool_size=options.pool_size, queues=queues, server=server, password=password, interval=manager_interval, logging_level=log_level, log_file=options.logfile, minions_interval=minions_interval, concat_minions_logs=concat_minions_logs, max_jobs=options.max_jobs)
def pyres_worker(): """Worker CLI, lightly modified from pyres. """ usage = 'usage: %prog [options] arg1' parser = OptionParser(usage=usage) parser.add_option('--host', dest='host', default=settings['host']) parser.add_option('--port', dest='port', type='int', default=settings['port']) parser.add_option('--password', dest='password', default=settings['password']) parser.add_option('-i', '--interval', dest='interval', default=None) parser.add_option('-l', '--log-level', dest='log_level', default='info') parser.add_option('-f', dest='logfile') parser.add_option('-p', dest='pidfile') parser.add_option('-t', '--timeout', dest='timeout') options, args = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') setup_logging(procname="pyres_worker", log_level=log_level, filename=options.logfile) setup_pidfile(options.pidfile) interval = options.interval if interval is not None: interval = int(interval) timeout = options.timeout and int(options.timeout) queues = args[0].split(',') server = '{0}:{1}'.format(options.host, options.port) Worker.run(queues, server, options.password, interval, timeout=timeout)
def pyres_manager(): usage = "usage: %prog [options] arg1" parser = OptionParser(usage=usage) #parser.add_option("-q", dest="queue_list") parser.add_option("--host", dest="host", default="localhost") parser.add_option("--port", dest="port", type="int", default=6379) parser.add_option("--password", dest="password", default=None) parser.add_option( "-i", '--interval', dest='manager_interval', default=None, help='the default time interval to sleep between runs - manager') parser.add_option( "--minions_interval", dest='minions_interval', default=None, help='the default time interval to sleep between runs - minions') parser.add_option( '-l', '--log-level', dest='log_level', default='info', help= 'log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.' ) parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.") parser.add_option("-j", "--process_max_jobs", dest="max_jobs", type=int, default=0, help='how many jobs should be processed on worker run.') parser.add_option( '-f', dest='logfile', help= 'If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.' ) parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.') parser.add_option("--concat_minions_logs", action="store_true", dest="concat_minions_logs", help='Concat all minions logs on same file.') (options, args) = parser.parse_args() if len(args) != 1: parser.print_help() parser.error("Argument must be a comma seperated list of queues") log_level = getattr(logging, options.log_level.upper(), 'INFO') #logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s") concat_minions_logs = options.concat_minions_logs setup_pidfile(options.pidfile) manager_interval = options.manager_interval if manager_interval is not None: manager_interval = float(manager_interval) minions_interval = options.minions_interval if minions_interval is not None: minions_interval = float(minions_interval) queues = args[0].split(',') server = '%s:%s' % (options.host, options.port) password = options.password Khan.run(pool_size=options.pool_size, queues=queues, server=server, password=password, interval=manager_interval, logging_level=log_level, log_file=options.logfile, minions_interval=minions_interval, concat_minions_logs=concat_minions_logs, max_jobs=options.max_jobs)