def pyres_manager(queue, pool_size, log_filename=None):
#setup_logging(log_level=logging.INFO, filename=log_filename)
Khan.run(pool_size=pool_size,
queues=[queue],
server='%s:%d' % (config.REDIS_HOST, config.REDIS_PORT),
logging_level=logging.INFO,
log_file=log_filename)
def pyres_manager():
usage = "usage: %prog [options] arg1"
parser = OptionParser(usage=usage)
#parser.add_option("-q", dest="queue_list")
parser.add_option("--host", dest="host", default="localhost")
parser.add_option("--port",dest="port",type="int", default=6379)
parser.add_option("-i", '--interval', dest='interval', default=None, help='the default time interval to sleep between runs')
parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.')
parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.")
parser.add_option('-f', dest='logfile', help='If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.')
parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.')
(options,args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
parser.error("Argument must be a comma seperated list of queues")
log_level = getattr(logging, options.log_level.upper(), 'INFO')
#logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s")
setup_pidfile(options.pidfile)
interval = options.interval
if interval is not None:
interval = float(interval)
queues = args[0].split(',')
server = '%s:%s' % (options.host,options.port)
Khan.run(pool_size=options.pool_size, queues=queues, server=server, logging_level=log_level, log_file=options.logfile)
def pyres_manager():
usage = "usage: %prog [options] arg1"
parser = OptionParser(usage=usage)
#parser.add_option("-q", dest="queue_list")
parser.add_option("--host", dest="host", default="localhost")
parser.add_option("--port",dest="port",type="int", default=6379)
parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.')
parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.")
parser.add_option('-f', dest='logfile', help='If present, a logfile will be used.')
parser.add_option("-n", "--nonblocking-pop", dest="blocking_pop", action="store_false", default=True, help="If absent, Pyres will use the Redis blocking pop (BLPOP) to obtain jobs from the queue(s). If present, Redis will use a non-blocking pop (LPOP) and will sleep for up to 8 seconds if no jobs are available.")
parser.add_option(
"-r", "--reinit-gevent", dest="reinit_gevent",
action="store_true", default=False,
help="If present, reinitialize gevent in minions after fork." )
parser.add_option("-m", "--monkey-patch", dest="monkey_patch",
action="store_true", default=False,
help="If present, also turn on gevent monkey patch.")
(options,args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
parser.error("Argument must be a comma seperated list of queues")
log_level = getattr(logging, options.log_level.upper(), 'INFO')
#logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s")
queues = args[0].split(',')
server = '%s:%s' % (options.host,options.port)
Khan.run(pool_size=options.pool_size, queues=queues, server=server,
logging_level=log_level, log_file=options.logfile,
blocking_pop=options.blocking_pop,
reinit_gevent=options.reinit_gevent,
monkey_patch=options.monkey_patch)
def pyres_manager(queue, pool_size, log_filename=None):
#setup_logging(log_level=logging.INFO, filename=log_filename)
Khan.run(pool_size=pool_size,
queues=[queue],
server='%s:%d' % (config.REDIS_HOST, config.REDIS_PORT),
logging_level=logging.INFO,
log_file=log_filename)
def pyres_manager():
usage = "usage: %prog [options] arg1"
parser = OptionParser(usage=usage)
#parser.add_option("-q", dest="queue_list")
parser.add_option("--host", dest="host", default="localhost")
parser.add_option("--port", dest="port", type="int", default=6379)
parser.add_option("-i",
'--interval',
dest='interval',
default=None,
help='the default time interval to sleep between runs')
parser.add_option(
'-l',
'--log-level',
dest='log_level',
default='info',
help=
'log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.'
)
parser.add_option("--pool",
type="int",
dest="pool_size",
default=1,
help="Number of minions to spawn under the manager.")
parser.add_option(
'-f',
dest='logfile',
help=
'If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.'
)
parser.add_option('-p',
dest='pidfile',
help='If present, a pidfile will be used.')
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
parser.error("Argument must be a comma seperated list of queues")
log_level = getattr(logging, options.log_level.upper(), 'INFO')
#logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s")
setup_pidfile(options.pidfile)
interval = options.interval
if interval is not None:
interval = float(interval)
queues = args[0].split(',')
server = '%s:%s' % (options.host, options.port)
Khan.run(pool_size=options.pool_size,
queues=queues,
server=server,
logging_level=log_level,
log_file=options.logfile)
def handle(self, queue_list, **options):
queues = queue_list.split(',')
log_level = getattr(logging, options['log_level'].upper(), 'INFO')
pool_size = options.get('pool_size')
log_file = options.get('logfile')
if pool_size == 0:
pool_size = settings.PYRES_MANAGER_POOL_SIZE
setup_pidfile(settings.PYRES_WORKER_PIDFILE)
worker = Khan(pool_size=pool_size,
queues=queues,
server=settings.PYRES_HOST,
password=settings.PYRES_PASSWORD,
logging_level=log_level,
log_file=log_file)
worker.work()
def handle(self, queue_list, **options):
queues = queue_list.split(",")
log_level = getattr(logging, options["log_level"].upper(), "INFO")
pool_size = options.get("pool_size")
log_file = options.get("logfile")
if pool_size == 0:
pool_size = settings.PYRES_MANAGER_POOL_SIZE
setup_pidfile(settings.PYRES_WORKER_PIDFILE)
worker = Khan(
pool_size=pool_size,
queues=queues,
server=settings.PYRES_HOST,
password=settings.PYRES_PASSWORD,
logging_level=log_level,
log_file=log_file,
)
worker.work()
def pyres_manager():
usage = "usage: %prog [options] arg1"
parser = OptionParser(usage=usage)
#parser.add_option("-q", dest="queue_list")
parser.add_option("--host", dest="host", default="localhost")
parser.add_option("--port", dest="port",type="int", default=6379)
parser.add_option("--password", dest="password", default=None)
parser.add_option("-i", '--interval', dest='manager_interval', default=None, help='the default time interval to sleep between runs - manager')
parser.add_option("--minions_interval", dest='minions_interval', default=None, help='the default time interval to sleep between runs - minions')
parser.add_option('-l', '--log-level', dest='log_level', default='info', help='log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.')
parser.add_option("--pool", type="int", dest="pool_size", default=1, help="Number of minions to spawn under the manager.")
parser.add_option("-j", "--process_max_jobs", dest="max_jobs", type=int, default=0, help='how many jobs should be processed on worker run.')
parser.add_option('-f', dest='logfile', help='If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.')
parser.add_option('-p', dest='pidfile', help='If present, a pidfile will be used.')
parser.add_option("--concat_minions_logs", action="store_true", dest="concat_minions_logs", help='Concat all minions logs on same file.')
(options,args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
parser.error("Argument must be a comma seperated list of queues")
log_level = getattr(logging, options.log_level.upper(), 'INFO')
#logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s")
concat_minions_logs = options.concat_minions_logs
setup_pidfile(options.pidfile)
manager_interval = options.manager_interval
if manager_interval is not None:
manager_interval = float(manager_interval)
minions_interval = options.minions_interval
if minions_interval is not None:
minions_interval = float(minions_interval)
queues = args[0].split(',')
server = '%s:%s' % (options.host,options.port)
password = options.password
Khan.run(pool_size=options.pool_size, queues=queues, server=server, password=password, interval=manager_interval,
logging_level=log_level, log_file=options.logfile, minions_interval=minions_interval,
concat_minions_logs=concat_minions_logs, max_jobs=options.max_jobs)
def pyres_manager():
usage = "usage: %prog [options] arg1"
parser = OptionParser(usage=usage)
#parser.add_option("-q", dest="queue_list")
parser.add_option("--host", dest="host", default="localhost")
parser.add_option("--port", dest="port", type="int", default=6379)
parser.add_option("--password", dest="password", default=None)
parser.add_option(
"-i",
'--interval',
dest='manager_interval',
default=None,
help='the default time interval to sleep between runs - manager')
parser.add_option(
"--minions_interval",
dest='minions_interval',
default=None,
help='the default time interval to sleep between runs - minions')
parser.add_option(
'-l',
'--log-level',
dest='log_level',
default='info',
help=
'log level. Valid values are "debug", "info", "warning", "error", "critical", in decreasing order of verbosity. Defaults to "info" if parameter not specified.'
)
parser.add_option("--pool",
type="int",
dest="pool_size",
default=1,
help="Number of minions to spawn under the manager.")
parser.add_option("-j",
"--process_max_jobs",
dest="max_jobs",
type=int,
default=0,
help='how many jobs should be processed on worker run.')
parser.add_option(
'-f',
dest='logfile',
help=
'If present, a logfile will be used. "stderr", "stdout", and "syslog" are all special values.'
)
parser.add_option('-p',
dest='pidfile',
help='If present, a pidfile will be used.')
parser.add_option("--concat_minions_logs",
action="store_true",
dest="concat_minions_logs",
help='Concat all minions logs on same file.')
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
parser.error("Argument must be a comma seperated list of queues")
log_level = getattr(logging, options.log_level.upper(), 'INFO')
#logging.basicConfig(level=log_level, format="%(asctime)s: %(levelname)s: %(message)s")
concat_minions_logs = options.concat_minions_logs
setup_pidfile(options.pidfile)
manager_interval = options.manager_interval
if manager_interval is not None:
manager_interval = float(manager_interval)
minions_interval = options.minions_interval
if minions_interval is not None:
minions_interval = float(minions_interval)
queues = args[0].split(',')
server = '%s:%s' % (options.host, options.port)
password = options.password
Khan.run(pool_size=options.pool_size,
queues=queues,
server=server,
password=password,
interval=manager_interval,
logging_level=log_level,
log_file=options.logfile,
minions_interval=minions_interval,
concat_minions_logs=concat_minions_logs,
max_jobs=options.max_jobs)
def workers():
workers = Khan(pool_size=PYRES, queues=['records'])
workers.work()