if __name__ == "__main__":
c = Config()
redis_section = "redis"
worker_section = "worker_broker"
worker_number = sys.argv[1]
if len(sys.argv) == 3:
if "redis_%s" % sys.argv[2] in c.sections():
redis_section = "redis_%s" % sys.argv[2]
rq = RedisQueue(c.get(worker_section, "listento"), "broker_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
if c.has_option(worker_section, "fanout_status_queue"):
# keep a queue of messages to deliver for a given push'd item
# better resumeability at the cost of more redis operations
topushq = RedisQueue(c.get(worker_section, "fanout_status_queue"), "fanout_broker_%s" % worker_number,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
fanout_queues = [x.strip() for x in c.get(worker_section, "fanout").split(",") if x]
if c.has_option(worker_section, "idletime"):
try:
idletime = float(c.get(worker_section, "idletime"))
except ValueError:
idletime = 10
if not os.path.exists("workers_available"):
os.mkdir("workers_available")
c = Config()
base_superv_conf = "supervisord.conf.base"
if len(sys.argv) == 2:
# use a different base supervisor file
base_superv_conf = sys.argv[1]
supervisord_config = Config(base_superv_conf)
if 'supervisor' in c.sections():
supervisord_config.add_section('inet_http_server')
params = {'username':'******',
'password':'******',
'port':'127.0.0.1:9001'}
for key in params:
if c.has_option('supervisor', key):
supervisord_config.set('inet_http_server', key, c.get('supervisor', key))
else:
supervisord_config.set('inet_http_server', key, params['key'])
with open("supervisord.conf", "w") as cfgfile:
supervisord_config.write(cfgfile)
# process_* for simple, single use processes that don't require additional configuration
# aside from the 'command' instruction
# eg command = ../redis/redis-server ../redis/redis.conf
for worker in [x for x in c.sections() if x.startswith("process_")]:
# Worker defaults:
params = {'autorestart':'true',
'numprocs':'1',
'process_name':'%s_%%(process_num)s' % worker,
db=c.get(redis_section, "db"),
host=c.get(redis_section, "host"),
port=c.get(redis_section, "port")
)
try:
plugin_name = c.get(worker_section, "repository_plugin")
plugin_module = __import__(plugin_name)
components = plugin_name.split('.')
for comp in components[1:]:
plugin_module = getattr(plugin_module, comp)
except ImportError, e:
logger.error("Coundn't import module: '%s' - %s" % (c.get(worker_section, "repository_plugin"), e))
sys.exit(2)
if c.has_option(worker_section, 'pauseonfail'):
try:
delay_on_fail = int(c.get(worker_section, 'pauseonfail'))
except:
delay_on_fail = 300
if c.has_option(worker_section, 'ratelimit'):
try:
delay = float(c.get(worker_section, 'ratelimit'))
except:
delay = 1
logger.debug("Delay on fail set to: %s Ratelimit set to: %s" % (delay_on_fail, delay))
while(True):
line = rq.pop()
if line:
