def start_servers():
if len(_PROCS) != 0:
return _PROCS
_PROCS.append(
start_process('loads.transport.broker', '--db', 'python',
'--heartbeat', 'tcp://0.0.0.0:9876'))
for x in range(10):
_PROCS.append(start_process('loads.transport.agent'))
_PROCS.append(start_process('loads.examples.echo_server'))
# wait for the echo server to be started
tries = 0
while True:
try:
requests.get('http://0.0.0.0:9000')
break
except requests.ConnectionError:
time.sleep(.2)
tries += 1
if tries > 20:
raise
# wait for the broker to be up with 3 slaves.
client = Pool()
while len(client.list()) != 10:
time.sleep(.1)
# control that the broker is responsive
client.ping()
for wid in client.list():
status = client.status(wid)['status']
assert status == {}, status
client.close()
if verify_broker() is None:
raise ValueError('Broker seem down')
return _PROCS
def start_servers():
if len(_PROCS) != 0:
return _PROCS
_PROCS.append(start_process('loads.transport.broker', '--db', 'python',
'--heartbeat', 'tcp://0.0.0.0:9876'))
for x in range(10):
_PROCS.append(start_process('loads.transport.agent'))
_PROCS.append(start_process('loads.examples.echo_server'))
# wait for the echo server to be started
tries = 0
while True:
try:
requests.get('http://0.0.0.0:9000')
break
except requests.ConnectionError:
time.sleep(.2)
tries += 1
if tries > 20:
raise
# wait for the broker to be up with 3 slaves.
client = Pool()
while len(client.list()) != 10:
time.sleep(.1)
# control that the broker is responsive
client.ping()
for wid in client.list():
status = client.status(wid)['status']
assert status == {}, status
client.close()
if verify_broker() is None:
raise ValueError('Broker seem down')
return _PROCS
def start_servers():
procs = []
procs.append(start_process("loads.transport.broker"))
for x in range(3):
procs.append(start_process("loads.transport.agent"))
procs.append(start_process("loads.examples.echo_server"))
# wait for the echo server to be started
tries = 0
while True:
try:
requests.get("http://0.0.0.0:9000")
break
except requests.ConnectionError:
time.sleep(0.2)
tries += 1
if tries > 20:
raise
# wait for the broker to be up with 3 slaves.
client = Pool()
while len(client.list()) != 3:
time.sleep(0.1)
# control that the broker is responsive
client.ping()
for wid in client.list():
status = client.status(wid)["status"]
assert status == {}, status
client.close()
if verify_broker() is None:
raise ValueError("Broker seem down")
return procs
def __init__(
self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db="python",
dboptions=None,
web_root=None,
):
# before doing anything, we verify if a broker is already up and
# running
logger.debug("Verifying if there is a running broker")
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug("Ooops, we have a running broker on that socket")
raise DuplicateBrokerError(pid)
self.endpoints = {
"frontend": frontend,
"backend": backend,
"register": register,
"receiver": receiver,
"publisher": publisher,
}
if heartbeat is not None:
self.endpoints["heartbeat"] = heartbeat
logger.debug("Initializing the broker.")
for endpoint in self.endpoints.values():
if endpoint.startswith("ipc"):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = "broker-" + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context, onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self, self.loop, db=db, dboptions=dboptions, agent_timeout=agent_timeout)
self.web_root = web_root
def main(args=sys.argv):
parser = argparse.ArgumentParser(description="Loads broker.")
parser.add_argument("--frontend", dest="frontend", default=DEFAULT_FRONTEND, help="ZMQ socket to receive jobs.")
parser.add_argument("--backend", dest="backend", default=DEFAULT_BACKEND, help="ZMQ socket for agents.")
parser.add_argument("--heartbeat", dest="heartbeat", default=None, help="ZMQ socket for the heartbeat.")
parser.add_argument("--register", dest="register", default=DEFAULT_REG, help="ZMQ socket for the registration.")
parser.add_argument(
"--receiver",
dest="receiver",
default=DEFAULT_BROKER_RECEIVER,
help="ZMQ socket to receive events from the runners",
)
parser.add_argument(
"--publisher", dest="publisher", default=DEFAULT_PUBLISHER, help="ZMQ socket for the registration."
)
parser.add_argument("--io-threads", type=int, default=DEFAULT_IOTHREADS, help="Number of I/O threads")
parser.add_argument("--debug", action="store_true", default=False, help="Debug mode")
parser.add_argument(
"--check",
action="store_true",
default=False,
help=("Use this option to check if there's a running " " broker. Returns the PID if a broker is up."),
)
parser.add_argument("--logfile", dest="logfile", default="stdout", help="File to log in to.")
parser.add_argument("--db", dest="db", default="python", help="Database backend.")
parser.add_argument("--web-root", help="Root url of the web dashboard.", type=str, default=None)
# add db args
for backend, options in get_backends():
for option, default, help, type_ in options:
option = "db_%s_%s" % (backend, option)
kargs = {"dest": option, "default": default}
if type_ is bool:
kargs["action"] = "store_true"
else:
kargs["type"] = type_
option = option.replace("_", "-")
parser.add_argument("--%s" % option, **kargs)
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
if args.check:
pid = verify_broker(args.frontend)
if pid is None:
logger.info("There seem to be no broker on this endpoint")
else:
logger.info("A broker is running. PID: %s" % pid)
return 0
# grabbing the db options
dboptions = {}
prefix = "db_%s_" % args.db
for key, value in args._get_kwargs():
if not key.startswith(prefix):
continue
dboptions[key[len(prefix) :]] = value
logger.info("Starting the broker")
try:
broker = Broker(
frontend=args.frontend,
backend=args.backend,
heartbeat=args.heartbeat,
register=args.register,
receiver=args.receiver,
publisher=args.publisher,
io_threads=args.io_threads,
db=args.db,
dboptions=dboptions,
web_root=args.web_root,
)
except DuplicateBrokerError, e:
logger.info("There is already a broker running on PID %s" % e)
logger.info("Exiting")
return 1
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Loads broker.')
parser.add_argument('--frontend', dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for agents.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=None,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--register', dest='register',
default=DEFAULT_REG,
help="ZMQ socket for the registration.")
parser.add_argument('--receiver', dest='receiver',
default=DEFAULT_BROKER_RECEIVER,
help="ZMQ socket to receive events from the runners")
parser.add_argument('--publisher', dest='publisher',
default=DEFAULT_PUBLISHER,
help="ZMQ socket for the registration.")
parser.add_argument('--io-threads', type=int,
default=DEFAULT_IOTHREADS,
help="Number of I/O threads")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--check', action='store_true', default=False,
help=("Use this option to check if there's a running "
" broker. Returns the PID if a broker is up."))
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to.")
parser.add_argument('--db', dest='db', default='python',
help="Database backend.")
# add db args
for backend, options in get_backends():
for option, default, help, type_ in options:
option = 'db_%s_%s' % (backend, option)
kargs = {'dest': option, 'default': default}
if type_ is bool:
kargs['action'] = 'store_true'
else:
kargs['type'] = type_
option = option.replace('_', '-')
parser.add_argument('--%s' % option, **kargs)
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
if args.check:
pid = verify_broker(args.frontend)
if pid is None:
logger.info('There seem to be no broker on this endpoint')
else:
logger.info('A broker is running. PID: %s' % pid)
return 0
# grabbing the db options
dboptions = {}
prefix = 'db_%s_' % args.db
for key, value in args._get_kwargs():
if not key.startswith(prefix):
continue
dboptions[key[len(prefix):]] = value
logger.info('Starting the broker')
try:
broker = Broker(frontend=args.frontend, backend=args.backend,
heartbeat=args.heartbeat, register=args.register,
receiver=args.receiver, publisher=args.publisher,
io_threads=args.io_threads, db=args.db,
dboptions=dboptions)
except DuplicateBrokerError, e:
logger.info('There is already a broker running on PID %s' % e)
logger.info('Exiting')
return 1
def __init__(self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db='python',
dboptions=None,
web_root=None):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
self.endpoints = {
'frontend': frontend,
'backend': backend,
'register': register,
'receiver': receiver,
'publisher': publisher
}
if heartbeat is not None:
self.endpoints['heartbeat'] = heartbeat
logger.debug('Initializing the broker.')
for endpoint in self.endpoints.values():
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat,
io_loop=self.loop,
ctx=self.context,
onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self,
self.loop,
db=db,
dboptions=dboptions,
agent_timeout=agent_timeout)
self.web_root = web_root
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Loads broker.')
parser.add_argument('--frontend',
dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend',
dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for agents.")
parser.add_argument('--heartbeat',
dest='heartbeat',
default=None,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--register',
dest='register',
default=DEFAULT_REG,
help="ZMQ socket for the registration.")
parser.add_argument('--receiver',
dest='receiver',
default=DEFAULT_BROKER_RECEIVER,
help="ZMQ socket to receive events from the runners")
parser.add_argument('--publisher',
dest='publisher',
default=DEFAULT_PUBLISHER,
help="ZMQ socket for the registration.")
parser.add_argument('--io-threads',
type=int,
default=DEFAULT_IOTHREADS,
help="Number of I/O threads")
parser.add_argument('--debug',
action='store_true',
default=False,
help="Debug mode")
parser.add_argument('--check',
action='store_true',
default=False,
help=("Use this option to check if there's a running "
" broker. Returns the PID if a broker is up."))
parser.add_argument('--logfile',
dest='logfile',
default='stdout',
help="File to log in to.")
parser.add_argument('--db',
dest='db',
default='python',
help="Database backend.")
parser.add_argument('--web-root',
help='Root url of the web dashboard.',
type=str,
default=None)
# add db args
for backend, options in get_backends():
for option, default, help, type_ in options:
option = 'db_%s_%s' % (backend, option)
kargs = {'dest': option, 'default': default}
if type_ is bool:
kargs['action'] = 'store_true'
else:
kargs['type'] = type_
option = option.replace('_', '-')
parser.add_argument('--%s' % option, **kargs)
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
if args.check:
pid = verify_broker(args.frontend)
if pid is None:
logger.info('There seem to be no broker on this endpoint')
else:
logger.info('A broker is running. PID: %s' % pid)
return 0
# grabbing the db options
dboptions = {}
prefix = 'db_%s_' % args.db
for key, value in args._get_kwargs():
if not key.startswith(prefix):
continue
dboptions[key[len(prefix):]] = value
logger.info('Starting the broker')
try:
broker = Broker(frontend=args.frontend,
backend=args.backend,
heartbeat=args.heartbeat,
register=args.register,
receiver=args.receiver,
publisher=args.publisher,
io_threads=args.io_threads,
db=args.db,
dboptions=dboptions,
web_root=args.web_root)
except DuplicateBrokerError, e:
logger.info('There is already a broker running on PID %s' % e)
logger.info('Exiting')
return 1
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=DEFAULT_HEARTBEAT, register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
worker_timeout=DEFAULT_TIMEOUT_MOVF,
receiver=DEFAULT_RECEIVER, publisher=DEFAULT_PUBLISHER,
dbdir=DEFAULT_DBDIR):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
logger.debug('Initializing the broker.')
for endpoint in (frontend, backend, heartbeat):
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_rcv)
# heartbeat
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context)
# status
self.started = False
self.poll_timeout = None
# workers registration and timers
self._workers = []
self._worker_times = {}
self.worker_timeout = worker_timeout
self._runs = {}
# local DB
self._db = BrokerDB(self.loop, dbdir)
def main(args=sys.argv):
parser = argparse.ArgumentParser(description='Loads broker.')
parser.add_argument('--frontend', dest='frontend',
default=DEFAULT_FRONTEND,
help="ZMQ socket to receive jobs.")
parser.add_argument('--backend', dest='backend',
default=DEFAULT_BACKEND,
help="ZMQ socket for workers.")
parser.add_argument('--heartbeat', dest='heartbeat',
default=DEFAULT_HEARTBEAT,
help="ZMQ socket for the heartbeat.")
parser.add_argument('--register', dest='register',
default=DEFAULT_REG,
help="ZMQ socket for the registration.")
parser.add_argument('--receiver', dest='receiver',
default=DEFAULT_RECEIVER,
help="ZMQ socket for the registration.")
parser.add_argument('--publisher', dest='publisher',
default=DEFAULT_PUBLISHER,
help="ZMQ socket for the registration.")
parser.add_argument('--io-threads', type=int,
default=DEFAULT_IOTHREADS,
help="Number of I/O threads")
parser.add_argument('--debug', action='store_true', default=False,
help="Debug mode")
parser.add_argument('--check', action='store_true', default=False,
help=("Use this option to check if there's a running "
" broker. Returns the PID if a broker is up."))
parser.add_argument('--purge-ghosts', action='store_true', default=False,
help="Use this option to purge ghost brokers.")
parser.add_argument('--logfile', dest='logfile', default='stdout',
help="File to log in to.")
parser.add_argument('--db-directory', dest='dbdir', default=DEFAULT_DBDIR,
help="Database Directory.")
args = parser.parse_args()
set_logger(args.debug, logfile=args.logfile)
if args.purge_ghosts:
broker_pids, ghosts = kill_ghost_brokers(args.frontend)
if broker_pids is None:
logger.info('No running broker.')
else:
logger.info('The active broker runs at PID: %s' % broker_pids)
if len(ghosts) == 0:
logger.info('No ghosts where killed.')
else:
logger.info('Ghost(s) killed: %s'
% ', '.join([str(g) for g in ghosts]))
return 0
if args.check:
pid = verify_broker(args.frontend)
if pid is None:
logger.info('There seem to be no broker on this endpoint')
else:
logger.info('A broker is running. PID: %s' % pid)
return 0
logger.info('Starting the broker')
try:
broker = Broker(frontend=args.frontend, backend=args.backend,
heartbeat=args.heartbeat, register=args.register,
receiver=args.receiver, publisher=args.publisher,
io_threads=args.io_threads, dbdir=args.dbdir)
except DuplicateBrokerError, e:
logger.info('There is already a broker running on PID %s' % e)
logger.info('Exiting')
return 1
