class Broker(object):
"""Class that route jobs to agents.
Options:
- **frontend**: the ZMQ socket to receive jobs.
- **backend**: the ZMQ socket to communicate with agents.
- **heartbeat**: the ZMQ socket to receive heartbeat requests.
- **register** : the ZMQ socket to register agents.
- **receiver**: the ZMQ socket that receives data from agents.
- **publisher**: the ZMQ socket to publish agents data
"""
def __init__(
self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db="python",
dboptions=None,
web_root=None,
):
# before doing anything, we verify if a broker is already up and
# running
logger.debug("Verifying if there is a running broker")
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug("Ooops, we have a running broker on that socket")
raise DuplicateBrokerError(pid)
self.endpoints = {
"frontend": frontend,
"backend": backend,
"register": register,
"receiver": receiver,
"publisher": publisher,
}
if heartbeat is not None:
self.endpoints["heartbeat"] = heartbeat
logger.debug("Initializing the broker.")
for endpoint in self.endpoints.values():
if endpoint.startswith("ipc"):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = "broker-" + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat, io_loop=self.loop, ctx=self.context, onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self, self.loop, db=db, dboptions=dboptions, agent_timeout=agent_timeout)
self.web_root = web_root
def _handle_recv(self, msg):
# publishing all the data received from agents
self._publisher.send(msg[0])
data = json.loads(msg[0])
agent_id = str(data.get("agent_id"))
hostname = data.get("hostname", "?")
# telling the controller that the agent is alive
self.ctrl.register_agent({"pid": agent_id, "hostname": hostname})
# saving the data locally
self.ctrl.save_data(agent_id, data)
def _deregister(self):
self.ctrl.unregister_agents("asked by the heartbeat.")
def _handle_reg(self, msg):
if msg[0] == "REGISTER":
self.ctrl.register_agent(json.loads(msg[1]))
elif msg[0] == "UNREGISTER":
self.ctrl.unregister_agent(msg[1], "asked via UNREGISTER")
def send_json(self, target, data):
assert isinstance(target, basestring), target
msg = [target, "", json.dumps(data)]
try:
self._frontstream.send_multipart(msg)
except ValueError:
logger.error("Could not dump %s" % str(data))
raise
def _handle_recv_front(self, msg, tentative=0):
"""front => back
All commands starting with CTRL_ are sent to the controller.
"""
target = msg[0]
try:
data = json.loads(msg[-1])
except ValueError:
exc = "Invalid JSON received."
logger.exception(exc)
self.send_json(target, {"error": exc})
return
cmd = data["command"]
# a command handled by the controller
if cmd.startswith("CTRL_"):
cmd = cmd[len("CTRL_") :]
logger.debug("calling %s" % cmd)
try:
res = self.ctrl.run_command(cmd, msg, data)
except Exception, e:
logger.debug("Failed")
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.send_json(target, {"error": exc})
else:
# sending back a synchronous result if needed.
if res is not None:
logger.debug("sync success %s" % str(res))
self.send_json(target, res)
else:
logger.debug("async success")
# misc commands
elif cmd == "PING":
res = {"result": {"pid": os.getpid(), "endpoints": self.endpoints, "agents": self.ctrl.agents}}
self.send_json(target, res)
class Broker(object):
"""Class that route jobs to agents.
Options:
- **frontend**: the ZMQ socket to receive jobs.
- **backend**: the ZMQ socket to communicate with agents.
- **heartbeat**: the ZMQ socket to receive heartbeat requests.
- **register** : the ZMQ socket to register agents.
- **receiver**: the ZMQ socket that receives data from agents.
- **publisher**: the ZMQ socket to publish agents data
"""
def __init__(self, frontend=DEFAULT_FRONTEND, backend=DEFAULT_BACKEND,
heartbeat=None, register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_TIMEOUT_MOVF,
receiver=DEFAULT_BROKER_RECEIVER, publisher=DEFAULT_PUBLISHER,
db='python', dboptions=None):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
self.endpoints = {'frontend': frontend,
'backend': backend,
'register': register,
'receiver': receiver,
'publisher': publisher}
if heartbeat is not None:
self.endpoints['heartbeat'] = heartbeat
logger.debug('Initializing the broker.')
for endpoint in self.endpoints.values():
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat, io_loop=self.loop,
ctx=self.context,
onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self, self.loop, db=db,
dboptions=dboptions,
agent_timeout=agent_timeout)
def _handle_recv(self, msg):
# publishing all the data received from agents
self._publisher.send(msg[0])
# saving the data locally
data = json.loads(msg[0])
agent_id = str(data.get('agent_id'))
self.ctrl.save_data(agent_id, data)
def _deregister(self):
self.ctrl.unregister_agents()
def _handle_reg(self, msg):
if msg[0] == 'REGISTER':
self.ctrl.register_agent(msg[1])
elif msg[0] == 'UNREGISTER':
self.ctrl.unregister_agent(msg[1])
def send_json(self, target, data):
try:
self._frontstream.send_multipart(target + [json.dumps(data)])
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def _handle_recv_front(self, msg, tentative=0):
"""front => back
All commands starting with CTRL_ are sent to the controller.
"""
target = msg[:-1]
try:
data = json.loads(msg[2])
except ValueError:
exc = 'Invalid JSON received.'
logger.exception(exc)
self.send_json(target, {'error': exc})
return
cmd = data['command']
# a command handled by the controller
if cmd.startswith('CTRL_'):
cmd = cmd[len('CTRL_'):]
logger.debug('calling %s' % cmd)
try:
res = self.ctrl.run_command(cmd, msg, data)
except Exception, e:
logger.debug('Failed')
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.send_json(target, {'error': exc})
else:
# sending back a synchronous result if needed.
if res is not None:
logger.debug('sync success %s' % str(res))
self.send_json(target, res)
else:
logger.debug('async success')
# misc commands
elif cmd == 'PING':
res = {'result': {'pid': os.getpid(),
'endpoints': self.endpoints,
'agents': self.ctrl.agents}}
self.send_json(target, res)
class TestBrokerController(unittest2.TestCase):
def setUp(self):
self.dbdir = tempfile.mkdtemp()
loop = ioloop.IOLoop()
self.broker = FakeBroker()
dboptions = {'directory': self.dbdir}
self.ctrl = BrokerController(self.broker, loop, dboptions=dboptions)
self.old_exists = psutil.pid_exists
psutil.pid_exists = lambda pid: True
def tearDown(self):
psutil.pid_exists = self.old_exists
Stream.msgs[:] = []
shutil.rmtree(self.dbdir)
def test_registration(self):
self.ctrl.register_agent({'pid': '1', 'agent_id': '1'})
self.assertTrue('1' in self.ctrl.agents)
# make the agent busy before we unregister it
self.ctrl.send_to_agent('1', ['something'])
self.ctrl.reserve_agents(1, 'run')
self.ctrl.unregister_agent('1')
self.assertFalse('1' in self.ctrl.agents)
def test_reserve_agents(self):
self.ctrl.register_agent({'pid': '1', 'agent_id': '1'})
self.ctrl.register_agent({'pid': '2', 'agent_id': '2'})
self.assertRaises(NotEnoughWorkersError, self.ctrl.reserve_agents, 10,
'run')
agents = self.ctrl.reserve_agents(2, 'run')
agents.sort()
self.assertEqual(agents, ['1', '2'])
def test_run_and_stop(self):
self.ctrl.register_agent({'pid': '1', 'agent_id': '1'})
self.ctrl.register_agent({'pid': '2', 'agent_id': '2'})
self.ctrl.register_agent({'pid': '3', 'agent_id': '3'})
self.ctrl.reserve_agents(1, 'run')
self.ctrl.reserve_agents(2, 'run2')
runs = self.ctrl.list_runs(None, None).keys()
runs.sort()
self.assertEqual(['run', 'run2'], runs)
self.ctrl.stop_run(['somemsg'], {'run_id': 'run'})
# make sure the STOP cmd made it through
msgs = [msg for msg in Stream.msgs if '_STATUS' not in msg[-1]]
self.assertEqual(msgs[0][-1], '{"command":"STOP"}')
self.assertEqual(len(msgs), 1)
def test_db_access(self):
self.ctrl.register_agent({'agent_id': '1', 'agent_id': '1'})
self.ctrl.reserve_agents(1, 'run')
# metadata
data = {'some': 'data'}
self.ctrl.save_metadata('run', data)
self.assertEqual(self.ctrl.get_metadata(None, {'run_id': 'run'}), data)
# save data by agent
self.ctrl.save_data('1', data)
self.ctrl.flush_db()
# we get extra run_id key, set for us
self.assertEqual(data['run_id'], 'run')
back = self.ctrl.get_data(None, {'run_id': 'run'})
self.assertTrue(back[0]['some'], 'data')
back2 = self.ctrl.get_data(None, {'run_id': 'run'})
self.assertEqual(back, back2)
def test_compute_observers(self):
obs = ['irc', 'loads.observers.irc']
observers = _compute_observers(obs)
self.assertEqual(len(observers), 2)
self.assertRaises(ImportError, _compute_observers, ['blah'])
def test_run(self):
msg = ['somedata', '', 'target']
data = {'agents': 1, 'args': {}}
# not enough agents
self.ctrl.run(msg, data)
res = self.broker.msgs.values()[0]
self.assertEqual(res, [{'error': 'Not enough agents'}])
# one agent, we're good
self.ctrl._agents['agent1'] = {'pid': '1234'}
self.ctrl.run(msg, data)
runs = self.broker.msgs.values()[0][-1]
self.assertEqual(runs['result']['agents'], ['agent1'])
def test_run_command(self):
msg = ['somedata', '', 'target']
data = {'agents': 1, 'args': {}, 'agent_id': '1'}
self.ctrl.run_command('RUN', msg, data)
self.ctrl.run_command('AGENT_STATUS', msg, data)
runs = self.broker.msgs.values()[0][-1]
self.assertEqual(runs['result']['agents'], ['agent1'])
msg = {"command": "_STATUS", "args": {}, "agents": 1, "agent_id": "1"}
msg = msg.items()
msg.sort()
self.assertTrue(len(self.broker._backstream.msgs), 1)
self.assertTrue(len(self.broker._backstream.msgs[0]), 1)
got = self.broker._backstream.msgs[0][-1]
got = json.loads(got)
got = got.items()
got.sort()
self.assertEqual(msg, got)
def test_clean(self):
self.ctrl.agent_timeout = 0.1
self.ctrl._associate('run', ['1', '2'])
self.ctrl.clean()
self.assertTrue('1' in self.ctrl._agent_times)
self.assertTrue('2' in self.ctrl._agent_times)
time.sleep(.2)
self.ctrl.clean()
self.assertEqual(self.ctrl._agent_times, {})
self.ctrl.test_ended('run')
class TestBrokerController(unittest2.TestCase):
def setUp(self):
self.dbdir = tempfile.mkdtemp()
loop = ioloop.IOLoop()
self.broker = FakeBroker()
dboptions = {'directory': self.dbdir}
self.ctrl = BrokerController(self.broker, loop, dboptions=dboptions)
self.old_exists = psutil.pid_exists
psutil.pid_exists = lambda pid: True
def tearDown(self):
psutil.pid_exists = self.old_exists
Stream.msgs[:] = []
shutil.rmtree(self.dbdir)
def test_registration(self):
self.ctrl.register_agent('1')
self.assertTrue('1' in self.ctrl.agents)
# make the agent busy before we unregister it
self.ctrl.send_to_agent('1', ['something'])
self.ctrl.reserve_agents(1, 'run')
self.ctrl.unregister_agent('1')
self.assertFalse('1' in self.ctrl.agents)
def test_reserve_agents(self):
self.ctrl.register_agent('1')
self.ctrl.register_agent('2')
self.assertRaises(NotEnoughWorkersError, self.ctrl.reserve_agents,
10, 'run')
agents = self.ctrl.reserve_agents(2, 'run')
agents.sort()
self.assertEqual(agents, ['1', '2'])
def test_run_and_stop(self):
self.ctrl.register_agent('1')
self.ctrl.register_agent('2')
self.ctrl.register_agent('3')
self.ctrl.reserve_agents(1, 'run')
self.ctrl.reserve_agents(2, 'run2')
runs = self.ctrl.list_runs(None, None).keys()
runs.sort()
self.assertEqual(['run', 'run2'], runs)
self.ctrl.stop_run(['somemsg'], {'run_id': 'run'})
# make sure the STOP cmd made it through
msgs = [msg for msg in Stream.msgs if '_STATUS' not in msg[-1]]
self.assertEqual(msgs[0][-1], '{"command":"STOP"}')
self.assertEqual(len(msgs), 1)
def test_db_access(self):
self.ctrl.register_agent('1')
self.ctrl.reserve_agents(1, 'run')
# metadata
data = {'some': 'data'}
self.ctrl.save_metadata('run', data)
self.assertEqual(self.ctrl.get_metadata(None, {'run_id': 'run'}),
data)
# save data by agent
self.ctrl.save_data('1', data)
self.ctrl.flush_db()
# we get extra run_id key, set for us
self.assertEqual(data['run_id'], 'run')
back = self.ctrl.get_data(None, {'run_id': 'run'})
self.assertTrue(back[0]['some'], 'data')
back2 = self.ctrl.get_data(None, {'run_id': 'run'})
self.assertEqual(back, back2)
def test_compute_observers(self):
obs = ['irc', 'loads.observers.irc']
observers = _compute_observers(obs)
self.assertEqual(len(observers), 2)
self.assertRaises(ImportError, _compute_observers, ['blah'])
def test_run(self):
msg = ['somedata', '', 'target']
data = {'agents': 1, 'args': {}}
# not enough agents
self.ctrl.run(msg, data)
res = self.broker.msgs.values()[0]
self.assertEqual(res, [{'error': 'Not enough agents'}])
# one agent, we're good
self.ctrl._agents.append('agent1')
self.ctrl.run(msg, data)
runs = self.broker.msgs.values()[0][-1]
self.assertEqual(runs['result']['agents'], ['agent1'])
def test_run_command(self):
msg = ['somedata', '', 'target']
data = {'agents': 1, 'args': {}, 'agent_id': '1'}
self.ctrl.run_command('RUN', msg, data)
self.ctrl.run_command('AGENT_STATUS', msg, data)
runs = self.broker.msgs.values()[0][-1]
self.assertEqual(runs['result']['agents'], ['agent1'])
msg = {"command": "STATUS", "args": {}, "agents": 1, "agent_id": "1"}
msg = msg.items()
msg.sort()
self.assertTrue(len(self.broker._backstream.msgs), 1)
self.assertTrue(len(self.broker._backstream.msgs[0]), 1)
got = self.broker._backstream.msgs[0][3]
got = json.loads(got)
got = got.items()
got.sort()
self.assertEqual(msg, got)
def test_clean(self):
self.ctrl.agent_timeout = 0.1
self.ctrl._associate('run', ['1', '2'])
self.ctrl.clean()
self.assertTrue('1' in self.ctrl._agent_times)
self.assertTrue('2' in self.ctrl._agent_times)
time.sleep(.2)
self.ctrl.clean()
self.assertEqual(self.ctrl._agent_times, {})
self.ctrl.test_ended('run')
class Broker(object):
"""Class that route jobs to agents.
Options:
- **frontend**: the ZMQ socket to receive jobs.
- **backend**: the ZMQ socket to communicate with agents.
- **heartbeat**: the ZMQ socket to receive heartbeat requests.
- **register** : the ZMQ socket to register agents.
- **receiver**: the ZMQ socket that receives data from agents.
- **publisher**: the ZMQ socket to publish agents data
"""
def __init__(self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db='python',
dboptions=None,
web_root=None):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
self.endpoints = {
'frontend': frontend,
'backend': backend,
'register': register,
'receiver': receiver,
'publisher': publisher
}
if heartbeat is not None:
self.endpoints['heartbeat'] = heartbeat
logger.debug('Initializing the broker.')
for endpoint in self.endpoints.values():
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat,
io_loop=self.loop,
ctx=self.context,
onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self,
self.loop,
db=db,
dboptions=dboptions,
agent_timeout=agent_timeout)
self.web_root = web_root
def _handle_recv(self, msg):
# publishing all the data received from agents
self._publisher.send(msg[0])
data = json.loads(msg[0])
agent_id = str(data.get('agent_id'))
hostname = data.get('hostname', '?')
# telling the controller that the agent is alive
self.ctrl.register_agent({'pid': agent_id, 'hostname': hostname})
# saving the data locally
self.ctrl.save_data(agent_id, data)
def _deregister(self):
self.ctrl.unregister_agents('asked by the heartbeat.')
def _handle_reg(self, msg):
if msg[0] == 'REGISTER':
self.ctrl.register_agent(json.loads(msg[1]))
elif msg[0] == 'UNREGISTER':
self.ctrl.unregister_agent(msg[1], 'asked via UNREGISTER')
def send_json(self, target, data):
assert isinstance(target, basestring), target
msg = [target, '', json.dumps(data)]
try:
self._frontstream.send_multipart(msg)
except ValueError:
logger.error('Could not dump %s' % str(data))
raise
def _handle_recv_front(self, msg, tentative=0):
"""front => back
All commands starting with CTRL_ are sent to the controller.
"""
target = msg[0]
try:
data = json.loads(msg[-1])
except ValueError:
exc = 'Invalid JSON received.'
logger.exception(exc)
self.send_json(target, {'error': exc})
return
cmd = data['command']
# a command handled by the controller
if cmd.startswith('CTRL_'):
cmd = cmd[len('CTRL_'):]
logger.debug('calling %s' % cmd)
try:
res = self.ctrl.run_command(cmd, msg, data)
except Exception, e:
logger.debug('Failed')
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.send_json(target, {'error': exc})
else:
# sending back a synchronous result if needed.
if res is not None:
logger.debug('sync success %s' % str(res))
self.send_json(target, res)
else:
logger.debug('async success')
# misc commands
elif cmd == 'PING':
res = {
'result': {
'pid': os.getpid(),
'endpoints': self.endpoints,
'agents': self.ctrl.agents
}
}
self.send_json(target, res)