def __init__(self, args):
super(DistributedRunner, self).__init__(args)
self.ssh = args.get('ssh')
self.run_id = None
self._stopped_agents = 0
self._nb_agents = args.get('agents')
# socket where the results are published
self.context = zmq.Context()
self.sub = self.context.socket(zmq.SUB)
self.sub.setsockopt(zmq.SUBSCRIBE, '')
self.sub.set_hwm(8096 * 10)
self.sub.setsockopt(zmq.LINGER, -1)
self.zmq_publisher = None
self.zstream = None
# io loop
self.loop = ioloop.IOLoop()
self.zstream = zmqstream.ZMQStream(self.sub, self.loop)
self.zstream.on_recv(self._recv_result)
self.agents = []
self._client = None
self.refresh_rate = 100
def setUp(self):
self.dbdir = tempfile.mkdtemp()
loop = ioloop.IOLoop()
broker = FakeBroker()
dboptions = {'directory': self.dbdir}
self.ctrl = BrokerController(broker, loop, dboptions=dboptions)
self.old_exists = psutil.pid_exists
psutil.pid_exists = lambda pid: True
def setUpClass(cls):
if 'TRAVIS' in os.environ:
return
start_servers()
cls.client = Client()
cls.location = os.getcwd()
cls.loop = ioloop.IOLoop()
cls.dirs = []
cls.db = BrokerDB(cls.loop, db='python')
def test_lost(self):
beats = []
lost = []
loop = ioloop.IOLoop()
def _hbreg():
beats.append('o')
def _onregister():
beats.append('+')
def _onbeat():
beats.append('.')
def _onbeatlost():
lost.append('.')
hb = Heartbeat('ipc:///tmp/stetho.ipc',
interval=0.1,
io_loop=loop,
onregister=_hbreg)
stetho = Stethoscope('ipc:///tmp/stetho.ipc',
onbeat=_onbeat,
onbeatlost=_onbeatlost,
delay=0.1,
io_loop=loop,
onregister=_onregister,
warmup_delay=0)
# scenario
def start():
hb.start()
stetho.start()
def stop_hb():
hb.stop()
def stop_st():
stetho.stop()
loop.stop()
# that starts the heartbeat and the client
loop.add_callback(start)
# the hb stops after 500ms
loop.add_timeout(time.time() + .5, stop_hb)
# the st stops after 1 second, then the loop
loop.add_timeout(time.time() + 1., stop_st)
loop.start()
self.assertTrue(len(beats) > 0)
self.assertEqual(beats[:2], ['o', '+'])
self.assertTrue(len(lost) > 0)
def test_lost(self):
beats = []
lost = []
loop = ioloop.IOLoop()
def _hbreg():
beats.append('o')
def _onregister():
beats.append('+')
def _onbeat():
beats.append('.')
def _onbeatlost():
lost.append('.')
hb = Heartbeat('ipc:///tmp/stetho.ipc',
interval=0.1,
io_loop=loop,
onregister=_hbreg)
stetho = Stethoscope('ipc:///tmp/stetho.ipc',
onbeat=_onbeat,
onbeatlost=_onbeatlost,
delay=0.1,
io_loop=loop,
onregister=_onregister)
# scenario
def start():
hb.start()
stetho.start()
def stop_hb():
hb.stop()
def stop_st():
stetho.stop()
loop.stop()
loop.add_callback(start)
# the hb stops after 500ms
cb = ioloop.DelayedCallback(stop_hb, 500, io_loop=loop)
cb.start()
# the st stops after 1 second, then the loop
cb = ioloop.DelayedCallback(stop_st, 1500, io_loop=loop)
cb.start()
loop.start()
self.assertTrue(len(beats) > 0)
self.assertEqual(beats[:2], ['o', '+'])
self.assertTrue(len(lost) > 3)
def test_working(self):
loop = ioloop.IOLoop()
beats = []
lost = []
def onbeat():
beats.append('.')
def onbeatlost():
lost.append('.')
hb = Heartbeat('ipc:///tmp/stetho.ipc', interval=0.1, io_loop=loop)
stetho = Stethoscope('ipc:///tmp/stetho.ipc',
onbeat=onbeat,
onbeatlost=onbeatlost,
delay=1.,
retries=5.,
io_loop=loop)
def start():
hb.start()
def start_2():
stetho.start()
# it's ok to try to start it again
stetho.start()
# hb starts immediatly
loop.add_callback(start)
# stetho 0.2 seconds after
cb = ioloop.DelayedCallback(start_2, 200, io_loop=loop)
cb.start()
def stop():
hb.stop()
stetho.stop()
loop.stop()
# all stops after 1s
cb = ioloop.DelayedCallback(stop, 1000, io_loop=loop)
cb.start()
# let's go
loop.start()
self.assertEqual(len(lost), 0, len(lost))
self.assertTrue(len(beats) > 2, len(beats))
def setUp(self):
self.loop = ioloop.IOLoop()
self.tmp = tempfile.mkdtemp()
dboptions = {'directory': self.tmp}
self.db = BrokerDB(self.loop, db='python',
dboptions=dboptions)
def test_restart(self):
# we want to make sure the Stethoscope can be restarted
beats = []
lost = []
loop = ioloop.IOLoop()
def _hbreg():
beats.append('o')
def _onregister():
beats.append('+')
def _onbeat():
beats.append('.')
def _onbeatlost():
lost.append('.')
hb = Heartbeat('ipc:///tmp/stetho.ipc', interval=0.1,
io_loop=loop, onregister=_hbreg)
stetho = Stethoscope('ipc:///tmp/stetho.ipc', onbeat=_onbeat,
onbeatlost=_onbeatlost, delay=0.1,
io_loop=loop, onregister=_onregister,
warmup_delay=0)
# scenario
def start():
hb.start()
stetho.start()
def stop_st():
stetho.stop()
def restart_st():
stetho.start()
beats.append('RESTARTED')
def stop():
stetho.stop()
loop.stop()
# that starts the heartbeat and the client
loop.add_callback(start)
# the st stops after 500ms
loop.add_timeout(time.time() + .5, stop_st)
# the st starts again after 500ms
loop.add_timeout(time.time() + .5, restart_st)
# the st stops after 1 second, then the loop
loop.add_timeout(time.time() + 1., stop)
loop.start()
self.assertTrue(len(beats) > 0)
self.assertTrue('RESTARTED' in beats)
# make sure the st gets the beats after a restart
rest = beats.index('RESTARTED')
self.assertTrue('o+' in ''.join(beats[rest:]), beats)
def setUp(self):
self.loop = ioloop.IOLoop()
self.db = RedisDB(self.loop)
self._redis = redis.StrictRedis()
def __init__(self,
frontend=DEFAULT_FRONTEND,
backend=DEFAULT_BACKEND,
heartbeat=None,
register=DEFAULT_REG,
io_threads=DEFAULT_IOTHREADS,
agent_timeout=DEFAULT_AGENT_TIMEOUT,
receiver=DEFAULT_BROKER_RECEIVER,
publisher=DEFAULT_PUBLISHER,
db='python',
dboptions=None,
web_root=None):
# before doing anything, we verify if a broker is already up and
# running
logger.debug('Verifying if there is a running broker')
pid = verify_broker(frontend)
if pid is not None: # oops. can't do this !
logger.debug('Ooops, we have a running broker on that socket')
raise DuplicateBrokerError(pid)
self.endpoints = {
'frontend': frontend,
'backend': backend,
'register': register,
'receiver': receiver,
'publisher': publisher
}
if heartbeat is not None:
self.endpoints['heartbeat'] = heartbeat
logger.debug('Initializing the broker.')
for endpoint in self.endpoints.values():
if endpoint.startswith('ipc'):
register_ipc_file(endpoint)
self.context = zmq.Context(io_threads=io_threads)
# setting up the sockets
self._frontend = self.context.socket(zmq.ROUTER)
self._frontend.identity = 'broker-' + frontend
self._frontend.bind(frontend)
self._backend = self.context.socket(zmq.ROUTER)
self.pid = str(os.getpid())
self._backend.identity = self.pid
self._backend.bind(backend)
self._registration = self.context.socket(zmq.PULL)
self._registration.bind(register)
self._receiver = self.context.socket(zmq.PULL)
self._receiver.bind(receiver)
self._publisher = self.context.socket(zmq.PUB)
self._publisher.bind(publisher)
# setting up the streams
self.loop = ioloop.IOLoop()
self._frontstream = zmqstream.ZMQStream(self._frontend, self.loop)
self._frontstream.on_recv(self._handle_recv_front)
self._backstream = zmqstream.ZMQStream(self._backend, self.loop)
self._backstream.on_recv(self._handle_recv_back)
self._regstream = zmqstream.ZMQStream(self._registration, self.loop)
self._regstream.on_recv(self._handle_reg)
self._rcvstream = zmqstream.ZMQStream(self._receiver, self.loop)
self._rcvstream.on_recv(self._handle_recv)
# heartbeat
if heartbeat is not None:
self.pong = Heartbeat(heartbeat,
io_loop=self.loop,
ctx=self.context,
onregister=self._deregister)
else:
self.pong = None
# status
self.started = False
self.poll_timeout = None
# controller
self.ctrl = BrokerController(self,
self.loop,
db=db,
dboptions=dboptions,
agent_timeout=agent_timeout)
self.web_root = web_root