def test_handler(self):
if os.getenv('TRAVIS', False):
return
log = self._get_file()
stream = {'stream': FileStream(log)}
self._run_circus('circus.tests.test_stats_client.run_process',
stdout_stream=stream, stderr_stream=stream,
stats=True)
time.sleep(.5)
# checking that our system is live and running
client = CircusClient()
res = client.send_message('list')
watchers = res['watchers']
watchers.sort()
self.assertEqual(['circusd-stats', 'test'], watchers)
# making sure the stats process run
res = client.send_message('status', name='test')
self.assertEqual(res['status'], 'active')
res = client.send_message('status', name='circusd-stats')
self.assertEqual(res['status'], 'active')
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient()
next = client.iter_messages().next
for i in range(10):
watcher, pid, stat = next()
self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'),
watcher)
def circus_status(endpoint=None, process=None):
default = {
'pid': 'unknown',
'status': 'unknown',
'uptime': 'unknown'
}
if endpoint and process:
client = CircusClient(endpoint=endpoint, timeout=2)
try:
status = client.send_message('status')
stats = client.send_message('stats')
# Assuming here there's only a process
pid = stats['infos'][process].keys()[0]
try:
uptime = int(stats['infos'][process][pid]['age'])
default['uptime'] = humanize.naturaltime(datetime.datetime.now() - datetime.timedelta(seconds=uptime))
default['pid'] = pid
except:
# circus running but process stopped
pass
default['status'] = status['statuses'][process].lower()
except Exception as exc:
if'TIMED OUT' in exc.message.upper():
# circus stopped
default['status'] = 'unknown'
return default
def test_handler(self):
log = self._get_file()
stream = {'stream': FileStream(log)}
self._run_circus('circus.tests.test_stats_client.run_process',
stdout_stream=stream,
stderr_stream=stream,
stats=True)
time.sleep(.5)
# checking that our system is live and running
client = CircusClient()
res = client.send_message('list')
watchers = res['watchers']
watchers.sort()
self.assertEqual(['circusd-stats', 'test'], watchers)
# making sure the stats process run
res = client.send_message('status', name='test')
self.assertEqual(res['status'], 'active')
res = client.send_message('status', name='circusd-stats')
self.assertEqual(res['status'], 'active')
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient()
next = client.iter_messages().next
for i in range(10):
watcher, pid, stat = next()
self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'),
watcher)
def _send_message(command, **properties):
# check if circusct.endpoint is in minion config
endpoint = __salt__['config.get']('circusctl.endpoint') or \
DEFAULT_ENDPOINT_DEALER
# sending keys with None values in the message to circus will result
# an error. removing them from properties
props = dict((k, v) for k, v in properties.iteritems() if v)
client = CircusClient(endpoint=endpoint)
return client.send_message(command, **props)
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.support.run_process'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
def _test_singleton(self):
yield self._stop_runners()
dummy_process = 'circus.tests.support.run_process'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
def test_singleton(self):
self._stop_runners()
dummy_process = "circus.tests.test_arbiter.run_dummy"
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message("incr", name="test")
self.assertEqual(res["numprocesses"], 1)
def test_handler(self):
if os.getenv('TRAVIS', False):
return
log = self._get_file()
stream = {'stream': FileStream(log)}
self._run_circus('circus.tests.test_stats_client.run_process',
stdout_stream=stream,
stderr_stream=stream,
stats=True)
# waiting for data to appear in the file stream
empty = True
while empty:
with open(log) as f:
empty = f.read() == ''
time.sleep(.1)
# checking that our system is live and running
client = CircusClient()
res = client.send_message('list')
watchers = res['watchers']
watchers.sort()
self.assertEqual(['circusd-stats', 'test'], watchers)
# making sure the stats process run
res = client.send_message('status', name='test')
self.assertEqual(res['status'], 'active')
res = client.send_message('status', name='circusd-stats')
self.assertEqual(res['status'], 'active')
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient()
next = client.iter_messages().next
for i in range(10):
watcher, pid, stat = next()
self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'),
watcher)
def test_handler(self):
log = self._get_file()
stream = {'stream': FileStream(log)}
self._run_circus('circus.tests.test_stats_client.run_process',
stdout_stream=stream, stderr_stream=stream,
stats=True)
# waiting for data to appear in the file stream
empty = True
while empty:
with open(log) as f:
empty = f.read() == ''
time.sleep(.1)
# checking that our system is live and running
client = CircusClient()
res = client.send_message('list')
watchers = res['watchers']
watchers.sort()
self.assertEqual(['circusd-stats', 'test'], watchers)
# making sure the stats process run
res = client.send_message('status', name='test')
self.assertEqual(res['status'], 'active')
res = client.send_message('status', name='circusd-stats')
self.assertEqual(res['status'], 'active')
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient()
next = client.iter_messages().next
for i in range(10):
watcher, pid, stat = next()
self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'),
watcher)
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.support.run_process'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
self.cli.send_message("incr", name="test", nb=2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 4)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable,
'circus.tests.support.run_process',
testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.support.run_process']
return cmd, args
def _get_options(self, **kwargs):
if 'graceful_timeout' not in kwargs:
kwargs['graceful_timeout'] = 4
return kwargs
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True, options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True,
options=self._get_options(flapping_window=100))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
self.assertTrue(poll_for(self.test_file, 'START')) # process started
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
self.assertTrue(poll_for(self.test_file, 'START')) # process started
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.support.run_process'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
def nb_processes():
return len(cli.send_message('list', name='test').get('pids'))
def incr_processes():
return cli.send_message('incr', name='test')
# wait for the plugin to be started
self.assertTrue(poll_for(datafile, 'PLUGIN STARTED'))
cli = CircusClient()
self.assertEqual(nb_processes(), 1)
incr_processes()
self.assertEqual(nb_processes(), 2)
# wait for the plugin to receive the signal
self.assertTrue(poll_for(datafile, 'test:spawn'))
truncate_file(datafile)
incr_processes()
self.assertEqual(nb_processes(), 3)
# wait for the plugin to receive the signal
self.assertTrue(poll_for(datafile, 'test:spawn'))
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.support.run_process'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
class StatsStreamer(object):
def __init__(self,
endpoint,
pubsub_endoint,
stats_endpoint,
ssh_server,
delay=1.):
self.topic = 'watcher.'
self.delay = delay
self.ctx = zmq.Context()
self.pubsub_endpoint = pubsub_endoint
self.sub_socket = self.ctx.socket(zmq.SUB)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic)
self.sub_socket.connect(self.pubsub_endpoint)
self.loop = ioloop.IOLoop.instance() # events coming from circusd
self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop)
self.substream.on_recv(self.handle_recv)
self.client = CircusClient(context=self.ctx,
endpoint=endpoint,
ssh_server=ssh_server)
self.cmds = get_commands()
self._pids = defaultdict(list)
self._callbacks = dict()
self.publisher = StatsPublisher(stats_endpoint, self.ctx)
self.running = False # should the streamer be running?
self.stopped = False # did the collect started yet?
self.circus_pids = {}
self.sockets = []
def get_watchers(self):
return self._pids.keys()
def get_sockets(self):
return self.sockets
def get_pids(self, watcher=None):
if watcher is not None:
if watcher == 'circus':
return self.circus_pids.keys()
return self._pids[watcher]
return chain(self._pid.values())
def get_circus_pids(self):
# getting the circusd, circusd-stats and circushttpd pids
res = self.client.send_message('dstats')
pids = {os.getpid(): 'circusd-stats', res['info']['pid']: 'circusd'}
httpd_pids = self.client.send_message('list', name='circushttpd')
if 'pids' in httpd_pids:
httpd_pids = httpd_pids['pids']
if len(httpd_pids) == 1:
pids[httpd_pids[0]] = 'circushttpd'
return pids
def _add_callback(self, name, start=True, kind='watcher'):
logger.debug('Callback added for %s' % name)
if kind == 'watcher':
klass = WatcherStatsCollector
elif kind == 'socket':
klass = SocketStatsCollector
else:
raise ValueError('Unknown callback kind %r' % kind)
self._callbacks[name] = klass(self, name, self.delay, self.loop)
if start:
self._callbacks[name].start()
def _init(self):
self._pids.clear()
# getting the initial list of watchers/pids
res = self.client.send_message('list')
for watcher in res['watchers']:
if watcher in ('circusd', 'circushttpd', 'circusd-stats'):
# this is dealt by the special 'circus' collector
continue
pids = self.client.send_message('list', name=watcher)['pids']
for pid in pids:
self.append_pid(watcher, pid)
# getting the circus pids
self.circus_pids = self.get_circus_pids()
if 'circus' not in self._callbacks:
self._add_callback('circus')
else:
self._callbacks['circus'].start()
# getting the initial list of sockets
res = self.client.send_message('listsockets')
for sock in res['sockets']:
fd = sock['fd']
address = '%s:%s' % (sock['host'], sock['port'])
# XXX type / family ?
sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
self.sockets.append((sock, address, fd))
self._add_callback('sockets', kind='socket')
def remove_pid(self, watcher, pid):
if pid in self._pids[watcher]:
logger.debug('Removing %d from %s' % (pid, watcher))
self._pids[watcher].remove(pid)
if len(self._pids[watcher]) == 0:
logger.debug(
'Stopping the periodic callback for {0}'.format(watcher))
self._callbacks[watcher].stop()
def append_pid(self, watcher, pid):
if watcher not in self._pids or len(self._pids[watcher]) == 0:
logger.debug(
'Starting the periodic callback for {0}'.format(watcher))
if watcher not in self._callbacks:
self._add_callback(watcher)
else:
self._callbacks[watcher].start()
if pid in self._pids[watcher]:
return
self._pids[watcher].append(pid)
logger.debug('Adding %d in %s' % (pid, watcher))
def start(self):
self.running = True
logger.info('Starting the stats streamer')
self._init()
logger.debug('Initial list is ' + str(self._pids))
logger.debug('Now looping to get circusd events')
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
self.stop()
def handle_recv(self, data):
"""called each time circusd sends an event"""
# maintains a periodic callback to compute mem and cpu consumption for
# each pid.
logger.debug('Received an event from circusd: %s' % data)
topic, msg = data
try:
__, watcher, action = topic.split('.')
msg = json.loads(msg)
if action == 'start' or (action != 'start' and self.stopped):
self._init()
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
pid = msg['process_pid']
self.append_pid(watcher, pid)
elif action == 'start':
self._init()
elif action == 'stop':
self.stop()
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to handle %r' % msg)
def stop(self):
# stop all the periodic callbacks running
for callback in self._callbacks.values():
callback.stop()
self.loop.stop()
self.ctx.destroy(0)
self.publisher.stop()
self.stopped = True
self.running = False
logger.info('Stats streamer stopped')
class StatsStreamer(object):
def __init__(self, endpoint, pubsub_endoint, stats_endpoint, ssh_server,
delay=1., loop=None):
self.topic = 'watcher.'
self.delay = delay
self.ctx = zmq.Context()
self.pubsub_endpoint = pubsub_endoint
self.sub_socket = self.ctx.socket(zmq.SUB)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic)
self.sub_socket.connect(self.pubsub_endpoint)
self.loop = loop or ioloop.IOLoop.instance()
self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop)
self.substream.on_recv(self.handle_recv)
self.client = CircusClient(context=self.ctx, endpoint=endpoint,
ssh_server=ssh_server)
self.cmds = get_commands()
self._pids = defaultdict(list)
self._callbacks = dict()
self.publisher = StatsPublisher(stats_endpoint, self.ctx)
self.running = False # should the streamer be running?
self.stopped = False # did the collect started yet?
self.circus_pids = {}
self.sockets = []
def get_watchers(self):
return self._pids.keys()
def get_sockets(self):
return self.sockets
def get_pids(self, watcher=None):
if watcher is not None:
if watcher == 'circus':
return self.circus_pids.keys()
return self._pids[watcher]
return chain(self._pids.values())
def get_circus_pids(self):
watchers = self.client.send_message('list').get('watchers', [])
# getting the circusd, circusd-stats and circushttpd pids
res = self.client.send_message('dstats')
pids = {os.getpid(): 'circusd-stats'}
if 'info' in res:
pids[res['info']['pid']] = 'circusd'
if 'circushttpd' in watchers:
httpd_pids = self.client.send_message('list', name='circushttpd')
if 'pids' in httpd_pids:
httpd_pids = httpd_pids['pids']
if len(httpd_pids) == 1:
pids[httpd_pids[0]] = 'circushttpd'
return pids
def _add_callback(self, name, start=True, kind='watcher'):
logger.debug('Callback added for %s' % name)
if kind == 'watcher':
klass = WatcherStatsCollector
elif kind == 'socket':
klass = SocketStatsCollector
else:
raise ValueError('Unknown callback kind %r' % kind)
self._callbacks[name] = klass(self, name, self.delay, self.loop)
if start:
self._callbacks[name].start()
def _init(self):
self._pids.clear()
# getting the initial list of watchers/pids
res = self.client.send_message('list')
for watcher in res['watchers']:
if watcher in ('circusd', 'circushttpd', 'circusd-stats'):
# this is dealt by the special 'circus' collector
continue
pid_list = self.client.send_message('list', name=watcher)
pids = pid_list.get('pids', [])
for pid in pids:
self._append_pid(watcher, pid)
# getting the circus pids
self.circus_pids = self.get_circus_pids()
if 'circus' not in self._callbacks:
self._add_callback('circus')
else:
self._callbacks['circus'].start()
# getting the initial list of sockets
res = self.client.send_message('listsockets')
for sock in res.get('sockets', []):
fd = sock['fd']
address = '%s:%s' % (sock['host'], sock['port'])
# XXX type / family ?
sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
self.sockets.append((sock, address, fd))
self._add_callback('sockets', kind='socket')
def stop_watcher(self, watcher):
for pid in self._pids[watcher]:
self.remove_pid(watcher, pid)
def remove_pid(self, watcher, pid):
if pid in self._pids[watcher]:
logger.debug('Removing %d from %s' % (pid, watcher))
self._pids[watcher].remove(pid)
if len(self._pids[watcher]) == 0:
logger.debug(
'Stopping the periodic callback for {0}' .format(watcher))
self._callbacks[watcher].stop()
def _append_pid(self, watcher, pid):
if watcher not in self._pids or len(self._pids[watcher]) == 0:
logger.debug(
'Starting the periodic callback for {0}'.format(watcher))
if watcher not in self._callbacks:
self._add_callback(watcher)
else:
self._callbacks[watcher].start()
if pid in self._pids[watcher]:
return
self._pids[watcher].append(pid)
logger.debug('Adding %d in %s' % (pid, watcher))
def start(self):
self.running = True
logger.info('Starting the stats streamer')
self._init()
logger.debug('Initial list is ' + str(self._pids))
logger.debug('Now looping to get circusd events')
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
self.stop()
def handle_recv(self, data):
"""called each time circusd sends an event"""
# maintains a periodic callback to compute mem and cpu consumption for
# each pid.
logger.debug('Received an event from circusd: %s' % data)
topic, msg = data
try:
watcher = topic.split('.')[1:-1][0]
action = topic.split('.')[-1]
msg = json.loads(msg)
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
# a process was added
pid = msg['process_pid']
self._append_pid(watcher, pid)
elif action == 'stop':
# the whole watcher was stopped.
self.stop_watcher(watcher)
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to handle %r' % msg)
def stop(self):
# stop all the periodic callbacks running
for callback in self._callbacks.values():
callback.stop()
self.loop.stop()
self.ctx.destroy(0)
self.publisher.stop()
self.stopped = True
self.running = False
logger.info('Stats streamer stopped')
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher == 'circusd-stats':
continue
options = self.client.send_message('options', name=watcher)
self.watchers.append((watcher, options['options']))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
except CallError:
self.connected = False
def killproc(self, name, pid):
res = self.client.send_message('signal',
name=name,
process=int(pid),
signum=9)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
self.cli.send_message("incr", name="test", nb=2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 4)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable, 'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def _get_options(self, **kwargs):
if 'graceful_timeout' not in kwargs:
kwargs['graceful_timeout'] = 4
return kwargs
def test_add_watcher(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options=self._get_options(flapping_window=100))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
def nb_processes():
return len(cli.send_message('list', name='test').get('pids'))
def incr_processes():
return cli.send_message('incr', name='test')
cli = CircusClient()
self.assertEqual(nb_processes(), 1)
incr_processes()
self.assertEqual(nb_processes(), 2)
incr_processes()
self.assertEqual(nb_processes(), 3)
# wait a bit
time.sleep(.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split('\n') if line != '']
wanted = ['test:spawn', 'test:spawn']
self.assertEqual(data, wanted)
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
class StatsStreamer(object):
def __init__(self, endpoint, pubsub_endoint, stats_endpoint, delay=1.):
self.topic = 'watcher.'
self.delay = delay
self.ctx = zmq.Context()
self.pubsub_endpoint = pubsub_endoint
self.sub_socket = self.ctx.socket(zmq.SUB)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic)
self.sub_socket.connect(self.pubsub_endpoint)
self.loop = ioloop.IOLoop() # events coming from circusd
self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop)
self.substream.on_recv(self.handle_recv)
self.client = CircusClient(context=self.ctx, endpoint=endpoint)
self.cmds = get_commands()
self._pids = defaultdict(list)
self._callbacks = dict()
self.collector = StatsCollector()
self.publisher = StatsPublisher(stats_endpoint, self.ctx)
self.running = False # should the streamer be running?
self.stopped = False # did the collect started yet?
self.circus_pids = {}
def publish_stats(self, watcher=None):
"""Get and publish the stats for the given watcher"""
logger.debug('Publishing stats about {0}'.format(watcher))
process_name = None
for watcher, pid, stats in self.collector.collect_stats(
watcher, self.get_pids(watcher)):
if watcher == 'circus':
if pid in self.circus_pids:
process_name = self.circus_pids[pid]
self.publisher.publish(watcher, process_name, pid, stats)
def get_watchers(self):
return self._pids.keys()
def get_pids(self, watcher=None):
if watcher is not None:
if watcher == 'circus':
return self.circus_pids.keys()
return self._pids[watcher]
return chain(self._pid.values())
def get_circus_pids(self):
# getting the circusd pid
res = self.client.send_message('dstats')
return {os.getpid(): 'circusd-stats',
res['info']['pid']: 'circusd'}
def _init(self):
self.circus_pids = self.get_circus_pids()
if 'circus' not in self._callbacks:
self._callbacks['circus'] = ioloop.PeriodicCallback(
lambda: self.publish_stats("circus"),
self.delay * 1000, self.loop)
self._callbacks['circus'].start()
self._pids.clear()
# getting the initial list of watchers/pids
res = self.client.send_message('list')
for watcher in res['watchers']:
pids = self.client.send_message('list', name=watcher)['pids']
for pid in pids:
self.append_pid(watcher, pid)
def remove_pid(self, watcher, pid):
if pid in self._pids[watcher]:
logger.debug('Removing %d from %s' % (pid, watcher))
self._pids[watcher].remove(pid)
if len(self._pids[watcher]) == 0:
logger.debug('Stopping the periodic callback for {0}'\
.format(watcher))
self._callbacks[watcher].stop()
def append_pid(self, watcher, pid):
if watcher not in self._pids or len(self._pids[watcher]) == 0:
if watcher not in self._callbacks:
self._callbacks[watcher] = ioloop.PeriodicCallback(
lambda: self.publish_stats(watcher),
self.delay * 1000, self.loop)
logger.debug('Starting the periodic callback for {0}'\
.format(watcher))
self._callbacks[watcher].start()
if pid in self._pids[watcher]:
return
self._pids[watcher].append(pid)
logger.debug('Adding %d in %s' % (pid, watcher))
def start(self):
self.running = True
logger.info('Starting the stats streamer')
self._init()
logger.debug('Initial list is ' + str(self._pids))
logger.debug('Now looping to get circusd events')
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
self.stop()
def handle_recv(self, data):
"""called each time circusd sends an event"""
# maintains a periodic callback to compute mem and cpu consumption for
# each pid.
logger.debug('Received an event from circusd: %s' % data)
topic, msg = data
try:
__, watcher, action = topic.split('.')
msg = json.loads(msg)
if action == 'start' or (action != 'start' and self.stopped):
self._init()
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
pid = msg['process_pid']
self.append_pid(watcher, pid)
elif action == 'start':
self._init()
elif action == 'stop':
self.stop()
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to handle %r' % msg)
def stop(self):
# stop all the periodic callbacks running
for callback in self._callbacks.values():
callback.stop()
self.loop.stop()
self.ctx.destroy(0)
self.publisher.stop()
self.stopped = True
self.running = False
logger.info('Stats streamer stopped')
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
self.sockets = None
self.use_sockets = False
self.embed_httpd = False
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher in ('circusd-stats', 'circushttpd'):
if watcher == 'circushttpd':
self.embed_httpd = True
continue
options = self.client.send_message('options',
name=watcher)['options']
self.watchers.append((watcher, options))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
if not self.use_sockets and options.get('use_sockets', False):
self.use_sockets = True
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
except CallError:
self.connected = False
def killproc(self, name, pid):
res = self.client.send_message('signal', name=name, pid=int(pid),
signum=9, children=True)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_sockets(self, force_reload=False):
if not self.sockets or force_reload:
res = self.client.send_message('listsockets')
self.sockets = res['sockets']
return self.sockets
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
def circus_control(action, endpoint=None, process=None):
if endpoint and process:
client = CircusClient(endpoint=endpoint, timeout=2)
client.send_message(action)
class LiveClient(object):
def __init__(self, endpoint, ssh_server=None):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint,
ssh_server=ssh_server)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
self.sockets = None
self.use_sockets = False
self.embed_httpd = False
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
self.plugins = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher in ('circusd-stats', 'circushttpd'):
if watcher == 'circushttpd':
self.embed_httpd = True
continue
options = self.client.send_message('options',
name=watcher)['options']
self.watchers.append((watcher, options))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
if not self.use_sockets and options.get('use_sockets', False):
self.use_sockets = True
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
if self.endpoint.startswith('tcp://'):
# In case of multi interface binding i.e: tcp://0.0.0.0:5557
anyaddr = '0.0.0.0'
ip = self.endpoint.lstrip('tcp://').split(':')[0]
self.stats_endpoint = self.stats_endpoint.replace(anyaddr, ip)
except CallError:
self.connected = False
def killproc(self, name, pid):
# killing a proc and its children
res = self.client.send_message('signal', name=name, pid=int(pid),
signum=9, recursive=True)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_sockets(self, force_reload=False):
if not self.sockets or force_reload:
res = self.client.send_message('listsockets')
self.sockets = res['sockets']
return self.sockets
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
