class TestCircus(unittest.TestCase):
def setUp(self):
self.arbiters = []
self.files = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
self.cli.stop()
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, 'w') as f:
f.write(content)
return file
@classmethod
def _create_circus(cls, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ['generic.py', callable, testfile]
worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir,
'name': 'test', 'graceful_timeout': 4}
worker.update(kw)
if stats:
arbiter = get_arbiter([worker], background=True, plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
debug=kw.get('debug', False))
else:
arbiter = get_arbiter([worker], background=True, plugins=plugins,
debug=kw.get('debug', False))
arbiter.start()
return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw):
testfile, arbiter = TestCircus._create_circus(callable, plugins, stats,
**kw)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
def get_pids():
return self.call('list', name='test').get('pids')
pids = get_pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(
self.status('signal',
name='test',
pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = get_pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(
self.call("list", name="test").get('processes'), [1, 2])
self.assertEquals(
self.status("signal",
name="test",
process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(
self.call("list", name="test").get('processes'), [1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(
self.status("signal", name="test", signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(
self.call("list", name="test").get('processes'), processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(2.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
def get_pids():
return self.call('list', name='test').get('pids')
pids = get_pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(self.status('signal', name='test', pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = get_pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 2])
self.assertEquals(self.status("signal", name="test", process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(self.status("signal", name="test",
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(self.call("list", name="test").get('processes'),
processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(2.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def _rm_process(self, name):
client = CircusClient(endpoint=self.circus_endpoint)
# https://github.com/circus-tent/circus/issues/927
name = name.lower()
cmd_rm = {"command": "rm", "properties": {"name": name}}
try:
call_rm = client.call(cmd_rm)
self.log.debug("_rm_device circus client call: %s", str(call_rm))
except CallError:
self.log.debug("Could not rm process: %s", name, exc_info=True)
def _test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
self._stop_runners()
cli = CircusClient()
dummy_process = 'circus.tests.support.run_process'
self.test_file = self._run_circus(dummy_process)
msg = make_message("numprocesses")
resp = cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def _test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
self._stop_runners()
cli = CircusClient()
dummy_process = 'circus.tests.support.run_process'
self.test_file = self._run_circus(dummy_process)
msg = make_message("numprocesses")
resp = cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def _running(self):
"""Return Brewpi instances running using suffix as filter"""
client = CircusClient(endpoint=self.circus_endpoint)
try:
call = client.call({"command": "list", "properties": {}})
except CallError:
self.log.error("Could not get running processes", exc_info=True)
return []
running_devices = [
x for x in call['watchers'] if x.startswith(self.prefix)
]
return running_devices
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
# checking what the plugin did
wanted = [('test', 'spawn'), ('test', 'start'), ('test', 'spawn')]
self.assertEqual(Plugin.events, wanted)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 2])
self.assertEquals(self.status("signal", name="test", process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(self.status("signal", name="test",
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(self.call("list", name="test").get('processes'),
processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'], 'python')
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = 'circus.tests.test_stream.run_process'
fd, log = tempfile.mkstemp()
self.log = log
os.close(fd)
stream = {'stream': FileStream(log)}
self.test_file = self._run_circus(dummy_process,
stdout_stream=stream, stderr_stream=stream, debug=True)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
os.remove(self.log)
def test_stream(self):
time.sleep(2.)
self.call("stats").get('infos')
# let's see what we got in the file
with open(self.log) as f:
data = f.read()
self.assertTrue('stderr' in data)
self.assertTrue('stdout' in data)
# restarting
self.call('restart')
time.sleep(1.)
current = time.time()
# should be running
with open(self.log) as f:
data = f.readlines()
# last log should be less than one second old
last = data[-1]
delta = abs(current - int(last.split('-')[0]))
self.assertTrue(delta < 1, delta)
def _stop_process(self, name):
client = CircusClient(endpoint=self.circus_endpoint)
# https://github.com/circus-tent/circus/issues/927
name = name.lower()
cmd_stop = {
"command": "stop",
"properties": {
"waiting": False,
"name": name,
"match": "glob"
}
}
try:
call_stop = client.call(cmd_stop)
self.log.debug("_stop_process circus client call: %s",
str(call_stop))
except CallError:
self.log.debug("Could not stop process: %s", name, exc_info=True)
class CircusMgr(object):
"""Fermentrack Circus Handler, It is a simple wrapper around
circus client, any errors raised as CircusException"""
def __init__(self, connection_timeout=2, circus_endpoint=DEFAULT_ENDPOINT_DEALER):
self._client = CircusClient(
timeout=connection_timeout, endpoint=circus_endpoint)
def _call(self, command, **props):
message = {"command": command, "properties": props or {}}
try:
res = self._client.call(message)
except CallError, callerr:
LOG.debug("Error from circus", exc_info=True)
raise CircusException("Could send message to circus: {}".format(callerr))
if res['status'] == u'error':
raise CircusException("Error: {}".format(res['reason']))
return res
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = "circus.tests.test_stream.run_process"
fd, self.stdout = tempfile.mkstemp()
os.close(fd)
fd, self.stderr = tempfile.mkstemp()
os.close(fd)
self.test_file = self._run_circus(
dummy_process,
stdout_stream={"stream": FileStream(self.stdout)},
stderr_stream={"stream": FileStream(self.stderr)},
debug=True,
)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
os.remove(self.stdout)
os.remove(self.stderr)
def test_stream(self):
# wait for the process to be started
self.assertTrue(poll_for(self.stdout, "stdout"))
self.assertTrue(poll_for(self.stderr, "stderr"))
# clean slate
truncate_file(self.stdout)
truncate_file(self.stderr)
# restart and make sure streams are still working
self.call("restart")
# wait for the process to be restarted
self.assertTrue(poll_for(self.stdout, "stdout"))
self.assertTrue(poll_for(self.stderr, "stderr"))
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = 'circus.tests.test_stream.run_process'
fd, self.stdout = tempfile.mkstemp()
os.close(fd)
fd, self.stderr = tempfile.mkstemp()
os.close(fd)
self.test_file = self._run_circus(
dummy_process,
stdout_stream={'stream': FileStream(self.stdout)},
stderr_stream={'stream': FileStream(self.stderr)},
debug=True)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
os.remove(self.stdout)
os.remove(self.stderr)
def test_stream(self):
# wait for the process to be started
self.assertTrue(poll_for(self.stdout, 'stdout'))
self.assertTrue(poll_for(self.stderr, 'stderr'))
# clean slate
truncate_file(self.stdout)
truncate_file(self.stderr)
# restart and make sure streams are still working
self.call('restart')
# wait for the process to be restarted
self.assertTrue(poll_for(self.stdout, 'stdout'))
self.assertTrue(poll_for(self.stderr, 'stderr'))
def _add_process(self, name):
"""Spawn a new brewpi.py process via circus, 'dev-' is appended to the name to
keep brewpi devices seperated from other devices
"""
client = CircusClient(endpoint=self.circus_endpoint)
proc_name = self.prefix + name
# https://github.com/circus-tent/circus/issues/927
proc_name = proc_name.lower()
try:
call = client.call({
"command": "add",
"properties": {
"cmd": self.command_tmpl % name,
"name": proc_name,
"options": {
"copy_env": True,
"stdout_stream": {
"class":
"FileStream",
"filename":
"%s/%s-stdout.log" % (self.logfilepath, proc_name),
},
"stderr_stream": {
"class":
"FileStream",
"filename":
"%s/%s-stderr.log" % (self.logfilepath, proc_name),
}
},
"start": True
}
})
self.log.debug("_add_process circus client call: %s", str(call))
except CallError:
self.log.error("Could not spawn process: %s",
proc_name,
exc_info=True)
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2, 3])
# wait a bit
time.sleep(.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split('\n') if line != '']
wanted = ['test:spawn', 'test:spawn']
self.assertEqual(data, wanted)
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = "circus.tests.test_arbiter.run_dummy"
plugin = "circus.tests.test_arbiter.Plugin"
plugins = [{"use": plugin, "file": datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2, 3])
# wait a bit
time.sleep(0.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split("\n") if line != ""]
wanted = ["test:spawn", "test:spawn"]
self.assertEqual(data, wanted)
class DaemonTamer(object):
_DAEMON_MAP_FILE = 'daemon_map.yaml'
_ENDPOINT_TPL = '{host}:{port}'
def __init__(self, profile):
self.profile = profile
self.daemon_name = get_daemon_name(profile)
self.config = self.get_update_profile()
self._client = CircusClient() # endpoint=self.endpoint)
def generate_new_portnum(self, config=None):
port = 6000
used_ports = [profile['port'] for profile in config.values()]
while port in used_ports:
port += 2
return port
def read_config(self):
if not path.isfile(self._DAEMON_MAP_FILE):
return {}
with open(self._DAEMON_MAP_FILE, 'r') as daemon_map_fo:
config = yaml.load(daemon_map_fo) or {}
return config
def write_config(self, config):
with open(self._DAEMON_MAP_FILE, 'w') as daemon_map_fo:
yaml.dump(config, daemon_map_fo)
def get_update_profile(self):
config = self.read_config()
profile_config = config.get(self.profile, {})
profile_config['uuid'] = get_profile_uuid(self.profile)
profile_config['host'] = profile_config.get('host', 'tcp://127.0.0.1')
profile_config['port'] = profile_config.get('port', self.generate_new_portnum(config))
config[self.profile] = profile_config
self.write_config(config)
return config
@property
def profile_config(self):
return self.config[self.profile]
@property
def endpoint(self):
return self._ENDPOINT_TPL.format(host=self.profile_config['host'], port=self.profile_config['port'])
@property
def pubsub_endpoint(self):
return self._ENDPOINT_TPL.format(host=self.profile_config['host'], port=self.profile_config['port'] + 1)
@property
def stats_endpoint(self):
return self._ENDPOINT_TPL.format(host=self.profile_config['host'], port=self.profile_config['port'] + 2)
@property
def arbiter_config(self):
return {
'logoutput': 'aiida-circus.log',
'loglevel': 'INFO',
'debug': False,
'statsd': True,
# ~ 'controller': self.endpoint,
# ~ 'pubsub_endpoint': self.pubsub_endpoint,
# ~ 'stats_endpoint': self.stats_endpoint
}
@property
def watcher_config(self):
return get_daemon_properties(self.profile)
def make_arbiter(self):
print self.arbiter_config
print self.watcher_config
return get_arbiter([self.watcher_config], **self.arbiter_config)
def pause_daemon(self):
self._client.stop()
def unpause_daemon(self):
self._client.call({
'command': 'start',
'properties': {}
})
def status(self):
response = self._client.call({
'command': 'status',
'properties': {
'name': self.daemon_name
}
})
if response.get('status', None):
return str(response['status'])
def quit(self):
try:
self._client.call({
'command': 'quit',
'properties': {}
})
except CallError:
pass
class CircusManager(object):
def __init__(self, config):
#self._arbiter = get_arbiter([])
#self._arbiter.start()
self._config = config
self._client = CircusClient(timeout=15, endpoint=self._config['circus_endpoint'])
def __call(self, message):
return self._client.call(message)
def add_application(self, name, command, arguments=[], autostart=True):
response = self.__call({
'command':'add',
'properties':{
'cmd': command,
'name':name,
'start': autostart,
'args':arguments,
'options':{
'singleton': True,
'stop_children': True,
'stop_signal': 9,
'env': self._config.get('environment', {}),
'graceful_timeout': self._config['graceful_timeout'],
'max_retry': self._config['max_retry'],
'stdout_stream': {
'class': self._config['logging']['stdout_stream'],
'filename': '{path}/{filename}.{ext}'.format(path=self._config['logging']['path'],
filename=name,
ext=self._config['logging']['stdout_extension'])
},
'stderr_stream': {
'class':self._config['logging']['stderr_stream'],
'filename': '{path}/{filename}.{ext}'.format(path=self._config['logging']['path'],
filename=name,
ext=self._config['logging']['stderr_extension'])
}
}
}
})
return (True, None) if response['status'] == u'ok' else (False, response['reason'])
def application_status(self, name):
response = self.__call({
'command': 'status',
'properties': {
'name': name
}
})
return str(response['status']) if response['status'] != u'error' else 'not running'
def kill_application(self, name):
self.__call({
'command': 'signal',
'properties':{
'name': name,
'signal': self._config['stop_signal']
}
})
def remove_application(self, name, nonstop=False):
response = self.__call({
'command': 'rm',
'properties':{
'name': name,
'nonstop': nonstop,
'waiting': False
}
})
return True if response['status'] == u'ok' else False
def reload_application(self, name, waiting=False, graceful=True, sequential=False):
response = self.__call({
'command': 'reload',
'properties':{
'name': name,
'graceful': graceful,
'sequential': sequential,
'waiting': waiting
}
})
return True if response['status'] == u'ok' else False
def start_application(self, name, waiting=False):
response = self.__call({
'command': 'start',
'properties':{
'name': name,
'waiting': waiting
}
})
return True if response['status'] == u'ok' else False
def stop_application(self, name, waiting=False):
response = self.__call({
'command': 'stop',
'properties':{
'name': name,
'waiting': waiting
}
})
return True if response['status'] == u'ok' else False
def stop_and_remove_application(self, name):
self.stop_application(name)
return self.remove_application(name)
def get_applications(self, verbose=False):
response = self.__call({
'command': 'list'
})
if not verbose:
return [{'name':str(w)} for w in response['watchers']] if response['status'] == u'ok' else []
return [{'name':str(w), 'status': self.application_status(str(w))} for w in response['watchers']] if response['status'] == u'ok' else []
def stop(self):
self._arbiter.stop()
import sys
from circus.client import CircusClient
from circus.util import DEFAULT_ENDPOINT_DEALER
client = CircusClient(endpoint=DEFAULT_ENDPOINT_DEALER)
command = '../bin/python dummy_fly.py 111'
name = 'dummy'
for i in range(50):
print client.call("""
{
"command": "add",
"properties": {
"cmd": "%s",
"name": "%s",
"options": {
"copy_env": true,
"stdout_stream": {
"filename": "stdout.log"
},
"stderr_stream": {
"filename": "stderr.log"
}
},
"start": true
}
}
""" % (command, name + str(i)))
class ProcfileWatcher(CircusPlugin):
name = "procfile_watcher"
def __init__(self, *args, **config):
super(ProcfileWatcher, self).__init__(*args, **config)
self.loop_rate = config.get("loop_rate", 3) # in seconds
self.procfile_path = config.get("app_path",
"/home/application/current/Procfile")
self.working_dir = config.get("working_dir",
"/home/application/current")
self.apprc = config.get("apprc", "/home/application/apprc")
self.port = config.get("port", "8888")
self.uid = config.get("uid", "ubuntu")
self.stderr_stream = {"class": config.get("stderr_stream",
"tsuru.stream.Stream")}
self.stdout_stream = {"class": config.get("stdout_stream",
"tsuru.stream.Stream")}
file_watcher = FileWatcher(self.procfile_path, self.reload_procfile)
self.period = ioloop.PeriodicCallback(file_watcher,
self.loop_rate * 1000,
self.loop)
self.circus_client = CircusClient()
def get_cmd(self, name):
return self.call("get", name=name, keys=["cmd"])["options"]["cmd"]
def handle_init(self):
self.period.start()
def handle_stop(self):
self.period.stop()
def handle_recv(self, data):
pass
def add_watcher(self, name, cmd):
env = {"port": self.port, "PORT": self.port}
env.update(common.load_envs(self.apprc))
cmd = replace_args(cmd, **env)
options = {
"env": env,
"copy_env": True,
"working_dir": self.working_dir,
"stderr_stream": self.stderr_stream,
"stdout_stream": self.stdout_stream,
"uid": self.uid,
"hooks": {
"before_start": "tsuru.hooks.before_start",
"after_start": "tsuru.hooks.after_start",
}
}
self.circus_client.call(json.dumps({
"command": "add",
"properties": {
"cmd": cmd,
"name": name,
"args": [],
"options": options,
"start": True,
},
}))
def remove_watcher(self, name):
self.call("rm", name=name)
def change_cmd(self, name, cmd):
env = {"port": self.port}
env.update(common.load_envs(self.apprc))
cmd = replace_args(cmd, **env)
self.call("set", name=name, options={"cmd": cmd})
def commands(self, procfile):
cmds = self.call("status")["statuses"]
cmds_names = set([k for k in cmds.keys()
if not k.startswith("plugin:")])
new_cmds = set(procfile.commands.keys())
to_remove = cmds_names.difference(new_cmds)
to_add = new_cmds.difference(cmds_names)
to_change_names = cmds_names.intersection(new_cmds)
to_change = {}
for name in to_change_names:
if self.get_cmd(name) != procfile.commands.get(name):
to_change[name] = procfile.commands[name]
return to_add, to_remove, to_change
def reload_procfile(self):
with open(self.procfile_path) as file:
procfile = Procfile(file.read())
to_add, to_remove, to_change = self.commands(procfile)
for name in to_remove:
self.remove_watcher(name)
for name in to_add:
self.add_watcher(name=name, cmd=procfile.commands[name])
for name, cmd in to_change.items():
self.change_cmd(name=name, cmd=procfile.commands[name])
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable, 'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('processes')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
self.test_file = self._run_circus('circus.tests.test_trainer.run_dummy')
time.sleep(0.5)
self.cli = CircusClient(TEST_ENDPOINT)
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numshows(self):
resp = self.cli.call("numshows")
self.assertEqual(resp, "1")
def test_numflies(self):
resp = self.cli.call("numflies")
self.assertEqual(resp, "1")
def test_flies(self):
resp = self.cli.call("flies")
self.assertEqual(resp, "test: 1")
def test_shows(self):
resp = self.cli.call("shows")
self.assertEqual(resp, "test")
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (sys.executable,
'circus.tests.test_trainer.run_dummy', testfile)
return cmd
def test_add_show(self):
cmd = self._get_cmd()
resp = self.cli.call("add_show test1 %s" % cmd)
self.assertEqual(resp, "ok")
def test_add_show1(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
resp = self.cli.call("shows")
self.assertTrue(resp.endswith("test1"))
def test_add_show2(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
resp = self.cli.call("numshows")
self.assertEqual(resp, "2")
def test_add_show3(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
resp = self.cli.call("add_show test1 %s" % cmd)
self.assertTrue(resp.startswith("error:"))
def test_del_show(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
self.cli.call("del_show test1")
resp = self.cli.call("numshows")
self.assertEqual(resp, "1")
def test_stop(self):
self.cli.call("stop")
self.assertRaises(CallError, self.cli.call, "shows")
def test_reload(self):
resp = self.cli.call("reload")
self.assertEqual(resp, "ok")
def test_reload1(self):
flies0 = self.cli.call("flies")
resp = self.cli.call("reload")
time.sleep(0.5)
flies1 = self.cli.call("flies")
self.assertNotEqual(flies0, flies1)
def test_reload(self):
self.cli.call("reload")
time.sleep(0.5)
flies = self.cli.call("flies")
self.assertEqual(flies, "test: 2")
def test_stop_shows(self):
resp = self.cli.call("stop_shows")
self.assertEqual(resp, "ok")
def test_stop_shows1(self):
self.cli.call("stop_shows")
resp = self.cli.call("status test")
self.assertEqual(resp, "stopped")
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.stats = defaultdict(list)
self.refresher = Refresher(self)
self.dstats = []
def stop(self):
self.client.stop()
self.refresher.running = False
self.refresher.join()
def verify(self):
self.watchers = []
# trying to list the watchers
msg = cmds['list'].make_message()
try:
res = self.client.call(msg)
self.connected = True
for watcher in res['watchers']:
if watcher == 'circusd-stats':
continue
msg = cmds['options'].make_message(name=watcher)
options = self.client.call(msg)
self.watchers.append((watcher, options['options']))
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
if not self.refresher.running:
self.refresher.start()
except CallError:
self.connected = False
def killproc(self, name, pid):
msg = cmds['signal'].make_message(name=name,
process=int(pid),
signum=9)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
msg = cmds['globaloptions'].make_message()
options = self.client.call(msg)
return options['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
msg = cmds['incr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def decrproc(self, name):
msg = cmds['decr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
msg = cmds['listpids'].make_message(name=name)
res = self.client.call(msg)
return res['pids']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
return res['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
msg = cmds['add'].make_message(name=name, cmd=cmd)
res = self.client.call(msg)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
msg = cmds['set'].make_message(name=name, options=options)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
else:
return False
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = "circus.tests.test_arbiter.run_dummy"
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get("watchers"), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = "%s generic.py %s %s" % (sys.executable, "circus.tests.test_arbiter.run_dummy", testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ["generic.py", "circus.tests.test_arbiter.run_dummy1"]
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get("watchers"), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get("status"), "error")
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True, options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get("options", {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get("processes")
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get("processes")
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get("processes")
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get("processes")
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = "circus.tests.test_arbiter.run_dummy"
plugin = "circus.tests.test_arbiter.Plugin"
plugins = [{"use": plugin, "file": datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2, 3])
# wait a bit
time.sleep(0.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split("\n") if line != ""]
wanted = ["test:spawn", "test:spawn"]
self.assertEqual(data, wanted)
def test_singleton(self):
self._stop_runners()
dummy_process = "circus.tests.test_arbiter.run_dummy"
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message("incr", name="test")
self.assertEqual(res["numprocesses"], 1)
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.support.run_process'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
self.cli.send_message("incr", name="test", nb=2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 4)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable,
'circus.tests.support.run_process',
testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.support.run_process']
return cmd, args
def _get_options(self, **kwargs):
if 'graceful_timeout' not in kwargs:
kwargs['graceful_timeout'] = 4
return kwargs
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True, options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True,
options=self._get_options(flapping_window=100))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
self.assertTrue(poll_for(self.test_file, 'START')) # process started
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
self.assertTrue(poll_for(self.test_file, 'START')) # process started
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.support.run_process'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
def nb_processes():
return len(cli.send_message('list', name='test').get('pids'))
def incr_processes():
return cli.send_message('incr', name='test')
# wait for the plugin to be started
self.assertTrue(poll_for(datafile, 'PLUGIN STARTED'))
cli = CircusClient()
self.assertEqual(nb_processes(), 1)
incr_processes()
self.assertEqual(nb_processes(), 2)
# wait for the plugin to receive the signal
self.assertTrue(poll_for(datafile, 'test:spawn'))
truncate_file(datafile)
incr_processes()
self.assertEqual(nb_processes(), 3)
# wait for the plugin to receive the signal
self.assertTrue(poll_for(datafile, 'test:spawn'))
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.support.run_process'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.arbiter = self.arbiters[-1]
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def pids(self):
return self.call('list', name='test').get('pids')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
pids = self.pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(self.status('signal', name='test', pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = self.pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_unexisting(self):
watcher = self.arbiter.get_watcher("test")
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
to_kill = process.pid
# the process is killed in an unsual way
os.kill(to_kill, signal.SIGSEGV)
# and wait for it to die
pid, status = os.waitpid(to_kill, 0)
# ansure the old process is considered "unexisting"
self.assertEquals(process.status, UNEXISTING)
# this should clean up and create a new process
watcher.reap_and_manage_processes()
# we should have a new process here now
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
# and that one needs to have a new pid.
self.assertNotEqual(process.pid, to_kill)
# and should not be unexisting...
self.assertNotEqual(process.status, UNEXISTING)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_max_age(self):
result = self.call('set', name='test',
options={'max_age': 1, 'max_age_variance': 0})
self.assertEquals(result.get('status'), 'ok')
initial_pids = self.pids()
time.sleep(3.0) # allow process to reach max_age and restart
current_pids = self.pids()
self.assertEqual(len(current_pids), 1)
self.assertNotEqual(initial_pids, current_pids)
class TestCircus(unittest.TestCase):
def setUp(self):
self.arbiters = []
self.files = []
self.dirs = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
for dir in self.dirs:
shutil.rmtree(dir)
self.cli.stop()
def get_tmpdir(self):
dir_ = mkdtemp()
self.dirs.append(dir_)
return dir_
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, "w") as f:
f.write(content)
return file
@classmethod
def _create_circus(cls, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ["generic.py", callable, testfile]
worker = {"cmd": _CMD, "args": args, "working_dir": wdir, "name": "test", "graceful_timeout": 4}
worker.update(kw)
debug = kw.get("debug", False)
if stats:
arbiter = get_arbiter(
[worker],
background=True,
plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
statsd=True,
debug=debug,
statsd_close_outputs=not debug,
)
else:
arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=debug)
arbiter.start()
return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw):
testfile, arbiter = TestCircus._create_circus(callable, plugins, stats, **kw)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class ProcfileWatcher(CircusPlugin):
name = "procfile_watcher"
def __init__(self, *args, **config):
super(ProcfileWatcher, self).__init__(*args, **config)
self.loop_rate = config.get("loop_rate", 3) # in seconds
self.procfile_path = config.get("app_path",
"/home/application/current/Procfile")
self.working_dir = config.get("working_dir",
"/home/application/current")
self.apprc = config.get("apprc", "/home/application/apprc")
self.port = config.get("port", "8888")
self.uid = config.get("uid", "ubuntu")
self.gid = config.get("gid", self.uid)
self.stderr_stream = {
"class": config.get("stderr_stream", "tsuru.stream.Stream")
}
self.stdout_stream = {
"class": config.get("stdout_stream", "tsuru.stream.Stream")
}
file_watcher = FileWatcher(self.procfile_path, self.reload_procfile)
self.period = ioloop.PeriodicCallback(file_watcher,
self.loop_rate * 1000, self.loop)
self.circus_client = CircusClient()
def get_cmd(self, name):
return self.call("get", name=name, keys=["cmd"])["options"]["cmd"]
def handle_init(self):
self.period.start()
def handle_stop(self):
self.period.stop()
def handle_recv(self, data):
pass
def load_envs(self):
env = {"port": self.port, "PORT": self.port}
env.update(common.load_envs(self.apprc))
return env
def add_watcher(self, name, cmd):
env = self.load_envs()
cmd = replace_args(cmd, **env)
stderr_stream = self.stderr_stream.copy()
stdout_stream = self.stdout_stream.copy()
stdout_stream["watcher_name"] = stderr_stream["watcher_name"] = name
options = {
"env": env,
"copy_env": True,
"working_dir": self.working_dir,
"stderr_stream": stderr_stream,
"stdout_stream": stdout_stream,
"uid": self.uid,
"gid": self.gid,
}
self.circus_client.call({
"command": "add",
"properties": {
"cmd": cmd,
"name": name,
"args": [],
"options": options,
"start": True,
"priority": 2,
},
})
def remove_watcher(self, name):
self.call("rm", name=name)
def change_cmd(self, name, cmd):
env = self.load_envs()
cmd = replace_args(cmd, **env)
self.call("set", name=name, options={"cmd": cmd})
def commands(self, procfile):
cmds = self.call("status")["statuses"]
if "tsuru-hooks" in cmds:
del cmds["tsuru-hooks"]
cmds_names = set(
[k for k in cmds.keys() if not k.startswith("plugin:")])
new_cmds = set(procfile.commands.keys())
to_remove = cmds_names.difference(new_cmds)
to_add = new_cmds.difference(cmds_names)
to_change_names = cmds_names.intersection(new_cmds)
to_change = {}
for name in to_change_names:
if self.get_cmd(name) != procfile.commands.get(name):
to_change[name] = procfile.commands[name]
return to_add, to_remove, to_change
def reload_procfile(self):
with open(self.procfile_path) as file:
procfile = Procfile(file.read())
to_add, to_remove, to_change = self.commands(procfile)
for name in to_remove:
self.remove_watcher(name)
for name in to_add:
self.add_watcher(name=name, cmd=procfile.commands[name])
for name, cmd in to_change.items():
self.change_cmd(name=name, cmd=procfile.commands[name])
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.arbiter = self.arbiters[-1]
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def pids(self):
return self.call('list', name='test').get('pids')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
pids = self.pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(
self.status('signal',
name='test',
pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = self.pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_unexisting(self):
watcher = self.arbiter.get_watcher("test")
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
to_kill = process.pid
# the process is killed in an unsual way
os.kill(to_kill, signal.SIGSEGV)
# and wait for it to die
pid, status = os.waitpid(to_kill, 0)
# ansure the old process is considered "unexisting"
self.assertEquals(process.status, UNEXISTING)
# this should clean up and create a new process
watcher.reap_and_manage_processes()
# we should have a new process here now
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
# and that one needs to have a new pid.
self.assertNotEqual(process.pid, to_kill)
# and should not be unexisting...
self.assertNotEqual(process.status, UNEXISTING)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_max_age(self):
result = self.call('set',
name='test',
options={
'max_age': 1,
'max_age_variance': 0
})
self.assertEquals(result.get('status'), 'ok')
initial_pids = self.pids()
time.sleep(3.0) # allow process to reach max_age and restart
current_pids = self.pids()
self.assertEqual(len(current_pids), 1)
self.assertNotEqual(initial_pids, current_pids)
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (sys.executable,
'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy',
testfile]
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('processes')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
class StatsStreamer(object):
def __init__(self, endpoint, pubsub_endoint, stats_endpoint):
self.topic = 'watcher.'
self.ctx = zmq.Context()
self.pubsub_endpoint = pubsub_endoint
self.sub_socket = self.ctx.socket(zmq.SUB)
self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic)
self.sub_socket.connect(self.pubsub_endpoint)
self.loop = ioloop.IOLoop()
self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop)
self.substream.on_recv(self.handle_recv)
self.client = CircusClient(context=self.ctx, endpoint=endpoint)
self.cmds = get_commands()
self.watchers = defaultdict(list)
self._pids = defaultdict(list)
self.running = False
self.stopped = False
self.lock = threading.RLock()
self.results = Queue.Queue()
self.stats = StatsCollector(self)
self.publisher = StatsPublisher(self, stats_endpoint, context=self.ctx)
def get_watchers(self):
return self._pids.keys()
def get_pids(self, watcher=None):
if watcher is not None:
return self._pids[watcher]
return chain(self._pid.values())
def get_circus_pids(self):
# getting the circusd pid
msg = self.cmds['dstats'].make_message()
res = self.client.call(msg)
return [('circusd-stats', os.getpid()),
('circusd', res['info']['pid'])]
def _init(self):
with self.lock:
self.stopped = False
self._pids.clear()
# getting the initial list of watchers/pids
msg = self.cmds['list'].make_message()
res = self.client.call(msg)
for watcher in res['watchers']:
msg = self.cmds['listpids'].make_message(name=watcher)
res = self.client.call(msg)
for pid in res['pids']:
if pid in self._pids[watcher]:
continue
self._pids[watcher].append(pid)
def remove_pid(self, watcher, pid):
logger.debug('Removing %d from %s' % (pid, watcher))
if pid in self._pids[watcher]:
with self.lock:
self._pids[watcher].remove(pid)
def append_pid(self, watcher, pid):
logger.debug('Adding %d in %s' % (pid, watcher))
if pid in self._pids[watcher]:
return
with self.lock:
self._pids[watcher].append(pid)
def start(self):
logger.info('Starting the stats streamer')
self._init()
logger.debug('Initial list is ' + str(self._pids))
self.running = True
self.stats.start()
self.publisher.start()
logger.debug('Now looping to get circusd events')
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
self.sub_socket.close()
def handle_recv(self, data):
topic, msg = data
try:
__, watcher, action = topic.split('.')
msg = json.loads(msg)
if action != 'start' and self.stopped:
self._init()
if action in ('reap', 'kill'):
# a process was reaped
pid = msg['process_pid']
self.remove_pid(watcher, pid)
elif action == 'spawn':
pid = msg['process_pid']
self.append_pid(watcher, pid)
elif action == 'start':
self._init()
elif action == 'stop':
# nothing to do
self.stopped = True
else:
logger.debug('Unknown action: %r' % action)
logger.debug(msg)
except Exception:
logger.exception('Failed to treat %r' % msg)
def stop(self):
self.running = False
self.publisher.stop()
self.stats.stop()
self.ctx.destroy(0)
logger.info('Stats streamer stopped')
class Client(object):
def __init__(self, host, port, timeout=15):
assert type(host) == str
assert type(port) == int and port >= 0 and port <= 65535
assert type(timeout) == int and timeout > 0
self._host = host
self._port = port
self._timeout = timeout
self._arbiter = get_arbiter([])
self._arbiter.start()
self._client = CircusClient(timeout=self._timeout,
endpoint='tcp://{0}:{1}'.format(
self._host, self._port))
"""
Add a watcher:
This command add a watcher dynamically to a arbiter
"""
def add_watcher(self, name, command, args=[], autostart=False):
assert type(name) == str
assert type(command) == str
assert type(args) == list
assert type(autostart) == bool
addWatcher_command = Dict()
addWatcher_command.command = 'add'
addWatcher_command.properties.cmd = command
addWatcher_command.properties.name = name
addWatcher_command.properties.args = args
addWatcher_command.properties.start = autostart
response = self._client.call(addWatcher_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Reload the arbiter or a watcher:
This command reloads all the process in a watcher or all watchers
"""
def reload(self,
watcher='',
graceful=True,
sequential=False,
waiting=False):
assert type(watcher) == str
assert type(graceful) == bool
assert type(sequential) == bool
assert type(waiting) == bool
reload_command = Dict()
reload_command.command = 'reload'
reload_command.properties.name = watcher
reload_command.properties.graceful = graceful
reload_command.properties.sequential = sequential
reload_command.properties.waiting = waiting
response = self._client.call(reload_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Restart the arbiter or a watcher:
This command restart all the process in a watcher or all watchers
"""
def restart(self, watcher='', waiting=False):
assert type(watcher) == str
assert type(waiting) == bool
restart_command = Dict()
restart_command.command = 'restart'
restart_command.properties.name = watcher
restart_command.properties.waiting = waiting
response = self._client.call(restart_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Remove a watcher:
This command removes a watcher dynamically from the arbiter
"""
def rm_watcher(self, watcher, nonstop=False, waiting=False):
assert type(watcher) == str
assert type(nonstop) == bool
assert type(waiting) == bool
rm_command = Dict()
rm_command.command = 'rm'
rm_command.properties.name = watcher
rm_command.properties.nonstop = nonstop
rm_command.properties.waiting = waiting
response = self._client.call(rm_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get the number of watchers:
Get the number of watchers in a arbiter
"""
def num_watchers(self):
numwatchers_command = Dict()
numwatchers_command.command = 'numwatchers'
response = self._client.call(numwatchers_command)
status = response.get('status', None)
if status and status == 'ok':
return response.get('numwatchers', 0)
return 0
"""
Get list of watchers or processes in a watcher
"""
# TODO pids not being shown when using a watcher
def list(self, watcher=''):
assert type(watcher) == str
list_command = Dict()
list_command.command = 'list'
if watcher:
list_command.properties.name = watcher
response = self._client.call(list_command)
return [int(pid) for pid in response['pids']
] if response['status'] == u'ok' else []
else:
response = self._client.call(list_command)
return [str(w) for w in response['watchers']
] if response['status'] == u'ok' else []
"""
Start the arbiter or a watcher:
This command starts all the processes in a watcher or all watchers.
"""
def start(self, watcher='', waiting=False):
assert type(watcher) == str
assert type(waiting) == bool
start_command = Dict()
start_command.command = 'start'
start_command.properties.name = watcher
start_command.properties.waiting = waiting
response = self._client.call(start_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Stop watchers:
This command stops a given watcher or all watchers.
"""
def stop(self, watcher='', waiting=False):
assert type(watcher) == str
assert type(waiting) == bool
stop_command = Dict()
stop_command.command = 'stop'
stop_command.properties.name = watcher
stop_command.properties.waiting = waiting
response = self._client.call(stop_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get the number of processes:
Get the number of processes in a watcher or in a arbiter
"""
def num_processes(self, watcher):
assert type(watcher) == str
num_command = Dict()
num_command.command = 'numprocesses'
num_command.properties.name = watcher
response = self._client.call(num_command)
status = response.get('status', None)
if status and status == 'ok':
return response.get('numprocesses', 0)
return 0
"""
Quit the arbiter immediately:
When the arbiter receive this command, the arbiter exit.
"""
def quit(self, waiting=False):
assert type(waiting) == bool
quit_command = Dict()
quit_command.command = 'quit'
quit_command.waiting = waiting
response = self._client.call(quit_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get circusd stats:
You can get at any time some statistics about circus with the dstat command.
"""
def dstats(self):
stats_command = Dict()
stats_command.command = 'dstats'
response = self._client.call(stats_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('info', None):
return literal_eval(str(response['info']))
return {}
"""
Get the status of a watcher or all watchers:
This command start get the status of a watcher or all watchers
"""
def status(self, watcher=''):
assert type(watcher) == str
status_command = Dict()
status_command.command = 'status'
if watcher:
status_command.properties.name = watcher
response = self._client.call(status_command)
if response.get('status', None):
return str(response['status'])
else:
response = self._client.call(status_command)
if response.get('statuses', None):
s = []
watchers = literal_eval(str(response[u'statuses']))
for w in response['statuses']:
s.append({'name': str(w), 'status': str(watchers[w])})
return s
return None
"""
Reload the configuration file:
This command reloads the configuration file, so changes in the
configuration file will be reflected in the configuration of
circus.
"""
def reload_configuration(self, waiting=False):
assert type(waiting) == bool
reload_command = Dict()
reload_command.command = 'reloadconfig'
reload_command.properties.waiting = waiting
response = self._client.call(reload_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
""""
Send a signal:
This command allows you to send a signal to all processes in a watcher,
a specific process in a watcher or its children.
"""
def send_signal(self,
watcher,
signum,
pid=0,
children=False,
childpid=0,
recursive=False):
assert type(watcher) == str
assert type(signum) == int and signum > 0 and signum < 32
assert type(pid) == int and pid >= 0
assert type(childpid) == int and childpid >= 0
assert type(children) == bool
assert type(recursive) == bool
signal_command = Dict()
signal_command.command = 'signal'
signal_command.properties.name = watcher
signal_command.properties.signum = signum
signal_command.properties.pid = pid if pid > 0 else ''
signal_command.properties.children = children
signal_command.properties.childpid = childpid if childpid > 0 else ''
signal_command.properties.recursive = recursive
signal_command.prune()
response = self._client.call(signal_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Set a watcher option
"""
def set(self, watcher, options=[], waiting=False):
assert type(watcher) == str
assert type(options) == list
assert type(waiting) == bool
for option in options:
assert type(option) == tuple
set_command = Dict()
set_command.command = 'set'
set_command.properties.name = watcher
for option in options:
set_command.properties.options[option[0]] = option[1]
response = self._client.call(set_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get process infos:
You can get at any time some statistics about your processes
with the stat command.
"""
def stats(self, watcher, process='', extended=False):
assert type(watcher) == str
assert type(process) == str
assert type(extended) == bool
stats_command = Dict()
stats_command.command = 'stats'
stats_command.properties.name = watcher
if process:
stats_command.properties.process = process
stats_command.properties.extended = extended
response = self._client.call(stats_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('info', None):
return literal_eval(str(response['info']))
return {}
"""
Get the value of all options for a watcher:
This command returns all option values for a given watcher.
"""
def options(self, watcher):
assert type(watcher) == str
options_command = Dict()
options_command.command = 'options'
options_command.properties.name = watcher
response = self._client.call(options_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('options', None):
return literal_eval(str(response['options']))
return {}
"""
Increment the number of processes in a watcher:
This comment increment the number of processes in a watcher by num.
"""
def inc(self, watcher, num=1, waiting=False):
assert type(watcher) == str
assert type(num) == int and num > 0
assert type(waiting) == bool
inc_command = Dict()
inc_command.command = 'incr'
inc_command.properties.name = watcher
inc_command.properties.nb = num
inc_command.properties.waiting = waiting
response = self._client.call(inc_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('numprocesses', None):
return response['numprocesses']
return 0
"""
Decrement the number of processes in a watcher:
This comment decrement the number of processes in a watcher by num
"""
def decr(self, watcher, num=1, waiting=False):
assert type(watcher) == str
assert type(num) == int and num > 0
assert type(waiting) == bool
decr_command = Dict()
decr_command.command = 'decr'
decr_command.properties.name = watcher
decr_command.properties.nb = num
decr_command.properties.waiting = waiting
response = self._client.call(decr_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('numprocesses', None):
return response['numprocesses']
return 0
"""
Get the arbiter options:
This command return the arbiter options
"""
def global_options(self, options=[]):
assert type(options) == list
global_options_command = Dict()
global_options_command.command = 'globaloptions'
response = self._client.call(global_options_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('options', None):
_options = literal_eval(str(response['options']))
if options:
selected_options = Dict()
for option in options:
if option in ("endpoint", "pubsub_endpoint",
"check_delay", "multicast_endpoint"):
selected_options[option] = _options[option]
return selected_options
else:
return _options
return {}
"""
Get the value of specific watcher options:
This command can be used to query the current value of one or more watcher options.
"""
def get(self, watcher, options=[]):
assert type(watcher) == str
assert type(options) == list and options
get_command = Dict()
get_command.command = 'get'
get_command.properties.name = watcher
get_command.properties['keys'] = options
response = self._client.call(get_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('options', None):
return literal_eval(str(response['options']))
return {}
from circus.client import CircusClient
from circus.util import DEFAULT_ENDPOINT_DEALER
client = CircusClient(endpoint=DEFAULT_ENDPOINT_DEALER)
command = '../bin/python dummy_fly.py 111'
name = 'dummy'
for i in range(50):
print(client.call("""
{
"command": "add",
"properties": {
"cmd": "%s",
"name": "%s",
"options": {
"copy_env": true,
"stdout_stream": {
"filename": "stdout.log"
},
"stderr_stream": {
"filename": "stderr.log"
}
},
"start": true
}
}
""" % (command, name + str(i))))
class ProcfileWatcher(CircusPlugin):
name = "procfile_watcher"
def __init__(self, *args, **config):
super(ProcfileWatcher, self).__init__(*args, **config)
self.loop_rate = config.get("loop_rate", 60) # in seconds
self.procfile_path = config.get("app_path", "/home/application/current/Procfile")
self.working_dir = config.get("working_dir", "/home/application/current")
self.port = config.get("port", "8888")
self.uid = config.get("uid", "ubuntu")
self.stderr_stream = {"class": config.get("stderr_stream", "tsuru.stream.Stream")}
self.stdout_stream = {"class": config.get("stdout_stream", "tsuru.stream.Stream")}
self.period = ioloop.PeriodicCallback(self.look_after, self.loop_rate * 1000, self.loop)
self.circus_client = CircusClient()
def handle_init(self):
self.period.start()
def handle_stop(self):
self.period.stop()
def handle_recv(self, data):
pass
def add_watcher(self, name, cmd):
options = {
"env": {"port": self.port},
"copy_env": True,
"working_dir": self.working_dir,
"stderr_stream": self.stderr_stream,
"stdout_stream": self.stdout_stream,
"uid": self.uid,
}
self.circus_client.call(json.dumps({
"command": "add",
"properties": {
"cmd": cmd,
"name": name,
"args": [],
"options": options,
"start": True,
}}))
def remove_watcher(self, name):
self.call("rm", name=name)
def commands(self, procfile):
cmds = set(self.call("status")["statuses"].keys())
new_cmds = set(procfile.commands.keys())
to_remove = cmds.difference(new_cmds)
to_add = new_cmds.difference(cmds)
return to_add, to_remove
def look_after(self):
if os.path.exists(self.procfile_path):
with open(self.procfile_path) as file:
procfile = Procfile(file.read())
to_add, to_remove = self.commands(procfile)
for name in to_remove:
self.remove_watcher(name)
for name in to_add:
self.add_watcher(name=name, cmd=procfile.commands[name])
class MetWorkCircusClient(object):
def __init__(self, timeout=10):
self.module = os.environ["MFMODULE"]
self.endpoint = os.environ["%s_CIRCUS_ENDPOINT" % self.module]
self.timeout = timeout
self.client = CircusClient(endpoint=self.endpoint,
timeout=self.timeout)
def check(self):
if not os.path.exists(self.endpoint.replace('ipc://', '')):
return False
tmp = self.cmd("globaloptions")
if tmp is None or (tmp.get('status', None) != 'ok'):
return False
return True
def wait(self, timeout=10, cli_display=True):
with self._mfprogress(cli_display=cli_display) as progress:
txt = "- Waiting for circus daemon..."
before = datetime.datetime.now()
after = datetime.datetime.now()
t = progress.add_task(txt, total=timeout)
while (after - before).total_seconds() < timeout:
if self.check():
progress.complete_task(t)
return True
time.sleep(1)
after = datetime.datetime.now()
delta = (after - before).total_seconds()
progress.update(t, completed=int(delta))
progress.complete_task_nok(t)
return False
def _cmd(self, cmd, **properties):
reply = self.client.call({"command": cmd, "properties": properties})
status = "nok"
try:
status = reply["status"]
except Exception:
pass
if status != "ok":
try:
if "arbiter is already running" in reply['reason']:
return {"status": "already_running"}
except Exception:
pass
return None
return reply
def cmd(self, cmd, **properties):
before = datetime.datetime.now()
while ((datetime.datetime.now() - before).total_seconds()) < 10:
try:
tmp = self._cmd(cmd, **properties)
except CallError:
return None
if tmp is None or (tmp.get('status', None) != "already_running"):
return tmp
time.sleep(0.2)
def list_watchers(self):
tmp = self.cmd("list")
if tmp is None:
return None
return tmp["watchers"]
def list_watchers_by_plugin_pattern(self, plugin_name_pattern):
plugin_key = self.module.lower() + "_plugin"
watchers = self.list_watchers()
res = []
if watchers is None:
return res
for watcher in watchers:
options = self.options(name=watcher)
if plugin_key not in options:
continue
if fnmatch.fnmatch(options[plugin_key], plugin_name_pattern):
res.append(watcher)
return res
def stop_watcher(self,
cli_display=True,
indent=0,
timeout=10,
**properties):
name = properties["name"]
with self._mfprogress(cli_display=cli_display) as progress:
txt = " " * indent + "- Scheduling stop of %s" % name
t = progress.add_task(txt, total=timeout)
statuses = self.statuses()
if name not in statuses:
progress.complete_task_warning(t)
return None
if statuses[name] == "stopped":
progress.complete_task_warning(t, "already stopped")
return "stopped"
if statuses[name] == "stopping":
progress.complete_task_warning(t, "already stopping")
return "stopping"
before = datetime.datetime.now()
after = datetime.datetime.now()
while (after - before).total_seconds() < timeout:
self.cmd("stop", **properties)
statuses = self.statuses()
if name not in statuses or statuses[name] in ("stopping",
"stopped"):
progress.complete_task(t)
return statuses[name]
time.sleep(1)
after = datetime.datetime.now()
delta = (after - before).total_seconds()
progress.update(t, completed=int(delta))
progress.complete_task_nok(t)
return "failed"
def start_watcher(self,
cli_display=True,
indent=0,
timeout=10,
**properties):
name = properties["name"]
with self._mfprogress(cli_display=cli_display) as progress:
txt = " " * indent + "- Starting of %s" % name
t = progress.add_task(txt, total=timeout)
statuses = self.statuses()
if name not in statuses:
progress.complete_task_nok(t)
return None
if statuses[name] == "starting":
progress.complete_task_warning(t, "already starting")
return "starting"
if statuses[name] == "active":
progress.complete_task_warning(t, "already started")
return "active"
before = datetime.datetime.now()
after = datetime.datetime.now()
while (after - before).total_seconds() < timeout:
tmp = self.cmd("start", **properties)
status = tmp.get("status", None) if tmp is not None else None
if status == "ok":
progress.complete_task(t)
return status
time.sleep(1)
after = datetime.datetime.now()
delta = (after - before).total_seconds()
progress.update(t, completed=int(delta))
progress.complete_task_nok(t)
return "failed"
def start_watchers(self, cli_display=True, indent=0, **properties):
names = properties["names"]
for name in names:
self.start_watcher(cli_display=cli_display,
indent=indent,
name=name,
**properties)
def wait_watcher_started(self,
cli_display=True,
timeout=20,
indent=0,
**properties):
name = properties["name"]
with self._mfprogress(cli_display=cli_display) as progress:
txt = " " * indent + "- Waiting for start of %s..." % name
t = progress.add_task(txt, total=timeout)
before = datetime.datetime.now()
after = datetime.datetime.now()
while (after - before).total_seconds() < timeout:
statuses = self.statuses()
if name in statuses:
if statuses[name] == "active":
progress.complete_task(t)
return True
time.sleep(1)
after = datetime.datetime.now()
delta = (after - before).total_seconds()
progress.update(t, completed=int(delta))
progress.complete_task_nok(t)
return False
def _mfprogress(self, cli_display=True):
console = None
if not cli_display:
console = rich.console.Console(file=open(os.devnull, "w"))
return MFProgress(console=console)
def wait_watcher_stopped(self,
cli_display=True,
timeout=None,
indent=0,
**properties):
name = properties["name"]
if timeout is None:
timeout = 300
try:
timeout = int(self.options(name=name)['graceful_timeout'])
except Exception:
pass
with self._mfprogress(cli_display=cli_display) as progress:
txt = " " * indent + "- Waiting for stop of %s..." % name
t = progress.add_task(txt, total=timeout)
before = datetime.datetime.now()
after = datetime.datetime.now()
while (after - before).total_seconds() < timeout:
statuses = self.statuses()
if name not in statuses or statuses[name] == "stopped":
progress.complete_task(t)
return True
time.sleep(1)
after = datetime.datetime.now()
delta = (after - before).total_seconds()
progress.update(t, completed=int(delta))
progress.complete_task_nok(t)
return False
def statuses(self):
tmp = self.cmd("status")
if tmp is None:
return None
return tmp["statuses"]
def list_pids(self, **properties):
tmp = self.cmd("list", **properties)
if tmp is None:
return None
return tmp["pids"]
def options(self, **properties):
tmp = self.cmd("options", **properties)
if tmp is None:
return None
return tmp["options"]
class TestCircus(unittest.TestCase):
def setUp(self):
self.arbiters = []
self.files = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
self.cli.stop()
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, 'w') as f:
f.write(content)
return file
def _run_circus(self, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ['generic.py', callable, testfile]
worker = {
'cmd': _CMD,
'args': args,
'working_dir': wdir,
'name': 'test',
'graceful_timeout': 4
}
worker.update(kw)
if stats:
arbiter = get_arbiter([worker],
background=True,
plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
debug=kw.get('debug', False))
else:
arbiter = get_arbiter([worker],
background=True,
plugins=plugins,
debug=kw.get('debug', False))
arbiter.start()
time.sleep(.3)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class TestCircus(unittest.TestCase):
arbiter_factory = get_arbiter
def setUp(self):
self.arbiters = []
self.files = []
self.dirs = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
for dir in self.dirs:
shutil.rmtree(dir)
self.cli.stop()
def get_tmpdir(self):
dir_ = mkdtemp()
self.dirs.append(dir_)
return dir_
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, 'w') as f:
f.write(content)
return file
@classmethod
def _create_circus(cls, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ['generic.py', callable, testfile]
worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir,
'name': 'test', 'graceful_timeout': 2}
worker.update(kw)
debug = kw.get('debug', False)
fact = cls.arbiter_factory
if stats:
arbiter = fact([worker], background=True, plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
statsd=True,
debug=debug, statsd_close_outputs=not debug)
else:
arbiter = fact([worker], background=True, plugins=plugins,
debug=debug)
arbiter.start()
return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw):
testfile, arbiter = TestCircus._create_circus(callable, plugins, stats,
**kw)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher == 'circusd-stats':
continue
options = self.client.send_message('options', name=watcher)
self.watchers.append((watcher, options['options']))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
except CallError:
self.connected = False
def killproc(self, name, pid):
res = self.client.send_message('signal',
name=name,
process=int(pid),
signum=9)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class CircusMgr(object):
"""Fermentrack Circus Handler, It is a simple wrapper around
circus client, any errors raised as CircusException"""
def __init__(self,
connection_timeout=2,
circus_endpoint=DEFAULT_ENDPOINT_DEALER):
self._client = CircusClient(timeout=connection_timeout,
endpoint=circus_endpoint)
def _call(self, command, **props):
message = {"command": command, "properties": props or {}}
try:
res = self._client.call(message)
except (CallError) as callerr:
LOG.debug("Error from circus", exc_info=True)
raise CircusException(
"Could not send message to circus: {}".format(callerr))
if res['status'] == u'error':
raise CircusException("Error: {}".format(res['reason']))
return res
def signal(self, name, signal=9):
"""Send signal to process, signal defaults to 9 (SIGTERM)"""
self._call("signal", name=name, signal=signal)
def reload(self, name, waiting=False, graceful=True, sequential=False):
"""Reload the arbiter/watcher
If ``waiting`` is False (default), the call will return immediately
after calling ``reload`` process.
"""
response = self._call("reload",
name=name,
graceful=graceful,
sequential=sequential,
waiting=waiting)
return True if response['status'] == u'ok' else False
def start(self, name, waiting=False):
"""Start circus process that has been stopped
If ``waiting`` is False (default), the call will return immediately
after calling ``start`` process.
"""
response = self._call("start", name=name, waiting=waiting)
return True if response['status'] == u'ok' else False
def restart(self, name=None):
"""Restart a or all circus process(es)
If ``name`` is None all processes under circus will be restarted
"""
if name:
response = self._call("restart", name=name)
else:
response = self._call("restart")
return True if response['status'] == u'ok' else False
def stop(self, name, waiting=False):
"""Stop a circus process, like suspend, the processess is stopped but still
in circus, to resume use ``start``
If ``waiting`` is False (default), the call will return immediately
after calling ``stop`` process.
"""
response = self._call("stop", name=name, waiting=waiting)
return True if response['status'] == u'ok' else False
def add_controller(self, cmd, name, logpath):
"""Add a new brewpi controller script"""
response = self._call("add",
cmd=cmd,
name=name,
start=True,
options={
"copy_env": True,
"stdout_stream": {
"class":
"FileStream",
"filename":
u"%s/%s-stdout.log" % (logpath, name),
"max_bytes":
2097152,
"backup_count":
2,
},
"stderr_stream": {
"class":
"FileStream",
"filename":
u"%s/%s-stderr.log" % (logpath, name),
"max_bytes":
2097152,
"backup_count":
2,
}
})
return response
def remove(self, name):
"""Stop and Remove ``name`` from circus fully"""
response = self._call("rm", name=name)
return response
def get_applications(self, verbose=False):
"""Get currently running processes
If ``verbose`` is False a simple list will be returned
if True circus information will be included.
"""
response = self._call("list")
if verbose:
return response
return response.get("watchers")
def application_status(self, name, verbose=False):
"""Get process status
If ``verbose`` is False it will return status, or "not running"
if True circus information will be included.
"""
response = self._call("status", name=name)
if verbose:
return response
return str(response['status']
) if response['status'] != u'error' else 'not running'
def quit_circus(self):
"""quit_circus will quit the circus daemon, and need to be started again
by some other means
"""
response = self._call("quit")
return response
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
self.cli.send_message("incr", name="test", nb=2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 4)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable, 'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def _get_options(self, **kwargs):
if 'graceful_timeout' not in kwargs:
kwargs['graceful_timeout'] = 4
return kwargs
def test_add_watcher(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options=self._get_options(flapping_window=100))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
def nb_processes():
return len(cli.send_message('list', name='test').get('pids'))
def incr_processes():
return cli.send_message('incr', name='test')
cli = CircusClient()
self.assertEqual(nb_processes(), 1)
incr_processes()
self.assertEqual(nb_processes(), 2)
incr_processes()
self.assertEqual(nb_processes(), 3)
# wait a bit
time.sleep(.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split('\n') if line != '']
wanted = ['test:spawn', 'test:spawn']
self.assertEqual(data, wanted)
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
class Client(object):
def __init__(self, host, port, timeout=15):
assert type(host) == str
assert type(port) == int and port >= 0 and port <= 65535
assert type(timeout) == int and timeout > 0
self._host = host
self._port = port
self._timeout = timeout
self._arbiter = get_arbiter([])
self._arbiter.start()
self._client = CircusClient(timeout=self._timeout,
endpoint='tcp://{0}:{1}'.format(self._host,
self._port))
"""
Add a watcher:
This command add a watcher dynamically to a arbiter
"""
def add_watcher(self, name, command, args=[], autostart=False):
assert type(name) == str
assert type(command) == str
assert type(args) == list
assert type(autostart) == bool
addWatcher_command = Dict()
addWatcher_command.command = 'add'
addWatcher_command.properties.cmd = command
addWatcher_command.properties.name = name
addWatcher_command.properties.args = args
addWatcher_command.properties.start = autostart
response = self._client.call(addWatcher_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Reload the arbiter or a watcher:
This command reloads all the process in a watcher or all watchers
"""
def reload(self, watcher='', graceful=True, sequential=False, waiting=False):
assert type(watcher) == str
assert type(graceful) == bool
assert type(sequential) == bool
assert type(waiting) == bool
reload_command = Dict()
reload_command.command = 'reload'
reload_command.properties.name = watcher
reload_command.properties.graceful = graceful
reload_command.properties.sequential = sequential
reload_command.properties.waiting = waiting
response = self._client.call(reload_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Restart the arbiter or a watcher:
This command restart all the process in a watcher or all watchers
"""
def restart(self, watcher='', waiting=False):
assert type(watcher) == str
assert type(waiting) == bool
restart_command = Dict()
restart_command.command = 'restart'
restart_command.properties.name = watcher
restart_command.properties.waiting = waiting
response = self._client.call(restart_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Remove a watcher:
This command removes a watcher dynamically from the arbiter
"""
def rm_watcher(self, watcher, nonstop=False, waiting=False):
assert type(watcher) == str
assert type(nonstop) == bool
assert type(waiting) == bool
rm_command = Dict()
rm_command.command = 'rm'
rm_command.properties.name = watcher
rm_command.properties.nonstop = nonstop
rm_command.properties.waiting = waiting
response = self._client.call(rm_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get the number of watchers:
Get the number of watchers in a arbiter
"""
def num_watchers(self):
numwatchers_command = Dict()
numwatchers_command.command = 'numwatchers'
response = self._client.call(numwatchers_command)
status = response.get('status', None)
if status and status == 'ok':
return response.get('numwatchers', 0)
return 0
"""
Get list of watchers or processes in a watcher
"""
# TODO pids not being shown when using a watcher
def list(self, watcher=''):
assert type(watcher) == str
list_command = Dict()
list_command.command = 'list'
if watcher:
list_command.properties.name = watcher
response = self._client.call(list_command)
return [int(pid)
for pid
in response['pids']] if response['status'] == u'ok' else []
else:
response = self._client.call(list_command)
return [str(w)
for w
in response['watchers']] if response['status'] == u'ok' else []
"""
Start the arbiter or a watcher:
This command starts all the processes in a watcher or all watchers.
"""
def start(self, watcher='', waiting=False):
assert type(watcher) == str
assert type(waiting) == bool
start_command = Dict()
start_command.command = 'start'
start_command.properties.name = watcher
start_command.properties.waiting = waiting
response = self._client.call(start_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Stop watchers:
This command stops a given watcher or all watchers.
"""
def stop(self, watcher='', waiting=False):
assert type(watcher) == str
assert type(waiting) == bool
stop_command = Dict()
stop_command.command = 'stop'
stop_command.properties.name = watcher
stop_command.properties.waiting = waiting
response = self._client.call(stop_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get the number of processes:
Get the number of processes in a watcher or in a arbiter
"""
def num_processes(self, watcher):
assert type(watcher) == str
num_command = Dict()
num_command.command = 'numprocesses'
num_command.properties.name = watcher
response = self._client.call(num_command)
status = response.get('status', None)
if status and status == 'ok':
return response.get('numprocesses', 0)
return 0
"""
Quit the arbiter immediately:
When the arbiter receive this command, the arbiter exit.
"""
def quit(self, waiting=False):
assert type(waiting) == bool
quit_command = Dict()
quit_command.command = 'quit'
quit_command.waiting = waiting
response = self._client.call(quit_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get circusd stats:
You can get at any time some statistics about circus with the dstat command.
"""
def dstats(self):
stats_command = Dict()
stats_command.command = 'dstats'
response = self._client.call(stats_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('info', None):
return literal_eval(str(response['info']))
return {}
"""
Get the status of a watcher or all watchers:
This command start get the status of a watcher or all watchers
"""
def status(self, watcher=''):
assert type(watcher) == str
status_command = Dict()
status_command.command = 'status'
if watcher:
status_command.properties.name = watcher
response = self._client.call(status_command)
if response.get('status', None):
return str(response['status'])
else:
response = self._client.call(status_command)
if response.get('statuses', None):
s = []
watchers = literal_eval(str(response[u'statuses']))
for w in response['statuses']:
s.append({'name':str(w), 'status':str(watchers[w])})
return s
return None
"""
Reload the configuration file:
This command reloads the configuration file, so changes in the
configuration file will be reflected in the configuration of
circus.
"""
def reload_configuration(self, waiting=False):
assert type(waiting) == bool
reload_command = Dict()
reload_command.command = 'reloadconfig'
reload_command.properties.waiting = waiting
response = self._client.call(reload_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
""""
Send a signal:
This command allows you to send a signal to all processes in a watcher,
a specific process in a watcher or its children.
"""
def send_signal(self, watcher, signum, pid=0, children=False, childpid=0, recursive=False):
assert type(watcher) == str
assert type(signum) == int and signum > 0 and signum < 32
assert type(pid) == int and pid >= 0
assert type(childpid) == int and childpid >= 0
assert type(children) == bool
assert type(recursive) == bool
signal_command = Dict()
signal_command.command = 'signal'
signal_command.properties.name = watcher
signal_command.properties.signum = signum
signal_command.properties.pid = pid if pid > 0 else ''
signal_command.properties.children = children
signal_command.properties.childpid = childpid if childpid > 0 else ''
signal_command.properties.recursive = recursive
signal_command.prune()
response = self._client.call(signal_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Set a watcher option
"""
def set(self, watcher, options=[], waiting=False):
assert type(watcher) == str
assert type(options) == list
assert type(waiting) == bool
for option in options:
assert type(option) == tuple
set_command = Dict()
set_command.command = 'set'
set_command.properties.name = watcher
for option in options:
set_command.properties.options[option[0]] = option[1]
response = self._client.call(set_command)
response = response.get('status', None)
if response and response == 'ok':
return True
return False
"""
Get process infos:
You can get at any time some statistics about your processes
with the stat command.
"""
def stats(self, watcher, process='', extended=False):
assert type(watcher) == str
assert type(process) == str
assert type(extended) == bool
stats_command = Dict()
stats_command.command = 'stats'
stats_command.properties.name = watcher
if process:
stats_command.properties.process = process
stats_command.properties.extended = extended
response = self._client.call(stats_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('info', None):
return literal_eval(str(response['info']))
return {}
"""
Get the value of all options for a watcher:
This command returns all option values for a given watcher.
"""
def options(self, watcher):
assert type(watcher) == str
options_command = Dict()
options_command.command = 'options'
options_command.properties.name = watcher
response = self._client.call(options_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('options', None):
return literal_eval(str(response['options']))
return {}
"""
Increment the number of processes in a watcher:
This comment increment the number of processes in a watcher by num.
"""
def inc(self, watcher, num=1, waiting=False):
assert type(watcher) == str
assert type(num) == int and num > 0
assert type(waiting) == bool
inc_command = Dict()
inc_command.command = 'incr'
inc_command.properties.name = watcher
inc_command.properties.nb = num
inc_command.properties.waiting = waiting
response = self._client.call(inc_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('numprocesses', None):
return response['numprocesses']
return 0
"""
Decrement the number of processes in a watcher:
This comment decrement the number of processes in a watcher by num
"""
def decr(self, watcher, num=1, waiting=False):
assert type(watcher) == str
assert type(num) == int and num > 0
assert type(waiting) == bool
decr_command = Dict()
decr_command.command = 'decr'
decr_command.properties.name = watcher
decr_command.properties.nb = num
decr_command.properties.waiting = waiting
response = self._client.call(decr_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('numprocesses', None):
return response['numprocesses']
return 0
"""
Get the arbiter options:
This command return the arbiter options
"""
def global_options(self, options=[]):
assert type(options) == list
global_options_command = Dict()
global_options_command.command = 'globaloptions'
response = self._client.call(global_options_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('options', None):
_options = literal_eval(str(response['options']))
if options:
selected_options = Dict()
for option in options:
if option in ("endpoint", "pubsub_endpoint", "check_delay", "multicast_endpoint"):
selected_options[option] = _options[option]
return selected_options
else:
return _options
return {}
"""
Get the value of specific watcher options:
This command can be used to query the current value of one or more watcher options.
"""
def get(self, watcher, options=[]):
assert type(watcher) == str
assert type(options) == list and options
get_command = Dict()
get_command.command = 'get'
get_command.properties.name = watcher
get_command.properties['keys'] = options
response = self._client.call(get_command)
status = response.get('status', None)
if status and status == 'ok':
if response.get('options', None):
return literal_eval(str(response['options']))
return {}
class LiveClient(object):
def __init__(self, endpoint, ssh_server=None):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint,
ssh_server=ssh_server)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
self.sockets = None
self.use_sockets = False
self.embed_httpd = False
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
self.plugins = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher in ('circusd-stats', 'circushttpd'):
if watcher == 'circushttpd':
self.embed_httpd = True
continue
options = self.client.send_message('options',
name=watcher)['options']
self.watchers.append((watcher, options))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
if not self.use_sockets and options.get('use_sockets', False):
self.use_sockets = True
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
if self.endpoint.startswith('tcp://'):
# In case of multi interface binding i.e: tcp://0.0.0.0:5557
anyaddr = '0.0.0.0'
ip = self.endpoint.lstrip('tcp://').split(':')[0]
self.stats_endpoint = self.stats_endpoint.replace(anyaddr, ip)
except CallError:
self.connected = False
def killproc(self, name, pid):
# killing a proc and its children
res = self.client.send_message('signal', name=name, pid=int(pid),
signum=9, recursive=True)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_sockets(self, force_reload=False):
if not self.sockets or force_reload:
res = self.client.send_message('listsockets')
self.sockets = res['sockets']
return self.sockets
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
self.sockets = None
self.use_sockets = False
self.embed_httpd = False
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher in ('circusd-stats', 'circushttpd'):
if watcher == 'circushttpd':
self.embed_httpd = True
continue
options = self.client.send_message('options',
name=watcher)['options']
self.watchers.append((watcher, options))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
if not self.use_sockets and options.get('use_sockets', False):
self.use_sockets = True
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
except CallError:
self.connected = False
def killproc(self, name, pid):
res = self.client.send_message('signal', name=name, pid=int(pid),
signum=9, children=True)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_sockets(self, force_reload=False):
if not self.sockets or force_reload:
res = self.client.send_message('listsockets')
self.sockets = res['sockets']
return self.sockets
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class Client(object):
def __init__(self, port=6000):
self._client = CircusClient(endpoint='tcp://127.0.0.1:{}'.format(port))
def startup(self):
click.echo('startup')
sp.call(['circusd', 'basic.ini', '--daemon'])
def quit(self):
self._client.call({
'command': 'quit',
'properties': {}
})
def stop(self):
self._client.stop()
def start_aiida_daemon(self, profile):
if not get_daemon_name(profile) in self.list_watchers():
# ~ command = {
# ~ 'command': 'add',
# ~ 'properties': {
# ~ 'name': WATCHER_NAME,
# ~ 'cmd': 'verdi devel run_daemon',
# ~ 'virtualenv': path.abspath(path.join(executable, '../../')),
# ~ 'copy_env': True,
# ~ 'pidfile': '{}/circus-aiida-{}'.format(path.dirname(path.abspath(__file__)), PROFILE_UUID),
# ~ 'autostart': True,
# ~ 'warmup_delay': 5
# ~ }
# ~ }
click.echo('adding the watcher...')
command = {
'command': 'add',
'properties': get_daemon_properties(profile)
}
click.echo(command)
self._client.call(command)
return self.start_watcher()
elif not self.is_watcher_active(get_daemon_name(profile)):
click.echo('starting the watcher...')
return self.start_watcher(get_daemon_name(profile))
return None
def start_outstreamer(self, msg):
if not streamer_name(msg) in self.list_watchers():
streamer = get_streamer(msg)
command = {
'command': 'add',
'properties': streamer,
}
self._client.call(command)
return self.start_watcher(streamer_name(msg))
self._client.call({
'command': 'set',
'properties': {
'name': streamer_name(msg),
'options': {'stdout_stream.filename': 'stream.log'}
}
})
elif not self.is_watcher_active(streamer_name(msg)):
return self.start_watcher(streamer_name(msg))
def stop_aiida_daemon(self, profile):
if get_daemon_name(profile) in self.list_watchers():
return self._client.call({
'command': 'stop',
'properties': {
'name': get_daemon_name(profile)
}
})
return None
def start_watcher(self, name=None):
response = self._client.call({
'command': 'start',
'properties': {
'name': name
}
})
return bool(response.get('status', None) == 'ok')
def status(self, name=None):
response = self._client.call({
'command': 'status',
'properties': {
'name': name
}
})
if response.get('status', None):
return str(response['status'])
def is_watcher_active(self, name):
return bool(self.status(name) == 'active')
def list_watchers(self):
response = self._client.call({
'command': 'list',
'properties': {}
})
return [str(watcher) for watcher in response['watchers']] if response['status'] == u'ok' else []
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.stats = defaultdict(list)
self.refresher = Refresher(self)
self.dstats = []
def stop(self):
self.client.stop()
self.refresher.running = False
self.refresher.join()
def verify(self):
self.watchers = []
# trying to list the watchers
msg = cmds['list'].make_message()
try:
res = self.client.call(msg)
self.connected = True
for watcher in res['watchers']:
msg = cmds['options'].make_message(name=watcher)
options = self.client.call(msg)
self.watchers.append((watcher, options['options']))
self.watchers.sort()
if not self.refresher.running:
self.refresher.start()
except CallError:
self.connected = False
def killproc(self, name, pid):
msg = cmds['signal'].make_message(name=name, process=int(pid),
signum=9)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
msg = cmds['globaloptions'].make_message()
options = self.client.call(msg)
return options['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
msg = cmds['incr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def decrproc(self, name):
msg = cmds['decr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
msg = cmds['list'].make_message(name=name)
res = self.client.call(msg)
return res['processes']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
pid = str(pid)
for stat in stats:
if pid not in stat:
continue
res.append(stat[pid][field])
return res
def get_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
return res['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
msg = cmds['add'].make_message(name=name, cmd=cmd)
res = self.client.call(msg)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
msg = cmds['set'].make_message(name=name, options=options)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
else:
return False
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (sys.executable,
'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True, options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('processes')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
# checking what the plugin did
wanted = [('test', 'spawn'), ('test', 'start'), ('test', 'spawn')]
self.assertEqual(Plugin.events, wanted)
