def handle_dealer(self, cmd, opts, msg, endpoint, timeout, ssh_server,
ssh_keyfile):
if endpoint is not None:
client = CircusClient(endpoint=endpoint,
timeout=timeout,
ssh_server=ssh_server,
ssh_keyfile=ssh_keyfile)
else:
client = self.client
try:
if isinstance(msg, list):
for i, command in enumerate(msg):
clm = self._console(client, command['cmd'], opts,
command['msg'])
print("%s: %s" % (i, clm))
else:
print(self._console(client, cmd, opts, msg))
except CallError as e:
sys.stderr.write(str(e) + " Try to raise the --timeout value\n")
return 1
finally:
if endpoint is not None:
client.stop()
return 0
def handle_dealer(self, cmd, opts, msg, endpoint, timeout, ssh_server,
ssh_keyfile):
if endpoint is not None:
client = CircusClient(endpoint=endpoint,
timeout=timeout,
ssh_server=ssh_server,
ssh_keyfile=ssh_keyfile)
else:
client = self.client
try:
if isinstance(msg, list):
for i, command in enumerate(msg):
clm = self._console(client, command['cmd'], opts,
command['msg'])
print("%s: %s" % (i, clm))
else:
print(self._console(client, cmd, opts, msg))
except CallError as e:
msg = str(e)
if 'timed out' in str(e).lower():
msg += TIMEOUT_MSG
sys.stderr.write(msg)
return 1
finally:
if endpoint is not None:
client.stop()
return 0
def handle_dealer(self, command, opts, msg, endpoint, timeout, ssh_server,
ssh_keyfile):
if endpoint is not None:
client = CircusClient(endpoint=endpoint, timeout=timeout,
ssh_server=ssh_server,
ssh_keyfile=ssh_keyfile)
else:
client = self.client
try:
if isinstance(msg, list):
for i, c in enumerate(msg):
clm = self._console(client, c['cmd'], opts,
c['msg'])
print("%s: %s" % (i, clm))
else:
print(self._console(client, command, opts, msg))
except CallError as e:
msg = str(e)
if 'timed out' in str(e).lower():
msg += TIMEOUT_MSG
sys.stderr.write(msg)
return 1
finally:
if endpoint is not None:
client.stop()
return 0
class TestCircus(unittest.TestCase):
def setUp(self):
self.arbiters = []
self.files = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
self.cli.stop()
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, 'w') as f:
f.write(content)
return file
@classmethod
def _create_circus(cls, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ['generic.py', callable, testfile]
worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir,
'name': 'test', 'graceful_timeout': 4}
worker.update(kw)
if stats:
arbiter = get_arbiter([worker], background=True, plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
debug=kw.get('debug', False))
else:
arbiter = get_arbiter([worker], background=True, plugins=plugins,
debug=kw.get('debug', False))
arbiter.start()
return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw):
testfile, arbiter = TestCircus._create_circus(callable, plugins, stats,
**kw)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def handle_dealer(self, cmd, opts, msg, endpoint, timeout, ssh_server,
ssh_keyfile):
if endpoint is not None:
client = CircusClient(endpoint=endpoint, timeout=timeout,
ssh_server=ssh_server,
ssh_keyfile=ssh_keyfile)
else:
client = self.client
try:
if isinstance(msg, list):
for i, command in enumerate(msg):
clm = self._console(client, command['cmd'], opts,
command['msg'])
print("%s: %s" % (i, clm))
else:
print(self._console(client, cmd, opts, msg))
except CallError as e:
sys.stderr.write(str(e) + " Try to raise the --timeout value\n")
return 1
finally:
if endpoint is not None:
client.stop()
return 0
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
def get_pids():
return self.call('list', name='test').get('pids')
pids = get_pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(
self.status('signal',
name='test',
pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = get_pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(
self.call("list", name="test").get('processes'), [1, 2])
self.assertEquals(
self.status("signal",
name="test",
process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(
self.call("list", name="test").get('processes'), [1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(
self.status("signal", name="test", signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(
self.call("list", name="test").get('processes'), processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(2.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
def get_pids():
return self.call('list', name='test').get('pids')
pids = get_pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(self.status('signal', name='test', pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = get_pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 2])
self.assertEquals(self.status("signal", name="test", process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(self.status("signal", name="test",
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(self.call("list", name="test").get('processes'),
processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(2.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def _client_test(self, ssh_server):
test_file = self._run_circus('circus.tests.support.run_process')
self.assertTrue(poll_for(test_file, 'START')) # process started
# playing around with the watcher
client = CircusClient(ssh_server=ssh_server)
def call(cmd, **props):
msg = make_message(cmd, **props)
return client.call(msg)
def status(cmd, **props):
resp = call(cmd, **props)
return resp.get('status')
def numprocesses(cmd, **props):
resp = call(cmd, **props)
return resp.get('numprocesses')
def numwatchers(cmd, **props):
resp = call(cmd, **props)
return resp.get('numwatchers')
def set(name, **opts):
return status("set", name=name, options=opts)
self.assertEquals(set("test", numprocesses=10), 'ok')
self.assertEquals(numprocesses("numprocesses"), 10)
self.assertEquals(set("test", numprocesses=1), 'ok')
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(numwatchers("numwatchers"), 1)
self.assertEquals(call("list").get('watchers'), ['test'])
self.assertEquals(numprocesses("incr", name="test"), 2)
self.assertEquals(numprocesses("numprocesses"), 2)
self.assertEquals(numprocesses("incr", name="test", nb=2), 4)
self.assertEquals(numprocesses("decr", name="test", nb=3), 1)
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error')
self.assertEquals(set("test", env={"test": '1', "test": '2'}),
'ok')
resp = call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEquals(options.get('env'), {'test': '1', 'test': '2'})
resp = call('stats', name='test')
self.assertEqual(resp['status'], 'ok')
resp = call('globaloptions', name='test')
self.assertEqual(resp['options']['pubsub_endpoint'],
'tcp://127.0.0.1:5556')
client.stop()
def test_handler(self):
self._run_circus('circus.tests.test_client.run_process')
time.sleep(.5)
# playing around with the watcher
client = CircusClient()
def call(cmd, **props):
msg = make_message(cmd, **props)
return client.call(msg)
def status(cmd, **props):
resp = call(cmd, **props)
return resp.get('status')
def numprocesses(cmd, **props):
resp = call(cmd, **props)
return resp.get('numprocesses')
def numwatchers(cmd, **props):
resp = call(cmd, **props)
return resp.get('numwatchers')
def set(name, **opts):
return status("set", name=name, options=opts)
self.assertEquals(set("test", numprocesses=10), 'ok')
self.assertEquals(numprocesses("numprocesses"), 10)
self.assertEquals(set("test", numprocesses=1), 'ok')
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(numwatchers("numwatchers"), 1)
self.assertEquals(call("list").get('watchers'), ['test'])
self.assertEquals(call("list", name="test").get('processes'), [10])
self.assertEquals(numprocesses("incr", name="test"), 2)
self.assertEquals(numprocesses("numprocesses"), 2)
self.assertEquals(numprocesses("decr", name="test"), 1)
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error')
self.assertEquals(set("test", env={"test": '1', "test": '2'}),
'ok')
resp = call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEquals(options.get('env'), {'test': '1', 'test': '2'})
resp = call('stats', name='test')
self.assertEqual(resp['status'], 'ok')
resp = call('globaloptions', name='test')
self.assertEqual(resp['options']['pubsub_endpoint'],
'tcp://127.0.0.1:5556')
client.stop()
def _client_test(self, ssh_server):
self._run_circus('circus.tests.support.run_process')
time.sleep(.5)
# playing around with the watcher
client = CircusClient(ssh_server=ssh_server)
def call(cmd, **props):
msg = make_message(cmd, **props)
return client.call(msg)
def status(cmd, **props):
resp = call(cmd, **props)
return resp.get('status')
def numprocesses(cmd, **props):
resp = call(cmd, **props)
return resp.get('numprocesses')
def numwatchers(cmd, **props):
resp = call(cmd, **props)
return resp.get('numwatchers')
def set(name, **opts):
return status("set", name=name, options=opts)
self.assertEquals(set("test", numprocesses=10), 'ok')
self.assertEquals(numprocesses("numprocesses"), 10)
self.assertEquals(set("test", numprocesses=1), 'ok')
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(numwatchers("numwatchers"), 1)
self.assertEquals(call("list").get('watchers'), ['test'])
self.assertEquals(numprocesses("incr", name="test"), 2)
self.assertEquals(numprocesses("numprocesses"), 2)
self.assertEquals(numprocesses("incr", name="test", nb=2), 4)
self.assertEquals(numprocesses("decr", name="test", nb=3), 1)
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error')
self.assertEquals(set("test", env={"test": '1', "test": '2'}), 'ok')
resp = call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEquals(options.get('env'), {'test': '1', 'test': '2'})
resp = call('stats', name='test')
self.assertEqual(resp['status'], 'ok')
resp = call('globaloptions', name='test')
self.assertEqual(resp['options']['pubsub_endpoint'],
'tcp://127.0.0.1:5556')
client.stop()
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 2])
self.assertEquals(self.status("signal", name="test", process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(self.status("signal", name="test",
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(self.call("list", name="test").get('processes'),
processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'], 'python')
def handle_dealer(self, cmd, opts, msg, endpoint, timeout):
client = CircusClient(endpoint=endpoint, timeout=timeout)
try:
if isinstance(msg, list):
for i, cmd in enumerate(msg):
clm = self._console(client, cmd, opts, msg)
print("%s: %s" % (i, clm))
else:
print(self._console(client, cmd, opts, msg))
except CallError as e:
sys.stderr.write(str(e))
return 1
finally:
client.stop()
return 0
def test_handler(self):
self._run_circus('circus.tests.test_client.run_process')
time.sleep(.5)
# playing around with the watcher
client = CircusClient()
def call(cmd, **props):
msg = make_message(cmd, **props)
return client.call(msg)
def status(cmd, **props):
resp = call(cmd, **props)
return resp.get('status')
def numprocesses(cmd, **props):
resp = call(cmd, **props)
return resp.get('numprocesses')
def numwatchers(cmd, **props):
resp = call(cmd, **props)
return resp.get('numwatchers')
def set(name, **opts):
return status("set", name=name, options=opts)
self.assertEquals(set("test", numprocesses=10), 'ok')
self.assertEquals(numprocesses("numprocesses"), 10)
self.assertEquals(set("test", numprocesses=1), 'ok')
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(numwatchers("numwatchers"), 1)
self.assertEquals(call("list").get('watchers'), ['test'])
self.assertEquals(call("list", name="test").get('processes'), [10])
self.assertEquals(numprocesses("incr", name="test"), 2)
self.assertEquals(numprocesses("numprocesses"), 2)
self.assertEquals(numprocesses("decr", name="test"), 1)
self.assertEquals(numprocesses("numprocesses"), 1)
self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error')
self.assertEquals(set("test", env={"test": '1', "test": '2'}), 'ok')
resp = call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEquals(options.get('env'), {'test': '1', 'test': '2'})
client.stop()
def handle_dealer(self, cmd, opts, msg, endpoint, timeout):
client = CircusClient(endpoint=endpoint, timeout=timeout)
try:
if isinstance(msg, list):
for i, command in enumerate(msg):
clm = self._console(client, command['cmd'], opts,
command['msg'])
print("%s: %s" % (i, clm))
else:
print(self._console(client, cmd, opts, msg))
except CallError as e:
sys.stderr.write(str(e))
return 1
finally:
client.stop()
return 0
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = 'circus.tests.test_stream.run_process'
fd, log = tempfile.mkstemp()
self.log = log
os.close(fd)
stream = {'stream': FileStream(log)}
self.test_file = self._run_circus(dummy_process,
stdout_stream=stream, stderr_stream=stream, debug=True)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
os.remove(self.log)
def test_stream(self):
time.sleep(2.)
self.call("stats").get('infos')
# let's see what we got in the file
with open(self.log) as f:
data = f.read()
self.assertTrue('stderr' in data)
self.assertTrue('stdout' in data)
# restarting
self.call('restart')
time.sleep(1.)
current = time.time()
# should be running
with open(self.log) as f:
data = f.readlines()
# last log should be less than one second old
last = data[-1]
delta = abs(current - int(last.split('-')[0]))
self.assertTrue(delta < 1, delta)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = "circus.tests.test_stream.run_process"
fd, self.stdout = tempfile.mkstemp()
os.close(fd)
fd, self.stderr = tempfile.mkstemp()
os.close(fd)
self.test_file = self._run_circus(
dummy_process,
stdout_stream={"stream": FileStream(self.stdout)},
stderr_stream={"stream": FileStream(self.stderr)},
debug=True,
)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
os.remove(self.stdout)
os.remove(self.stderr)
def test_stream(self):
# wait for the process to be started
self.assertTrue(poll_for(self.stdout, "stdout"))
self.assertTrue(poll_for(self.stderr, "stderr"))
# clean slate
truncate_file(self.stdout)
truncate_file(self.stderr)
# restart and make sure streams are still working
self.call("restart")
# wait for the process to be restarted
self.assertTrue(poll_for(self.stdout, "stdout"))
self.assertTrue(poll_for(self.stderr, "stderr"))
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
dummy_process = 'circus.tests.test_stream.run_process'
fd, self.stdout = tempfile.mkstemp()
os.close(fd)
fd, self.stderr = tempfile.mkstemp()
os.close(fd)
self.test_file = self._run_circus(
dummy_process,
stdout_stream={'stream': FileStream(self.stdout)},
stderr_stream={'stream': FileStream(self.stderr)},
debug=True)
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
os.remove(self.stdout)
os.remove(self.stderr)
def test_stream(self):
# wait for the process to be started
self.assertTrue(poll_for(self.stdout, 'stdout'))
self.assertTrue(poll_for(self.stderr, 'stderr'))
# clean slate
truncate_file(self.stdout)
truncate_file(self.stderr)
# restart and make sure streams are still working
self.call('restart')
# wait for the process to be restarted
self.assertTrue(poll_for(self.stdout, 'stdout'))
self.assertTrue(poll_for(self.stderr, 'stderr'))
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.stats = defaultdict(list)
self.refresher = Refresher(self)
self.dstats = []
def stop(self):
self.client.stop()
self.refresher.running = False
self.refresher.join()
def verify(self):
self.watchers = []
# trying to list the watchers
msg = cmds['list'].make_message()
try:
res = self.client.call(msg)
self.connected = True
for watcher in res['watchers']:
if watcher == 'circusd-stats':
continue
msg = cmds['options'].make_message(name=watcher)
options = self.client.call(msg)
self.watchers.append((watcher, options['options']))
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
if not self.refresher.running:
self.refresher.start()
except CallError:
self.connected = False
def killproc(self, name, pid):
msg = cmds['signal'].make_message(name=name,
process=int(pid),
signum=9)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
msg = cmds['globaloptions'].make_message()
options = self.client.call(msg)
return options['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
msg = cmds['incr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def decrproc(self, name):
msg = cmds['decr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
msg = cmds['listpids'].make_message(name=name)
res = self.client.call(msg)
return res['pids']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
return res['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
msg = cmds['add'].make_message(name=name, cmd=cmd)
res = self.client.call(msg)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
msg = cmds['set'].make_message(name=name, options=options)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
else:
return False
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (sys.executable,
'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy',
testfile]
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('processes')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.arbiter = self.arbiters[-1]
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def pids(self):
return self.call('list', name='test').get('pids')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
pids = self.pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(
self.status('signal',
name='test',
pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = self.pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_unexisting(self):
watcher = self.arbiter.get_watcher("test")
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
to_kill = process.pid
# the process is killed in an unsual way
os.kill(to_kill, signal.SIGSEGV)
# and wait for it to die
pid, status = os.waitpid(to_kill, 0)
# ansure the old process is considered "unexisting"
self.assertEquals(process.status, UNEXISTING)
# this should clean up and create a new process
watcher.reap_and_manage_processes()
# we should have a new process here now
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
# and that one needs to have a new pid.
self.assertNotEqual(process.pid, to_kill)
# and should not be unexisting...
self.assertNotEqual(process.status, UNEXISTING)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_max_age(self):
result = self.call('set',
name='test',
options={
'max_age': 1,
'max_age_variance': 0
})
self.assertEquals(result.get('status'), 'ok')
initial_pids = self.pids()
time.sleep(3.0) # allow process to reach max_age and restart
current_pids = self.pids()
self.assertEqual(len(current_pids), 1)
self.assertNotEqual(initial_pids, current_pids)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.arbiter = self.arbiters[-1]
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def pids(self):
return self.call('list', name='test').get('pids')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
pids = self.pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(self.status('signal', name='test', pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = self.pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_unexisting(self):
watcher = self.arbiter.get_watcher("test")
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
to_kill = process.pid
# the process is killed in an unsual way
os.kill(to_kill, signal.SIGSEGV)
# and wait for it to die
pid, status = os.waitpid(to_kill, 0)
# ansure the old process is considered "unexisting"
self.assertEquals(process.status, UNEXISTING)
# this should clean up and create a new process
watcher.reap_and_manage_processes()
# we should have a new process here now
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
# and that one needs to have a new pid.
self.assertNotEqual(process.pid, to_kill)
# and should not be unexisting...
self.assertNotEqual(process.status, UNEXISTING)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_max_age(self):
result = self.call('set', name='test',
options={'max_age': 1, 'max_age_variance': 0})
self.assertEquals(result.get('status'), 'ok')
initial_pids = self.pids()
time.sleep(3.0) # allow process to reach max_age and restart
current_pids = self.pids()
self.assertEqual(len(current_pids), 1)
self.assertNotEqual(initial_pids, current_pids)
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (sys.executable,
'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True, options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('processes')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
# checking what the plugin did
wanted = [('test', 'spawn'), ('test', 'start'), ('test', 'spawn')]
self.assertEqual(Plugin.events, wanted)
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher == 'circusd-stats':
continue
options = self.client.send_message('options', name=watcher)
self.watchers.append((watcher, options['options']))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
except CallError:
self.connected = False
def killproc(self, name, pid):
res = self.client.send_message('signal',
name=name,
process=int(pid),
signum=9)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class TestCircus(unittest.TestCase):
def setUp(self):
self.arbiters = []
self.files = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
self.cli.stop()
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, 'w') as f:
f.write(content)
return file
def _run_circus(self, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ['generic.py', callable, testfile]
worker = {
'cmd': _CMD,
'args': args,
'working_dir': wdir,
'name': 'test',
'graceful_timeout': 4
}
worker.update(kw)
if stats:
arbiter = get_arbiter([worker],
background=True,
plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
debug=kw.get('debug', False))
else:
arbiter = get_arbiter([worker],
background=True,
plugins=plugins,
debug=kw.get('debug', False))
arbiter.start()
time.sleep(.3)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.stats = defaultdict(list)
self.refresher = Refresher(self)
self.dstats = []
def stop(self):
self.client.stop()
self.refresher.running = False
self.refresher.join()
def verify(self):
self.watchers = []
# trying to list the watchers
msg = cmds['list'].make_message()
try:
res = self.client.call(msg)
self.connected = True
for watcher in res['watchers']:
msg = cmds['options'].make_message(name=watcher)
options = self.client.call(msg)
self.watchers.append((watcher, options['options']))
self.watchers.sort()
if not self.refresher.running:
self.refresher.start()
except CallError:
self.connected = False
def killproc(self, name, pid):
msg = cmds['signal'].make_message(name=name, process=int(pid),
signum=9)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
msg = cmds['globaloptions'].make_message()
options = self.client.call(msg)
return options['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
msg = cmds['incr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def decrproc(self, name):
msg = cmds['decr'].make_message(name=name)
res = self.client.call(msg)
self.verify() # will do better later
return res['numprocesses']
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
msg = cmds['list'].make_message(name=name)
res = self.client.call(msg)
return res['processes']
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
pid = str(pid)
for stat in stats:
if pid not in stat:
continue
res.append(stat[pid][field])
return res
def get_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
return res['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
msg = cmds['add'].make_message(name=name, cmd=cmd)
res = self.client.call(msg)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
msg = cmds['set'].make_message(name=name, options=options)
res = self.client.call(msg)
self.verify() # will do better later
return res['status'] == 'ok'
else:
return False
class TestCircus(unittest.TestCase):
def setUp(self):
self.arbiters = []
self.files = []
self.dirs = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
for dir in self.dirs:
shutil.rmtree(dir)
self.cli.stop()
def get_tmpdir(self):
dir_ = mkdtemp()
self.dirs.append(dir_)
return dir_
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, "w") as f:
f.write(content)
return file
@classmethod
def _create_circus(cls, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ["generic.py", callable, testfile]
worker = {"cmd": _CMD, "args": args, "working_dir": wdir, "name": "test", "graceful_timeout": 4}
worker.update(kw)
debug = kw.get("debug", False)
if stats:
arbiter = get_arbiter(
[worker],
background=True,
plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
statsd=True,
debug=debug,
statsd_close_outputs=not debug,
)
else:
arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=debug)
arbiter.start()
return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw):
testfile, arbiter = TestCircus._create_circus(callable, plugins, stats, **kw)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class Client(object):
def __init__(self, port=6000):
self._client = CircusClient(endpoint='tcp://127.0.0.1:{}'.format(port))
def startup(self):
click.echo('startup')
sp.call(['circusd', 'basic.ini', '--daemon'])
def quit(self):
self._client.call({
'command': 'quit',
'properties': {}
})
def stop(self):
self._client.stop()
def start_aiida_daemon(self, profile):
if not get_daemon_name(profile) in self.list_watchers():
# ~ command = {
# ~ 'command': 'add',
# ~ 'properties': {
# ~ 'name': WATCHER_NAME,
# ~ 'cmd': 'verdi devel run_daemon',
# ~ 'virtualenv': path.abspath(path.join(executable, '../../')),
# ~ 'copy_env': True,
# ~ 'pidfile': '{}/circus-aiida-{}'.format(path.dirname(path.abspath(__file__)), PROFILE_UUID),
# ~ 'autostart': True,
# ~ 'warmup_delay': 5
# ~ }
# ~ }
click.echo('adding the watcher...')
command = {
'command': 'add',
'properties': get_daemon_properties(profile)
}
click.echo(command)
self._client.call(command)
return self.start_watcher()
elif not self.is_watcher_active(get_daemon_name(profile)):
click.echo('starting the watcher...')
return self.start_watcher(get_daemon_name(profile))
return None
def start_outstreamer(self, msg):
if not streamer_name(msg) in self.list_watchers():
streamer = get_streamer(msg)
command = {
'command': 'add',
'properties': streamer,
}
self._client.call(command)
return self.start_watcher(streamer_name(msg))
self._client.call({
'command': 'set',
'properties': {
'name': streamer_name(msg),
'options': {'stdout_stream.filename': 'stream.log'}
}
})
elif not self.is_watcher_active(streamer_name(msg)):
return self.start_watcher(streamer_name(msg))
def stop_aiida_daemon(self, profile):
if get_daemon_name(profile) in self.list_watchers():
return self._client.call({
'command': 'stop',
'properties': {
'name': get_daemon_name(profile)
}
})
return None
def start_watcher(self, name=None):
response = self._client.call({
'command': 'start',
'properties': {
'name': name
}
})
return bool(response.get('status', None) == 'ok')
def status(self, name=None):
response = self._client.call({
'command': 'status',
'properties': {
'name': name
}
})
if response.get('status', None):
return str(response['status'])
def is_watcher_active(self, name):
return bool(self.status(name) == 'active')
def list_watchers(self):
response = self._client.call({
'command': 'list',
'properties': {}
})
return [str(watcher) for watcher in response['watchers']] if response['status'] == u'ok' else []
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
self.test_file = self._run_circus('circus.tests.test_trainer.run_dummy')
time.sleep(0.5)
self.cli = CircusClient(TEST_ENDPOINT)
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numshows(self):
resp = self.cli.call("numshows")
self.assertEqual(resp, "1")
def test_numflies(self):
resp = self.cli.call("numflies")
self.assertEqual(resp, "1")
def test_flies(self):
resp = self.cli.call("flies")
self.assertEqual(resp, "test: 1")
def test_shows(self):
resp = self.cli.call("shows")
self.assertEqual(resp, "test")
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (sys.executable,
'circus.tests.test_trainer.run_dummy', testfile)
return cmd
def test_add_show(self):
cmd = self._get_cmd()
resp = self.cli.call("add_show test1 %s" % cmd)
self.assertEqual(resp, "ok")
def test_add_show1(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
resp = self.cli.call("shows")
self.assertTrue(resp.endswith("test1"))
def test_add_show2(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
resp = self.cli.call("numshows")
self.assertEqual(resp, "2")
def test_add_show3(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
resp = self.cli.call("add_show test1 %s" % cmd)
self.assertTrue(resp.startswith("error:"))
def test_del_show(self):
cmd = self._get_cmd()
self.cli.call("add_show test1 %s" % cmd)
self.cli.call("del_show test1")
resp = self.cli.call("numshows")
self.assertEqual(resp, "1")
def test_stop(self):
self.cli.call("stop")
self.assertRaises(CallError, self.cli.call, "shows")
def test_reload(self):
resp = self.cli.call("reload")
self.assertEqual(resp, "ok")
def test_reload1(self):
flies0 = self.cli.call("flies")
resp = self.cli.call("reload")
time.sleep(0.5)
flies1 = self.cli.call("flies")
self.assertNotEqual(flies0, flies1)
def test_reload(self):
self.cli.call("reload")
time.sleep(0.5)
flies = self.cli.call("flies")
self.assertEqual(flies, "test: 2")
def test_stop_shows(self):
resp = self.cli.call("stop_shows")
self.assertEqual(resp, "ok")
def test_stop_shows1(self):
self.cli.call("stop_shows")
resp = self.cli.call("status test")
self.assertEqual(resp, "stopped")
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable, 'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('processes')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.support.run_process'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
self.cli.send_message("incr", name="test", nb=2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 4)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable,
'circus.tests.support.run_process',
testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.support.run_process']
return cmd, args
def _get_options(self, **kwargs):
if 'graceful_timeout' not in kwargs:
kwargs['graceful_timeout'] = 4
return kwargs
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True, options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True,
options=self._get_options(flapping_window=100))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
self.assertTrue(poll_for(self.test_file, 'START')) # process started
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
self.assertTrue(poll_for(self.test_file, 'START')) # process started
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.support.run_process'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
def nb_processes():
return len(cli.send_message('list', name='test').get('pids'))
def incr_processes():
return cli.send_message('incr', name='test')
# wait for the plugin to be started
self.assertTrue(poll_for(datafile, 'PLUGIN STARTED'))
cli = CircusClient()
self.assertEqual(nb_processes(), 1)
incr_processes()
self.assertEqual(nb_processes(), 2)
# wait for the plugin to receive the signal
self.assertTrue(poll_for(datafile, 'test:spawn'))
truncate_file(datafile)
incr_processes()
self.assertEqual(nb_processes(), 3)
# wait for the plugin to receive the signal
self.assertTrue(poll_for(datafile, 'test:spawn'))
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.support.run_process'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
class TestCircus(unittest.TestCase):
arbiter_factory = get_arbiter
def setUp(self):
self.arbiters = []
self.files = []
self.dirs = []
self.tmpfiles = []
self.cli = CircusClient()
def tearDown(self):
self._stop_runners()
for file in self.files + self.tmpfiles:
if os.path.exists(file):
os.remove(file)
for dir in self.dirs:
shutil.rmtree(dir)
self.cli.stop()
def get_tmpdir(self):
dir_ = mkdtemp()
self.dirs.append(dir_)
return dir_
def get_tmpfile(self, content=None):
fd, file = mkstemp()
os.close(fd)
self.tmpfiles.append(file)
if content is not None:
with open(file, 'w') as f:
f.write(content)
return file
@classmethod
def _create_circus(cls, callable, plugins=None, stats=False, **kw):
resolve_name(callable) # used to check the callable
fd, testfile = mkstemp()
os.close(fd)
wdir = os.path.dirname(__file__)
args = ['generic.py', callable, testfile]
worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir,
'name': 'test', 'graceful_timeout': 2}
worker.update(kw)
debug = kw.get('debug', False)
fact = cls.arbiter_factory
if stats:
arbiter = fact([worker], background=True, plugins=plugins,
stats_endpoint=DEFAULT_ENDPOINT_STATS,
statsd=True,
debug=debug, statsd_close_outputs=not debug)
else:
arbiter = fact([worker], background=True, plugins=plugins,
debug=debug)
arbiter.start()
return testfile, arbiter
def _run_circus(self, callable, plugins=None, stats=False, **kw):
testfile, arbiter = TestCircus._create_circus(callable, plugins, stats,
**kw)
self.arbiters.append(arbiter)
self.files.append(testfile)
return testfile
def _stop_runners(self):
for arbiter in self.arbiters:
arbiter.stop()
self.arbiters = []
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
class LiveClient(object):
def __init__(self, endpoint, ssh_server=None):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint,
ssh_server=ssh_server)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
self.sockets = None
self.use_sockets = False
self.embed_httpd = False
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
self.plugins = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher in ('circusd-stats', 'circushttpd'):
if watcher == 'circushttpd':
self.embed_httpd = True
continue
options = self.client.send_message('options',
name=watcher)['options']
self.watchers.append((watcher, options))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
if not self.use_sockets and options.get('use_sockets', False):
self.use_sockets = True
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
if self.endpoint.startswith('tcp://'):
# In case of multi interface binding i.e: tcp://0.0.0.0:5557
anyaddr = '0.0.0.0'
ip = self.endpoint.lstrip('tcp://').split(':')[0]
self.stats_endpoint = self.stats_endpoint.replace(anyaddr, ip)
except CallError:
self.connected = False
def killproc(self, name, pid):
# killing a proc and its children
res = self.client.send_message('signal', name=name, pid=int(pid),
signum=9, recursive=True)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_sockets(self, force_reload=False):
if not self.sockets or force_reload:
res = self.client.send_message('listsockets')
self.sockets = res['sockets']
return self.sockets
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = "circus.tests.test_arbiter.run_dummy"
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get("watchers"), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = "%s generic.py %s %s" % (sys.executable, "circus.tests.test_arbiter.run_dummy", testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ["generic.py", "circus.tests.test_arbiter.run_dummy1"]
return cmd, args
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get("watchers"), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get("status"), "error")
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True, options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get("options", {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get("processes")
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get("processes")
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get("processes")
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get("processes")
self.assertEqual(processes2, [2])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = "circus.tests.test_arbiter.run_dummy"
plugin = "circus.tests.test_arbiter.Plugin"
plugins = [{"use": plugin, "file": datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2, 3])
# wait a bit
time.sleep(0.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split("\n") if line != ""]
wanted = ["test:spawn", "test:spawn"]
self.assertEqual(data, wanted)
def test_singleton(self):
self._stop_runners()
dummy_process = "circus.tests.test_arbiter.run_dummy"
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message("incr", name="test")
self.assertEqual(res["numprocesses"], 1)
class LiveClient(object):
def __init__(self, endpoint):
self.endpoint = str(endpoint)
self.stats_endpoint = None
self.client = CircusClient(endpoint=self.endpoint)
self.connected = False
self.watchers = []
self.plugins = []
self.stats = defaultdict(list)
self.dstats = []
self.sockets = None
self.use_sockets = False
self.embed_httpd = False
def stop(self):
self.client.stop()
def update_watchers(self):
"""Calls circus and initialize the list of watchers.
If circus is not connected raises an error.
"""
self.watchers = []
# trying to list the watchers
try:
self.connected = True
for watcher in self.client.send_message('list')['watchers']:
if watcher in ('circusd-stats', 'circushttpd'):
if watcher == 'circushttpd':
self.embed_httpd = True
continue
options = self.client.send_message('options',
name=watcher)['options']
self.watchers.append((watcher, options))
if watcher.startswith('plugin:'):
self.plugins.append(watcher)
if not self.use_sockets and options.get('use_sockets', False):
self.use_sockets = True
self.watchers.sort()
self.stats_endpoint = self.get_global_options()['stats_endpoint']
except CallError:
self.connected = False
def killproc(self, name, pid):
res = self.client.send_message('signal', name=name, pid=int(pid),
signum=9, children=True)
self.update_watchers() # will do better later
return res
def get_option(self, name, option):
watchers = dict(self.watchers)
return watchers[name][option]
def get_global_options(self):
return self.client.send_message('globaloptions')['options']
def get_options(self, name):
watchers = dict(self.watchers)
return watchers[name].items()
def incrproc(self, name):
res = self.client.send_message('incr', name=name)
self.update_watchers() # will do better later
return res
def decrproc(self, name):
res = self.client.send_message('decr', name=name)
self.update_watchers() # will do better later
return res
def get_stats(self, name, start=0, end=-1):
return self.stats[name][start:end]
def get_dstats(self, field, start=0, end=-1):
stats = self.dstats[start:end]
res = []
for stat in stats:
res.append(stat[field])
return res
def get_pids(self, name):
res = self.client.send_message('list', name=name)
return res['pids']
def get_sockets(self, force_reload=False):
if not self.sockets or force_reload:
res = self.client.send_message('listsockets')
self.sockets = res['sockets']
return self.sockets
def get_series(self, name, pid, field, start=0, end=-1):
stats = self.get_stats(name, start, end)
res = []
for stat in stats:
pids = stat['pid']
if isinstance(pids, list):
continue
if str(pid) == str(stat['pid']):
res.append(stat[field])
return res
def get_status(self, name):
return self.client.send_message('status', name=name)['status']
def switch_status(self, name):
msg = cmds['status'].make_message(name=name)
res = self.client.call(msg)
status = res['status']
if status == 'active':
# stopping the watcher
msg = cmds['stop'].make_message(name=name)
else:
msg = cmds['start'].make_message(name=name)
res = self.client.call(msg)
return res
def add_watcher(self, name, cmd, **kw):
res = self.client.send_message('add', name=name, cmd=cmd)
if res['status'] == 'ok':
# now configuring the options
options = {}
options['numprocesses'] = int(kw.get('numprocesses', '5'))
options['working_dir'] = kw.get('working_dir')
options['shell'] = kw.get('shell', 'off') == 'on'
res = self.client.send_message('set', name=name, options=options)
self.update_watchers() # will do better later
return res
class TestTrainer(TestCircus):
def setUp(self):
super(TestTrainer, self).setUp()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self.test_file = self._run_circus(dummy_process)
self.cli = CircusClient()
def tearDown(self):
super(TestTrainer, self).tearDown()
self.cli.stop()
def test_numwatchers(self):
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
self.cli.send_message("incr", name="test", nb=2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 4)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def _get_cmd(self):
fd, testfile = mkstemp()
os.close(fd)
cmd = '%s generic.py %s %s' % (
sys.executable, 'circus.tests.test_arbiter.run_dummy', testfile)
return cmd
def _get_cmd_args(self):
cmd = sys.executable
args = ['generic.py', 'circus.tests.test_arbiter.run_dummy1']
return cmd, args
def _get_options(self, **kwargs):
if 'graceful_timeout' not in kwargs:
kwargs['graceful_timeout'] = 4
return kwargs
def test_add_watcher(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher1(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def test_add_watcher2(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_add_watcher3(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher5(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
start=True,
options=self._get_options(flapping_window=100))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get('options', {})
self.assertEqual(options.get("flapping_window"), 100)
def test_rm_watcher(self):
msg = make_message("add",
name="test1",
cmd=self._get_cmd(),
options=self._get_options())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers1(self):
self.cli.call(make_message("stop"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add",
name="test1",
cmd=cmd,
args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin, 'file': datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
def nb_processes():
return len(cli.send_message('list', name='test').get('pids'))
def incr_processes():
return cli.send_message('incr', name='test')
cli = CircusClient()
self.assertEqual(nb_processes(), 1)
incr_processes()
self.assertEqual(nb_processes(), 2)
incr_processes()
self.assertEqual(nb_processes(), 3)
# wait a bit
time.sleep(.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split('\n') if line != '']
wanted = ['test:spawn', 'test:spawn']
self.assertEqual(data, wanted)
def test_singleton(self):
self._stop_runners()
dummy_process = 'circus.tests.test_arbiter.run_dummy'
self._run_circus(dummy_process, singleton=True)
cli = CircusClient()
# adding more than one process should fail
res = cli.send_message('incr', name='test')
self.assertEqual(res['numprocesses'], 1)
class DaemonTamer(object):
_DAEMON_MAP_FILE = 'daemon_map.yaml'
_ENDPOINT_TPL = '{host}:{port}'
def __init__(self, profile):
self.profile = profile
self.daemon_name = get_daemon_name(profile)
self.config = self.get_update_profile()
self._client = CircusClient() # endpoint=self.endpoint)
def generate_new_portnum(self, config=None):
port = 6000
used_ports = [profile['port'] for profile in config.values()]
while port in used_ports:
port += 2
return port
def read_config(self):
if not path.isfile(self._DAEMON_MAP_FILE):
return {}
with open(self._DAEMON_MAP_FILE, 'r') as daemon_map_fo:
config = yaml.load(daemon_map_fo) or {}
return config
def write_config(self, config):
with open(self._DAEMON_MAP_FILE, 'w') as daemon_map_fo:
yaml.dump(config, daemon_map_fo)
def get_update_profile(self):
config = self.read_config()
profile_config = config.get(self.profile, {})
profile_config['uuid'] = get_profile_uuid(self.profile)
profile_config['host'] = profile_config.get('host', 'tcp://127.0.0.1')
profile_config['port'] = profile_config.get('port', self.generate_new_portnum(config))
config[self.profile] = profile_config
self.write_config(config)
return config
@property
def profile_config(self):
return self.config[self.profile]
@property
def endpoint(self):
return self._ENDPOINT_TPL.format(host=self.profile_config['host'], port=self.profile_config['port'])
@property
def pubsub_endpoint(self):
return self._ENDPOINT_TPL.format(host=self.profile_config['host'], port=self.profile_config['port'] + 1)
@property
def stats_endpoint(self):
return self._ENDPOINT_TPL.format(host=self.profile_config['host'], port=self.profile_config['port'] + 2)
@property
def arbiter_config(self):
return {
'logoutput': 'aiida-circus.log',
'loglevel': 'INFO',
'debug': False,
'statsd': True,
# ~ 'controller': self.endpoint,
# ~ 'pubsub_endpoint': self.pubsub_endpoint,
# ~ 'stats_endpoint': self.stats_endpoint
}
@property
def watcher_config(self):
return get_daemon_properties(self.profile)
def make_arbiter(self):
print self.arbiter_config
print self.watcher_config
return get_arbiter([self.watcher_config], **self.arbiter_config)
def pause_daemon(self):
self._client.stop()
def unpause_daemon(self):
self._client.call({
'command': 'start',
'properties': {}
})
def status(self):
response = self._client.call({
'command': 'status',
'properties': {
'name': self.daemon_name
}
})
if response.get('status', None):
return str(response['status'])
def quit(self):
try:
self._client.call({
'command': 'quit',
'properties': {}
})
except CallError:
pass
