def test_message_id(self):
hooks = {'before_stop': ('circus.tests.test_client.long_hook', False)}
testfile, arbiter = self.run_with_hooks(hooks)
yield arbiter.start()
try:
self.assertTrue(os.path.exists(tmp_filename))
msg = make_message("numwatchers")
resp = yield self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
# this should timeout
resp = yield self.cli.call(make_message("stop"))
self.assertEqual(resp.get('status'), 'ok')
while arbiter.watchers[0].status() != 'stopped':
yield tornado_sleep(.1)
resp = yield self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
self.assertFalse(os.path.exists(tmp_filename))
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
arbiter.stop()
def test_rm_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
msg = make_message("rm", name="test1")
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
def check(self, watcher_name):
timeline = self.timelines[watcher_name]
if watcher_name in self.configs:
conf = self.configs[watcher_name]
else:
conf = self.update_conf(watcher_name)
tries = self.tries.get(watcher_name, 0)
if len(timeline) == conf["times"]:
duration = timeline[-1] - timeline[0] - self.check_delay
if duration <= conf["within"]:
if tries < conf["max_retry"]:
logger.info("%s: flapping detected: retry in %2ds", watcher_name, conf["retry_in"])
self.cast(make_message("stop", name=watcher_name))
self.timelines[watcher_name] = []
self.tries[watcher_name] = tries + 1
def _start():
self.cast(make_message("start", name=watcher_name))
timer = Timer(conf["retry_in"], _start)
timer.start()
self.timers[watcher_name] = timer
else:
logger.info("%s: flapping detected: max retry limit", watcher_name)
self.timelines[watcher_name] = []
self.tries[watcher_name] = 0
self.cast(make_message("stop", name=watcher_name))
else:
self.timelines[watcher_name] = []
self.tries[watcher_name] = 0
def test_add_watcher6(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
dummy_process = 'circus.tests.support.run_process'
self.test_file = self._run_circus(dummy_process)
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher_arbiter_stopped(self):
# stop the arbiter
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
msg = make_message("add", name="test1", cmd=self._get_cmd(),
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 1)
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(len(resp.get('pids')), 2)
def test_processes(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
self.cli.call(msg2)
resp = self.cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
start=True, options=dict(numprocesses=2))
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
time.sleep(0.1)
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("numprocesses", name="test1"))
self.assertEqual(resp.get("numprocesses"), 2)
def test_add_watcher7(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args, start=True, options={"flapping_window": 100})
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "active")
resp = self.cli.call(make_message("options", name="test1"))
options = resp.get("options", {})
self.assertEqual(options.get("flapping_window"), 100)
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get("processes")
self.cli.call(make_message("reload"))
time.sleep(0.5)
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get("processes")
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('processes')
self.assertEqual(processes1, [1])
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('processes')
self.assertEqual(processes2, [2])
def test_stop_watchers3(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get("status"), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "active")
def test_reload1(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
truncate_file(self.test_file) # clean slate
self.cli.call(make_message("reload"))
self.assertTrue(poll_for(self.test_file, 'START')) # restarted
msg2 = make_message("list", name="test")
resp = self.cli.call(msg2)
processes2 = resp.get('pids')
self.assertNotEqual(processes1, processes2)
def test_reload2(self):
msg1 = make_message("list", name="test")
resp = self.cli.call(msg1)
processes1 = resp.get('pids')
self.assertEqual(len(processes1), 1)
self.cli.call(make_message("reload"))
time.sleep(0.5)
make_message("list", name="test")
resp = self.cli.call(msg1)
processes2 = resp.get('pids')
self.assertEqual(len(processes2), 1)
self.assertNotEqual(processes1[0], processes2[0])
def test_client(self):
# playing around with the watcher
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
self.assertEquals(self.numprocesses("numprocesses"), 1)
self.assertEquals(self.set("test", numprocesses=2), 'ok')
self.assertEquals(self.numprocesses("numprocesses"), 2)
self.assertEquals(self.set("test", numprocesses=1), 'ok')
self.assertEquals(self.numprocesses("numprocesses"), 1)
self.assertEquals(self.numwatchers("numwatchers"), 1)
self.assertEquals(self.call("list").get('watchers'), ['test'])
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(self.numprocesses("numprocesses"), 2)
self.assertEquals(self.numprocesses("incr", name="test", nb=2), 4)
self.assertEquals(self.numprocesses("decr", name="test", nb=3), 1)
self.assertEquals(self.numprocesses("numprocesses"), 1)
self.assertEquals(self.set("test", env={"test": 1, "test": 2}),
'error')
self.assertEquals(self.set("test", env={"test": '1', "test": '2'}),
'ok')
resp = self.call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEquals(options.get('env'), {'test': '1', 'test': '2'})
resp = self.call('stats', name='test')
self.assertEqual(resp['status'], 'ok')
resp = self.call('globaloptions', name='test')
self.assertEqual(resp['options']['pubsub_endpoint'],
'tcp://127.0.0.1:5556')
def test_stop_watchers3(self):
if 'TRAVIS' in os.environ:
return
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args,
options=self._get_options())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
resp = self.cli.call(make_message("start", name="test1"))
self.assertEqual(resp.get("status"), "ok")
self.cli.call(make_message("stop", name="test1"))
resp = self.cli.call(make_message("status", name="test1"))
self.assertEqual(resp.get('status'), "stopped")
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "active")
def test_message_id(self):
hooks = {'before_stop': ('circus.tests.test_client.long_hook', False)}
try:
testfile, arbiter = self.run_with_hooks(hooks)
msg = make_message("numwatchers")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
# this should timeout
self.assertRaises(CallError, self.cli.call, make_message("stop"))
# and we should get back on our feet
del arbiter.watchers[0].hooks['before_stop']
while arbiter.watchers[0].status() != 'stopped':
time.sleep(.1)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
finally:
arbiter.stop()
def call(self, command, **props):
"""Sends the command to **circusd**
Options:
- **command** -- the command to call
- **props** -- keyword arguments to add to the call
Returns the JSON mapping sent back by **circusd**
"""
msg = make_message(command, **props)
self.client.send(json.dumps(msg))
msg = self.client.recv()
return json.loads(msg)
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
# setting up a circusd with a plugin
dummy_process = 'circus.tests.test_arbiter.run_dummy'
plugin = 'circus.tests.test_arbiter.Plugin'
plugins = [{'use': plugin}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get('processes'), [1, 2])
# checking what the plugin did
wanted = [('test', 'spawn'), ('test', 'start'), ('test', 'spawn')]
self.assertEqual(Plugin.events, wanted)
def test_plugins(self):
# killing the setUp runner
self._stop_runners()
self.cli.stop()
fd, datafile = mkstemp()
os.close(fd)
# setting up a circusd with a plugin
dummy_process = "circus.tests.test_arbiter.run_dummy"
plugin = "circus.tests.test_arbiter.Plugin"
plugins = [{"use": plugin, "file": datafile}]
self._run_circus(dummy_process, plugins=plugins)
# doing a few operations
cli = CircusClient()
msg1 = make_message("list", name="test")
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1])
msg2 = make_message("incr", name="test")
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2])
cli.call(msg2)
resp = cli.call(msg1)
self.assertEqual(resp.get("processes"), [1, 2, 3])
# wait a bit
time.sleep(0.2)
# checking what the plugin did
with open(datafile) as f:
data = [line for line in f.read().split("\n") if line != ""]
wanted = ["test:spawn", "test:spawn"]
self.assertEqual(data, wanted)
def check(self, show_name):
timeline = self.timelines[show_name]
if show_name in self.configs:
conf = self.configs[show_name]
else:
conf = self.update_conf(show_name)
tries = self.tries.get(show_name, 0)
if len(timeline) == conf['times']:
duration = timeline[-1] - timeline[0] - self.check_delay
if duration <= conf['within']:
if tries < conf['max_retry']:
logger.info("%s: flapping detected: retry in %2ds",
show_name, conf['retry_in'])
self.call(make_message("stop", name=show_name))
self.timelines[show_name] = []
self.tries[show_name] = tries + 1
def _start():
self.call(make_message("start", name=show_name))
timer = Timer(conf['retry_in'], _start)
timer.start()
self.timers[show_name] = timer
else:
logger.info("%s: flapping detected: max retry limit",
show_name)
self.timelines[show_name] = []
self.tries[show_name] = 0
self.call(make_message("stop", name=show_name))
else:
self.timelines[show_name] = []
self.tries[show_name] = 0
def test_client(self):
# playing around with the watcher
yield self.start_arbiter()
msg = make_message("numwatchers")
resp = yield self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
self.assertEqual((yield self.numprocesses("numprocesses")), 1)
self.assertEqual((yield self.set("test", numprocesses=2)), 'ok')
self.assertEqual((yield self.numprocesses("numprocesses")), 2)
self.assertEqual((yield self.set("test", numprocesses=1)), 'ok')
self.assertEqual((yield self.numprocesses("numprocesses")), 1)
self.assertEqual((yield self.numwatchers("numwatchers")), 1)
self.assertEqual((yield self.call("list")).get('watchers'), ['test'])
self.assertEqual((yield self.numprocesses("incr", name="test")), 2)
self.assertEqual((yield self.numprocesses("numprocesses")), 2)
self.assertEqual((yield self.numprocesses("incr", name="test", nb=2)),
4)
self.assertEqual((yield self.numprocesses("decr", name="test", nb=3)),
1)
self.assertEqual((yield self.numprocesses("numprocesses")), 1)
if IS_WINDOWS:
# On Windows we can't set an env to a process without some keys
env = dict(os.environ)
else:
env = {}
env['test'] = 2
self.assertEqual((yield self.set("test", env=env)), 'error')
env['test'] = '2'
self.assertEqual((yield self.set("test", env=env)), 'ok')
resp = yield self.call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEqual(options.get('env', {}), env)
resp = yield self.call('stats', name='test')
self.assertEqual(resp['status'], 'ok')
resp = yield self.call('globaloptions', name='test')
self.assertEqual(resp['options']['pubsub_endpoint'],
self.arbiter.pubsub_endpoint)
yield self.stop_arbiter()
def test_client(self):
# playing around with the watcher
yield self.start_arbiter()
msg = make_message("numwatchers")
resp = yield self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
self.assertEqual((yield self.numprocesses("numprocesses")), 1)
self.assertEqual((yield self.set("test", numprocesses=2)), 'ok')
self.assertEqual((yield self.numprocesses("numprocesses")), 2)
self.assertEqual((yield self.set("test", numprocesses=1)), 'ok')
self.assertEqual((yield self.numprocesses("numprocesses")), 1)
self.assertEqual((yield self.numwatchers("numwatchers")), 1)
self.assertEqual((yield self.call("list")).get('watchers'), ['test'])
self.assertEqual((yield self.numprocesses("incr", name="test")), 2)
self.assertEqual((yield self.numprocesses("numprocesses")), 2)
self.assertEqual((yield self.numprocesses("incr", name="test", nb=2)),
4)
self.assertEqual((yield self.numprocesses("decr", name="test", nb=3)),
1)
self.assertEqual((yield self.numprocesses("numprocesses")), 1)
self.assertEqual((yield self.set("test", env={"test": 2})),
'error')
self.assertEqual((yield self.set("test", env={"test": '2'})),
'ok')
resp = yield self.call('get', name='test', keys=['env'])
options = resp.get('options', {})
self.assertEqual(options.get('env'), {'test': '2'})
resp = yield self.call('stats', name='test')
self.assertEqual(resp['status'], 'ok')
resp = yield self.call('globaloptions', name='test')
self.assertEqual(resp['options']['pubsub_endpoint'],
self.arbiter.pubsub_endpoint)
yield self.stop_arbiter()
def call(cmd, **props):
msg = make_message(cmd, **props)
return client.call(msg)
def test_watchers(self):
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test"])
def call(self, _cmd, **props):
msg = make_message(_cmd, **props)
resp = yield self.cli.call(msg)
raise tornado.gen.Return(resp)
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get("status"), "stopped")
def send_message(self, command, callback=None, **props):
return self.call(make_message(command, **props), callback)
def test_stop(self):
resp = self.cli.call(make_message("quit"))
self.assertEqual(resp.get("status"), "ok")
self.assertRaises(CallError, self.cli.call, make_message("list"))
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers2(self):
self.cli.call(make_message("stop", name="test"))
resp = self.cli.call(make_message("status", name="test"))
self.assertEqual(resp.get('status'), "stopped")
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def call(self, _cmd, **props):
msg = make_message(_cmd, **props)
resp = yield self.cli.call(msg)
raise tornado.gen.Return(resp)
def test_reload(self):
resp = self.cli.call(make_message("reload"))
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher4(self):
cmd, args = self._get_cmd_args()
msg = make_message("add", name="test1", cmd=cmd, args=args)
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")
def test_stop_watchers(self):
resp = self.cli.call(make_message("stop"))
self.assertEqual(resp.get("status"), "ok")
def test_add_watcher3(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(msg)
self.assertTrue(resp.get('status'), 'error')
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def test_add_watcher2(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 2)
def test_numprocesses(self):
msg = make_message("numprocesses")
resp = self.cli.call(msg)
self.assertEqual(resp.get("numprocesses"), 1)
def test_add_watcher1(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
self.cli.call(msg)
resp = self.cli.call(make_message("list"))
self.assertEqual(resp.get('watchers'), ["test", "test1"])
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def test_add_watcher(self):
msg = make_message("add", name="test1", cmd=self._get_cmd())
resp = self.cli.call(msg)
self.assertEqual(resp.get("status"), "ok")