def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
def get_pids():
return self.call('list', name='test').get('pids')
pids = get_pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(self.status('signal', name='test', pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = get_pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def testSignal(self):
self.assertEquals(self.numprocesses("incr", name="test"), 2)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 2])
self.assertEquals(self.status("signal", name="test", process=2,
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertEquals(self.call("list", name="test").get('processes'),
[1, 3])
processes = self.call("list", name="test").get('processes')
self.assertEquals(self.status("signal", name="test",
signum=signal.SIGKILL), "ok")
time.sleep(1.0)
self.assertNotEqual(self.call("list", name="test").get('processes'),
processes)
def testStats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
self.assertEqual(resp['test']['1']['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(2.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_copy_path(self):
stream = QueueStream()
qstream = {'stream': stream}
old_environ = os.environ
old_paths = sys.path[:]
try:
sys.path = ['XYZ']
os.environ = {'COCONUTS': 'MIGRATE'}
cmd = ('%s -c "import sys; '
'sys.stdout.write(\':\'.join(sys.path)); '
' sys.stdout.flush()"') % sys.executable
watcher = Watcher('xx', cmd, copy_env=True, copy_path=True,
stdout_stream=qstream)
watcher.start()
time.sleep(3.)
watcher.stop()
data = [v for k, v in stream.get().items()][1]
data = ''.join(data)
self.assertTrue('XYZ' in data, data)
finally:
os.environ = old_environ
sys.path[:] = old_paths
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(dummy_process,
stdout_stream={'stream': self.stream})
self.arbiter = self.arbiters[-1]
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def pids(self):
return self.call('list', name='test').get('pids')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
pids = self.pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(self.status('signal', name='test', pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = self.pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_unexisting(self):
watcher = self.arbiter.get_watcher("test")
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
to_kill = process.pid
# the process is killed in an unsual way
os.kill(to_kill, signal.SIGSEGV)
# and wait for it to die
try:
pid, status = os.waitpid(to_kill, 0)
except OSError:
pass
# ansure the old process is considered "unexisting"
self.assertEquals(process.status, UNEXISTING)
# this should clean up and create a new process
watcher.reap_and_manage_processes()
# we should have a new process here now
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
# and that one needs to have a new pid.
self.assertNotEqual(process.pid, to_kill)
# and should not be unexisting...
self.assertNotEqual(process.status, UNEXISTING)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_max_age(self):
result = self.call('set', name='test',
options={'max_age': 1, 'max_age_variance': 0})
self.assertEquals(result.get('status'), 'ok')
initial_pids = self.pids()
time.sleep(3.0) # allow process to reach max_age and restart
current_pids = self.pids()
self.assertEqual(len(current_pids), 1)
self.assertNotEqual(initial_pids, current_pids)
def test_arbiter_reference(self):
self.assertEqual(self.arbiters[0].watchers[0].arbiter,
self.arbiters[0])
def test_handler(self):
stream = QueueStream()
cmd = 'circus.tests.test_command_signal.run_process_recursive'
stdout_stream = {'stream': stream}
stderr_stream = {'stream': stream}
yield self.start_arbiter(cmd=cmd,
stdout_stream=stdout_stream,
stderr_stream=stderr_stream,
stats=True,
stop_signal=signal.SIGINT,
debug=False)
def assert_read(channel, *values):
for value in values:
data = yield read_from_stream(stream, channel)
self.assertEqual(data, value)
# waiting for all processes to start
for c in (0, 1, 2, 11, 12):
assert_read(c, 'STARTED')
# checking that our system is live and running
client = AsyncCircusClient(endpoint=self.arbiter.endpoint)
res = yield client.send_message('list')
watchers = sorted(res['watchers'])
self.assertEqual(['circusd-stats', 'test'], watchers)
# send USR1 to parent only
res = yield client.send_message('signal', name='test', signum='usr1')
self.assertEqual(res['status'], 'ok')
assert_read(0, 'USR1')
# send USR2 to children only
res = yield client.send_message('signal',
name='test',
signum='usr2',
children=True)
self.assertEqual(res['status'], 'ok')
for c in (1, 2):
assert_read(c, 'USR2')
# send HUP to parent and children
res = yield client.send_message('signal',
name='test',
signum='hup',
recursive=True)
self.assertEqual(res['status'], 'ok')
for c in (0, 1, 2, 11, 12):
assert_read(c, 'HUP')
# stop process
res = yield client.send_message('stop', name='test')
self.assertEqual(res['status'], 'ok')
assert_read(0, 'INT', 'EXITING')
for c in (1, 2, 11, 12):
assert_read(c, 'TERM', 'EXITING')
timeout = time.time() + 5
stopped = False
while time.time() < timeout:
res = yield client.send_message('status', name='test')
if res['status'] == 'stopped':
stopped = True
break
self.assertEqual(res['status'], 'stopping')
self.assertTrue(stopped)
yield self.stop_arbiter()
def __init__(self, **kw):
threading.Thread.__init__(self)
self.stream = QueueStream()
self.loop = self.watcher = None
self.kw = kw
def setUp(self):
super(KillCommandTest, self).setUp()
self.stream = QueueStream()
self.reader = StreamReader(self.stream)
self._client = None
class TestWatcher(TestCircus):
def setUp(self):
super(TestWatcher, self).setUp()
self.stream = QueueStream()
dummy_process = 'circus.tests.test_watcher.run_process'
self.test_file = self._run_circus(
dummy_process, stdout_stream={'stream': self.stream})
self.arbiter = self.arbiters[-1]
self.cli = CircusClient()
def call(self, cmd, **props):
msg = make_message(cmd, **props)
return self.cli.call(msg)
def tearDown(self):
super(TestWatcher, self).tearDown()
self.cli.stop()
def status(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('status')
def numprocesses(self, cmd, **props):
resp = self.call(cmd, **props)
return resp.get('numprocesses')
def pids(self):
return self.call('list', name='test').get('pids')
def test_signal(self):
self.assertEquals(self.numprocesses('incr', name='test'), 2)
pids = self.pids()
self.assertEquals(len(pids), 2)
to_kill = pids[0]
self.assertEquals(
self.status('signal',
name='test',
pid=to_kill,
signum=signal.SIGKILL), 'ok')
time.sleep(1) # wait for the process to die
# we still should have two processes, but not the same pids for them
pids = self.pids()
self.assertEquals(len(pids), 2)
self.assertTrue(to_kill not in pids)
def test_unexisting(self):
watcher = self.arbiter.get_watcher("test")
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
to_kill = process.pid
# the process is killed in an unsual way
os.kill(to_kill, signal.SIGSEGV)
# and wait for it to die
pid, status = os.waitpid(to_kill, 0)
# ansure the old process is considered "unexisting"
self.assertEquals(process.status, UNEXISTING)
# this should clean up and create a new process
watcher.reap_and_manage_processes()
# we should have a new process here now
self.assertEquals(len(watcher.processes), 1)
process = watcher.processes.values()[0]
# and that one needs to have a new pid.
self.assertNotEqual(process.pid, to_kill)
# and should not be unexisting...
self.assertNotEqual(process.status, UNEXISTING)
def test_stats(self):
resp = self.call("stats").get('infos')
self.assertTrue("test" in resp)
watchers = resp['test']
self.assertEqual(watchers[watchers.keys()[0]]['cmdline'],
sys.executable.split(os.sep)[-1])
def test_streams(self):
time.sleep(1.)
# let's see what we got
self.assertTrue(self.stream.qsize() > 1)
def test_max_age(self):
result = self.call('set',
name='test',
options={
'max_age': 1,
'max_age_variance': 0
})
self.assertEquals(result.get('status'), 'ok')
initial_pids = self.pids()
time.sleep(3.0) # allow process to reach max_age and restart
current_pids = self.pids()
self.assertEqual(len(current_pids), 1)
self.assertNotEqual(initial_pids, current_pids)
def run_with_hooks(self, hooks):
self.stream = QueueStream()
dummy_process = 'circus.tests.support.run_process'
self._run_circus(dummy_process,
stdout_stream={'stream': self.stream},
hooks=hooks)
class KillCommandTest(TestCircus):
@skipIf(IS_WINDOWS, "Streams not supported")
def setUp(self):
super(KillCommandTest, self).setUp()
self.stream = QueueStream()
self.reader = StreamReader(self.stream)
self._client = None
def tearDown(self):
self._client.stop()
self.stream.close()
@property
def client(self):
if not self._client:
self._client = AsyncCircusClient(endpoint=self.arbiter.endpoint)
return self._client
@tornado.gen.coroutine
def assertMessage(self, msg, timeout=5):
try:
actual = yield self.reader.read(timeout=timeout)
except TimeoutException:
raise AssertionError(
'Timeout while waiting for message: {}'.format(msg))
self.assertEqual(actual, msg)
@tornado.testing.gen_test
def test_exits_within_graceful_timeout(self):
yield self.start_arbiter(
cmd='circus.tests.test_command_kill.obedient_process',
stdout_stream={'stream': self.stream},
stderr_stream={'stream': self.stream})
yield self.assertMessage('STARTED')
res = yield self.client.send_message('kill',
name='test',
signum='sigint',
graceful_timeout=0.1,
waiting=True)
self.assertEqual(res['status'], 'ok')
yield self.assertMessage('SIGINT')
yield self.assertMessage('STOPPED')
yield self.stop_arbiter()
@tornado.testing.gen_test
def test_kills_after_graceful_timeout(self):
yield self.start_arbiter(
cmd='circus.tests.test_command_kill.hanged_process',
stdout_stream={'stream': self.stream},
stderr_stream={'stream': self.stream})
yield self.assertMessage('STARTED')
res = yield self.client.send_message('kill',
name='test',
signum='sigint',
graceful_timeout=0.1,
waiting=True)
self.assertEqual(res['status'], 'ok')
yield self.assertMessage('SIGINT')
yield self.stop_arbiter()
def test_handler(self):
stream = QueueStream()
cmd = 'circus.tests.test_command_signal.run_process'
stdout_stream = {'stream': stream}
stderr_stream = {'stream': stream}
yield self.start_arbiter(cmd=cmd, stdout_stream=stdout_stream,
stderr_stream=stderr_stream, stats=True,
stop_signal=signal.SIGINT)
# waiting for data to appear in the queue
data = yield read_from_stream(stream, 0)
self.assertEqual('STARTED', data)
# waiting for children
data = yield read_from_stream(stream, 3)
self.assertEqual('STARTED', data)
data = yield read_from_stream(stream, 2)
self.assertEqual('STARTED', data)
data = yield read_from_stream(stream, 1)
self.assertEqual('STARTED', data)
# checking that our system is live and running
client = AsyncCircusClient()
res = yield client.send_message('list')
watchers = sorted(res['watchers'])
self.assertEqual(['circusd-stats', 'test'], watchers)
# send USR1 to parent only
res = yield client.send_message('signal', name='test', signum='usr1')
self.assertEqual(res['status'], 'ok')
res = yield read_from_stream(stream, 0)
self.assertEqual(res, 'USR1')
# send USR2 to children only
res = yield client.send_message('signal', name='test', signum='usr2',
children=True)
self.assertEqual(res['status'], 'ok')
res = yield read_from_stream(stream, 1)
self.assertEqual(res, 'USR2')
res = yield read_from_stream(stream, 2)
self.assertEqual(res, 'USR2')
res = yield read_from_stream(stream, 3)
self.assertEqual(res, 'USR2')
# send HUP to parent and children
res = yield client.send_message('signal', name='test', signum='hup',
recursive=True)
self.assertEqual(res['status'], 'ok')
res = yield read_from_stream(stream, 0)
self.assertEqual(res, 'HUP')
res = yield read_from_stream(stream, 1)
self.assertEqual(res, 'HUP')
res = yield read_from_stream(stream, 2)
self.assertEqual(res, 'HUP')
res = yield read_from_stream(stream, 3)
self.assertEqual(res, 'HUP')
# stop process
res = yield client.send_message('stop', name='test')
self.assertEqual(res['status'], 'ok')
res = yield read_from_stream(stream, 0)
self.assertEqual(res, 'INT')
res = yield read_from_stream(stream, 0)
self.assertEqual(res, 'EXITING')
res = yield read_from_stream(stream, 1)
self.assertEqual(res, 'TERM')
res = yield read_from_stream(stream, 1)
self.assertEqual(res, 'EXITING')
res = yield read_from_stream(stream, 2)
self.assertEqual(res, 'TERM')
res = yield read_from_stream(stream, 2)
self.assertEqual(res, 'EXITING')
res = yield read_from_stream(stream, 3)
self.assertEqual(res, 'TERM')
res = yield read_from_stream(stream, 3)
self.assertEqual(res, 'EXITING')
timeout = time.time() + 5
stopped = False
while time.time() < timeout:
res = yield client.send_message('status', name='test')
if res['status'] == 'stopped':
stopped = True
break
self.assertEqual(res['status'], 'stopping')
self.assertTrue(stopped)
yield self.stop_arbiter()
def setUp(self):
super(KillCommandTest, self).setUp()
self.stream = QueueStream()
self.reader = StreamReader(self.stream)
self._client = None
class KillCommandTest(TestCircus):
@skipIf(IS_WINDOWS, "Streams not supported")
def setUp(self):
super(KillCommandTest, self).setUp()
self.stream = QueueStream()
self.reader = StreamReader(self.stream)
self._client = None
def tearDown(self):
self._client.stop()
self.stream.close()
@property
def client(self):
if not self._client:
self._client = AsyncCircusClient(endpoint=self.arbiter.endpoint)
return self._client
@tornado.gen.coroutine
def assertMessage(self, msg, timeout=5):
try:
actual = yield self.reader.read(timeout=timeout)
except TimeoutException:
raise AssertionError('Timeout while waiting for message: {}'
.format(msg))
self.assertEqual(actual, msg)
@tornado.testing.gen_test
def test_exits_within_graceful_timeout(self):
yield self.start_arbiter(
cmd='circus.tests.test_command_kill.obedient_process',
stdout_stream={'stream': self.stream},
stderr_stream={'stream': self.stream})
yield self.assertMessage('STARTED')
res = yield self.client.send_message(
'kill', name='test', signum='sigint',
graceful_timeout=0.1, waiting=True)
self.assertEqual(res['status'], 'ok')
yield self.assertMessage('SIGINT')
yield self.assertMessage('STOPPED')
yield self.stop_arbiter()
@tornado.testing.gen_test
def test_kills_after_graceful_timeout(self):
yield self.start_arbiter(
cmd='circus.tests.test_command_kill.hanged_process',
stdout_stream={'stream': self.stream},
stderr_stream={'stream': self.stream})
yield self.assertMessage('STARTED')
res = yield self.client.send_message(
'kill', name='test', signum='sigint',
graceful_timeout=0.1, waiting=True)
self.assertEqual(res['status'], 'ok')
yield self.assertMessage('SIGINT')
yield self.stop_arbiter()
