def test_stop_and_restart(self): # cf https://github.com/circus-tent/circus/issues/912 yield self._start_arbiter() # wait for the process to be started res1 = yield async_poll_for(self.stdout, 'stdout') res2 = yield async_poll_for(self.stderr, 'stderr') self.assertTrue(res1) self.assertTrue(res2) self.assertFalse(self.stdout_stream._file.closed) self.assertFalse(self.stderr_stream._file.closed) # clean slate truncate_file(self.stdout) truncate_file(self.stderr) # stop the watcher yield self.arbiter.watchers[0].stop() self.assertTrue(self.stdout_stream._file.closed) self.assertTrue(self.stderr_stream._file.closed) # start it again yield self.arbiter.watchers[0].start() # wait for the process to be restarted res1 = yield async_poll_for(self.stdout, 'stdout') res2 = yield async_poll_for(self.stderr, 'stderr') self.assertTrue(res1) self.assertTrue(res2) self.assertFalse(self.stdout_stream._file.closed) self.assertFalse(self.stderr_stream._file.closed) yield self.stop_arbiter()
def test_reload_wid_1_worker(self): yield self.start_arbiter(graceful_timeout=0) resp = yield self._call("stats", name="test") processes1 = list(resp['info'].keys()) self.assertEqual(len(processes1), 1) wids1 = [resp['info'][process]['wid'] for process in processes1] self.assertEqual(wids1, [1]) truncate_file(self.test_file) # clean slate yield self._call("reload") res = yield async_poll_for(self.test_file, 'START') # restarted self.assertTrue(res) resp = yield self._call("stats", name="test") processes2 = list(resp['info'].keys()) self.assertEqual(len(processes2), 1) self.assertNotEqual(processes1, processes2) wids2 = [resp['info'][process]['wid'] for process in processes2] self.assertEqual(wids2, [2]) truncate_file(self.test_file) # clean slate yield self._call("reload") res = yield async_poll_for(self.test_file, 'START') # restarted self.assertTrue(res) resp = yield self._call("stats", name="test") processes3 = list(resp['info'].keys()) self.assertEqual(len(processes3), 1) self.assertNotIn(processes3[0], (processes1[0], processes2[0])) wids3 = [resp['info'][process]['wid'] for process in processes3] self.assertEqual(wids3, [1]) yield self.stop_arbiter()
def test_stats(self): envs = { "TSURU_METRICS_BACKEND": "fake", } os.environ.update(envs) dummy_process = 'circus.tests.support.run_process' yield self.start_arbiter(dummy_process) async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.2} stats_class = Stats stats_class.disk_usage = lambda x: 0 stats_class.net_io = lambda x: (0, 0) stats_class.connections_established = lambda x: 0 stats_class.connections = lambda x: [] gauges = yield async_run_plugin( stats_class, config, plugin_info_callback=get_gauges, duration=1000, endpoint=self.arbiter.endpoint, pubsub_endpoint=self.arbiter.pubsub_endpoint ) # we should have a bunch of stats events here self.assertTrue(len(gauges) >= 5) last_batch = sorted(name for name, value in gauges[-5:]) wanted = ['test.cpu_sum', 'test.mem_max', 'test.mem_pct_max', 'test.mem_pct_sum', 'test.mem_sum'] self.assertEqual(last_batch, wanted) yield self.stop_arbiter()
def test_signal(self): yield self.start_arbiter(check_delay=1.0) resp = yield self.numprocesses('incr', name='test') self.assertEqual(resp, 2) # wait for both to have started resp = yield async_poll_for(self.test_file, 'STARTSTART') self.assertTrue(resp) truncate_file(self.test_file) pids = yield self.pids() self.assertEqual(len(pids), 2) to_kill = pids[0] status = yield self.status('signal', name='test', pid=to_kill, signum=SIGKILL) self.assertEqual(status, 'ok') # make sure the process is restarted res = yield async_poll_for(self.test_file, 'START') self.assertTrue(res) # we still should have two processes, but not the same pids for them pids = yield self.pids() count = 0 while len(pids) < 2 and count < 10: pids = yield self.pids() time.sleep(.1) self.assertEqual(len(pids), 2) self.assertTrue(to_kill not in pids) yield self.stop_arbiter()
def _test_plugins(self): fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin dummy_process = 'circus.tests.support.run_process' plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin, 'file': datafile}] self._run_circus(dummy_process, plugins=plugins) # doing a few operations def nb_processes(): return len(cli.send_message('list', name='test').get('pids')) def incr_processes(): return cli.send_message('incr', name='test') # wait for the plugin to be started self.assertTrue(async_poll_for(datafile, 'PLUGIN STARTED')) cli = CircusClient() self.assertEqual(nb_processes(), 1) incr_processes() self.assertEqual(nb_processes(), 2) # wait for the plugin to receive the signal self.assertTrue(async_poll_for(datafile, 'test:spawn')) truncate_file(datafile) incr_processes() self.assertEqual(nb_processes(), 3) # wait for the plugin to receive the signal self.assertTrue(async_poll_for(datafile, 'test:spawn'))
def test_reload_wid_4_workers(self): yield self.start_arbiter(graceful_timeout=0) resp = yield self._call("incr", name="test", nb=3) self.assertEqual(resp.get('numprocesses'), 4) resp = yield self._call("stats", name="test") processes1 = list(resp['info'].keys()) self.assertEqual(len(processes1), 4) wids1 = set(resp['info'][process]['wid'] for process in processes1) self.assertSetEqual(wids1, set([1, 2, 3, 4])) truncate_file(self.test_file) # clean slate yield self._call("reload") self.assertTrue(async_poll_for(self.test_file, 'START')) # restarted resp = yield self._call("stats", name="test") processes2 = list(resp['info'].keys()) self.assertEqual(len(processes2), 4) self.assertEqual(len(set(processes1) & set(processes2)), 0) wids2 = set(resp['info'][process]['wid'] for process in processes2) self.assertSetEqual(wids2, set([5, 6, 7, 8])) truncate_file(self.test_file) # clean slate yield self._call("reload") self.assertTrue(async_poll_for(self.test_file, 'START')) # restarted resp = yield self._call("stats", name="test") processes3 = list(resp['info'].keys()) self.assertEqual(len(processes3), 4) self.assertEqual(len(set(processes1) & set(processes3)), 0) self.assertEqual(len(set(processes2) & set(processes3)), 0) wids3 = set([resp['info'][process]['wid'] for process in processes3]) self.assertSetEqual(wids3, set([1, 2, 3, 4])) yield self.stop_arbiter()
def test_watchdog_discovery_not_found(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, "START") config = {"loop_rate": 0.1, "watchers_regex": "^foo.*$"} with warnings.catch_warnings(): pid_status = yield async_run_plugin(WatchDog, config, get_pid_status) self.assertEqual(len(pid_status), 0, pid_status) yield self.stop_arbiter()
def test_handler(self): yield self.start_arbiter() # wait for the process to be started self.assertTrue(async_poll_for(self.test_file, 'START')) # stopping... yield self.arbiter.stop() # wait for the process to be stopped self.assertTrue(async_poll_for(self.test_file, 'QUIT'))
def test_watchdog_discovery_found(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, 'STARTWD') config = {'loop_rate': 0.1, 'watchers_regex': "^test.*$"} with warnings.catch_warnings(): pid_status = yield async_run_plugin(WatchDog, config, get_pid_status) self.assertEqual(len(pid_status), 1, pid_status) yield self.stop_arbiter() async_poll_for(self.test_file, 'STOPWD')
def test_resource_watcher_min_mem_abs(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.1, 'min_mem': '100M', 'watcher': 'test'} statsd_increments = yield async_run_plugin(ResourceWatcher, config, get_statsd_increments) self._check_statsd(statsd_increments, '_resource_watcher.test.under_memory') yield self.stop_arbiter()
def test_watchdog_discovery_not_found(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, "START") pubsub = self.arbiter.pubsub_endpoint config = {"loop_rate": 0.1, "watchers_regex": "^foo.*$"} with warnings.catch_warnings(): pid_status = yield arp( WatchDog, config, get_pid_status, endpoint=self.arbiter.endpoint, pubsub_endpoint=pubsub ) self.assertEqual(len(pid_status), 0, pid_status) yield self.stop_arbiter()
def test_resource_watcher_max_cpu(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.1, 'max_cpu': 0.1, 'watcher': 'test'} statsd_increments = yield async_run_plugin(ResourceWatcher, config, get_statsd_increments) self._check_statsd(statsd_increments, '_resource_watcher.test.over_cpu') yield self.stop_arbiter()
def test_add_start(self): yield self.start_arbiter() async_poll_for(self.test_file, 'START') stdout, stderr = yield async_run_ctl('add --start test2 "sleep 1"') if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(stdout, 'ok\n') stdout, stderr = yield async_run_ctl('status test2') if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(stdout, 'active\n') yield self.stop_arbiter()
def test_add_start(self): yield self.start_arbiter() async_poll_for(self.test_file, "START") stdout, stderr = yield async_run_ctl('add --start test2 "sleep 1"') if stderr: self.assertIn("UserWarning", stderr) self.assertEqual(stdout, "ok\n") stdout, stderr = yield async_run_ctl("status test2") if stderr: self.assertIn("UserWarning", stderr) self.assertEqual(stdout, "active\n") yield self.stop_arbiter()
def test_resource_watcher_min_mem(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.1, 'min_mem': 100000.1, 'watcher': 'test'} kw = {'endpoint': self.arbiter.endpoint, 'pubsub_endpoint': self.arbiter.pubsub_endpoint} statsd_increments = yield async_run_plugin(ResourceWatcher, config, get_statsd_increments, **kw) self._check_statsd(statsd_increments, '_resource_watcher.test.under_memory') yield self.stop_arbiter()
def test_launch_cli(self): yield self.start_arbiter() async_poll_for(self.test_file, 'START') stdout, stderr = yield self.run_ctl(endpoint=self.arbiter.endpoint) if stderr: self.assertIn('UserWarning', stderr) output = stdout.splitlines() self.assertEqual(output[0], VERSION) # strip of term escape characters, if any prompt = output[2][-len(CircusCtl.prompt):] self.assertEqual(prompt, CircusCtl.prompt) yield self.stop_arbiter()
def test_watchdog_discovery_found(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, 'STARTWD') pubsub = self.arbiter.pubsub_endpoint config = {'loop_rate': 0.1, 'watchers_regex': "^test.*$"} with warnings.catch_warnings(): pid_status = yield arp(WatchDog, config, get_pid_status, endpoint=self.arbiter.endpoint, pubsub_endpoint=pubsub) self.assertEqual(len(pid_status), 1, pid_status) yield self.stop_arbiter() async_poll_for(self.test_file, 'STOPWD')
def test_resource_watcher_min_mem(self): yield self.start_arbiter(fqn) async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.1, 'min_mem': 100000.1, 'watcher': 'test'} kw = { 'endpoint': self.arbiter.endpoint, 'pubsub_endpoint': self.arbiter.pubsub_endpoint } statsd_increments = yield async_run_plugin(ResourceWatcher, config, get_statsd_increments, **kw) self._check_statsd(statsd_increments, '_resource_watcher.test.under_memory') yield self.stop_arbiter()
def test_add_start(self): yield self.start_arbiter() async_poll_for(self.test_file, 'START') ep = self.arbiter.endpoint stdout, stderr = yield async_run_ctl('add --start test2 "sleep 1"', endpoint=ep) if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(stdout, 'ok\n') stdout, stderr = yield async_run_ctl('status test2', endpoint=ep) if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(stdout, 'active\n') yield self.stop_arbiter()
def test_launch_cli(self): yield self.start_arbiter() async_poll_for(self.test_file, 'START') stdout, stderr = yield self.run_ctl(endpoint=self.arbiter.endpoint) if stderr: self.assertIn('UserWarning', stderr) output = stdout.splitlines() self.assertEqual(output[0], VERSION) # strip off term escape characters, if any if not output[2].startswith(CircusCtl.prompt): prompt = output[2][-len(CircusCtl.prompt):] self.assertEqual(prompt, CircusCtl.prompt) yield self.stop_arbiter()
def test_add(self): yield self.start_arbiter() async_poll_for(self.test_file, 'START') ep = self.arbiter.endpoint stdout, stderr = yield async_run_ctl('add test2 "%s"' % SLEEP % 1, endpoint=ep) if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(stdout.strip(), 'ok') stdout, stderr = yield async_run_ctl('status test2', endpoint=ep) if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(stdout.strip(), 'stopped') yield self.stop_arbiter()
def test_full_stats(self): dummy_process = 'circus.tests.support.run_process' yield self.start_arbiter(dummy_process) yield async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.2} gauges = yield async_run_plugin( FullStats, config, plugin_info_callback=get_gauges, duration=1000, endpoint=self.arbiter.endpoint, pubsub_endpoint=self.arbiter.pubsub_endpoint) # we should have a bunch of stats events here self.assertTrue(len(gauges) >= 5) last_batch = sorted(name for name, value in gauges[-5:]) wanted = [ '_stats.test.cpu_sum', '_stats.test.mem_max', '_stats.test.mem_pct_max', '_stats.test.mem_pct_sum', '_stats.test.mem_sum' ] self.assertEqual(last_batch, wanted) yield self.stop_arbiter()
def test_plugins(self): fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin, 'file': datafile}] yield self.start_arbiter(graceful_timeout=0, plugins=plugins, loop=get_ioloop()) def incr_processes(cli): # return a coroutine if cli is Async return cli.send_message('incr', name='test') # wait for the plugin to be started res = yield async_poll_for(datafile, 'PLUGIN STARTED') self.assertTrue(res) cli = AsyncCircusClient(endpoint=self.arbiter.endpoint) res = yield cli.send_message('list', name='test') self.assertEqual(len(res.get('pids')), 1) yield incr_processes(cli) res = yield cli.send_message('list', name='test') self.assertEqual(len(res.get('pids')), 2) # wait for the plugin to receive the signal res = yield async_poll_for(datafile, 'test:spawn') self.assertTrue(res) truncate_file(datafile) yield incr_processes(cli) res = yield cli.send_message('list', name='test') self.assertEqual(len(res.get('pids')), 3) # wait for the plugin to receive the signal res = yield async_poll_for(datafile, 'test:spawn') self.assertTrue(res) os.remove(datafile) yield self.stop_arbiter()
def test_full_stats(self): dummy_process = 'circus.tests.support.run_process' yield self.start_arbiter(dummy_process) async_poll_for(self.test_file, 'START') config = {'loop_rate': 0.2} gauges = yield async_run_plugin( FullStats, config, plugin_info_callback=get_gauges, duration=1000) # we should have a bunch of stats events here self.assertTrue(len(gauges) >= 5) last_batch = sorted(name for name, value in gauges[-5:]) wanted = ['_stats.test.cpu_max', '_stats.test.cpu_sum', '_stats.test.mem_max', '_stats.test.mem_sum', '_stats.test.watchers_num'] self.assertEqual(last_batch, wanted) yield self.stop_arbiter()
def test_stream(self): yield self._start_arbiter() # wait for the process to be started res1 = yield async_poll_for(self.stdout, 'stdout') res2 = yield async_poll_for(self.stderr, 'stderr') self.assertTrue(res1) self.assertTrue(res2) # clean slate truncate_file(self.stdout) truncate_file(self.stderr) # restart and make sure streams are still working yield self.restart_arbiter() # wait for the process to be restarted res1 = yield async_poll_for(self.stdout, 'stdout') res2 = yield async_poll_for(self.stderr, 'stderr') self.assertTrue(res1) self.assertTrue(res2) yield self.stop_arbiter()
def test_command_already_running(self): yield self.start_arbiter() yield async_poll_for(self.test_file, 'START') with patch.object(self.arbiter, '_exclusive_running_command', 'foo'): retcode, stdout, stderr = yield async_run_ctl( 'restart', endpoint=self.arbiter.endpoint) self.assertEqual(retcode, 3) self.assertEqual(stdout.strip(), '') self.assertEqual(stderr.strip(), 'error: arbiter is already running foo command')
def test_plugins(self): fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin, 'file': datafile}] yield self.start_arbiter(graceful_timeout=0, plugins=plugins, loop=get_ioloop()) def incr_processes(cli): return cli.send_message('incr', name='test') # wait for the plugin to be started res = yield async_poll_for(datafile, 'PLUGIN STARTED') self.assertTrue(res) cli = AsyncCircusClient(endpoint=self.arbiter.endpoint) res = yield cli.send_message('list', name='test') self.assertEqual(len(res.get('pids')), 1) incr_processes(cli) res = yield cli.send_message('list', name='test') self.assertEqual(len(res.get('pids')), 2) # wait for the plugin to receive the signal res = yield async_poll_for(datafile, 'test:spawn') self.assertTrue(res) truncate_file(datafile) incr_processes(cli) res = yield cli.send_message('list', name='test') self.assertEqual(len(res.get('pids')), 3) # wait for the plugin to receive the signal res = yield async_poll_for(datafile, 'test:spawn') self.assertTrue(res) os.remove(datafile) yield self.stop_arbiter()
def test_watchdog_discovery_not_found(self): yield self.start_arbiter(fqn) yield async_poll_for(self.test_file, 'START') pubsub = self.arbiter.pubsub_endpoint config = {'loop_rate': 0.1, 'watchers_regex': "^foo.*$"} with warnings.catch_warnings(): pid_status = yield arp(WatchDog, config, get_pid_status, endpoint=self.arbiter.endpoint, pubsub_endpoint=pubsub) self.assertEqual(len(pid_status), 0, pid_status) yield self.stop_arbiter()
def test_max_age(self): yield self.start_arbiter() # let's run 15 processes yield self.numprocesses('incr', name='test', nb=14) initial_pids = yield self.pids() # we want to make sure the watcher is really up and running 14 # processes, and stable async_poll_for(self.test_file, 'START' * 15) truncate_file(self.test_file) # make sure we have a clean slate # we want a max age of 1 sec. options = {'max_age': 1, 'max_age_variance': 0} result = yield self.call('set', name='test', waiting=True, options=options) self.assertEqual(result.get('status'), 'ok') current_pids = yield self.pids() self.assertEqual(len(current_pids), 15) self.assertNotEqual(initial_pids, current_pids) yield self.stop_arbiter()
def test_reload_wid_4_workers(self): yield self.start_arbiter(graceful_timeout=0) resp = yield self._call("incr", name="test", nb=3) self.assertEqual(resp.get('numprocesses'), 4) resp = yield self._call("stats", name="test") processes1 = list(resp['info'].keys()) self.assertEqual(len(processes1), 4) wids1 = set(resp['info'][process]['wid'] for process in processes1) self.assertSetEqual(wids1, set([1, 2, 3, 4])) truncate_file(self.test_file) # clean slate yield self._call("reload") res = yield async_poll_for(self.test_file, 'START') # restarted self.assertTrue(res) resp = yield self._call("stats", name="test") processes2 = list(resp['info'].keys()) self.assertEqual(len(processes2), 4) self.assertEqual(len(set(processes1) & set(processes2)), 0) wids2 = set(resp['info'][process]['wid'] for process in processes2) self.assertSetEqual(wids2, set([5, 6, 7, 8])) truncate_file(self.test_file) # clean slate yield self._call("reload") res = yield async_poll_for(self.test_file, 'START') # restarted self.assertTrue(res) resp = yield self._call("stats", name="test") processes3 = list(resp['info'].keys()) self.assertEqual(len(processes3), 4) self.assertEqual(len(set(processes1) & set(processes3)), 0) self.assertEqual(len(set(processes2) & set(processes3)), 0) wids3 = set([resp['info'][process]['wid'] for process in processes3]) self.assertSetEqual(wids3, set([1, 2, 3, 4])) yield self.stop_arbiter()
def _test_plugins(self): fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin dummy_process = 'circus.tests.support.run_process' plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin, 'file': datafile}] self._run_circus(dummy_process, plugins=plugins) # doing a few operations def nb_processes(): return len(cli.send_message('list', name='test').get('pids')) def incr_processes(): return cli.send_message('incr', name='test') # wait for the plugin to be started res = yield async_poll_for(datafile, 'PLUGIN STARTED') self.assertTrue(res) cli = CircusClient() self.assertEqual(nb_processes(), 1) incr_processes() self.assertEqual(nb_processes(), 2) # wait for the plugin to receive the signal res = yield async_poll_for(datafile, 'test:spawn') self.assertTrue(res) truncate_file(datafile) incr_processes() self.assertEqual(nb_processes(), 3) # wait for the plugin to receive the signal res = yield async_poll_for(datafile, 'test:spawn') self.assertTrue(res) os.remove(datafile)
def test_reload_sequential(self): yield self.start_arbiter(graceful_timeout=0) name = 'test_reload_sequential' options = self._get_options(numprocesses=4) yield self._call("add", name=name, cmd=self._get_cmd(), start=True, options=options) resp = yield self._call("list", name=name) processes1 = resp.get('pids') truncate_file(self.test_file) # clean slate yield self._call("reload", sequential=True) self.assertTrue(async_poll_for(self.test_file, 'START')) # restarted resp = yield self._call("list", name=name) processes2 = resp.get('pids') self.assertNotEqual(processes1, processes2) yield self.stop_arbiter()
def test_reload2(self): yield self.start_arbiter(graceful_timeout=0) resp = yield self._call("list", name="test") processes1 = resp.get('pids') self.assertEqual(len(processes1), 1) truncate_file(self.test_file) # clean slate yield self._call("reload") self.assertTrue(async_poll_for(self.test_file, 'START')) # restarted resp = yield self._call("list", name="test") processes2 = resp.get('pids') self.assertEqual(len(processes2), 1) self.assertNotEqual(processes1[0], processes2[0]) yield self.stop_arbiter()
def test_add_start(self): yield self.start_arbiter() yield async_poll_for(self.test_file, 'START') ep = self.arbiter.endpoint retcode, stdout, stderr = yield async_run_ctl( 'add --start test2 "%s"' % SLEEP % 1, endpoint=ep) if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(retcode, 0) self.assertEqual(stdout.strip(), 'ok') retcode, stdout, stderr = yield async_run_ctl( 'status test2', endpoint=ep) if stderr: self.assertIn('UserWarning', stderr) self.assertEqual(retcode, 0) self.assertEqual(stdout.strip(), 'active') yield self.stop_arbiter()
def test_reload_uppercase(self): yield self.start_arbiter(graceful_timeout=0) name = 'test_RELOAD' yield self._call("add", name=name, cmd=self._get_cmd(), start=True, options=self._get_options()) resp = yield self._call("list", name=name) processes1 = resp.get('pids') truncate_file(self.test_file) # clean slate yield self._call("reload") res = yield async_poll_for(self.test_file, 'START') self.assertTrue(res) # restarted resp = yield self._call("list", name=name) processes2 = resp.get('pids') self.assertNotEqual(processes1, processes2) yield self.stop_arbiter()
def test_dummy(self): yield self.start_arbiter('circus.tests.test_runner.Dummy') self.assertTrue(async_poll_for(self.test_file, '..........')) yield self.stop_arbiter()