def handle_dealer(self, cmd, opts, msg, endpoint, timeout, ssh_server, ssh_keyfile): if endpoint is not None: client = CircusClient(endpoint=endpoint, timeout=timeout, ssh_server=ssh_server, ssh_keyfile=ssh_keyfile) else: client = self.client try: if isinstance(msg, list): for i, command in enumerate(msg): clm = self._console(client, command['cmd'], opts, command['msg']) print("%s: %s" % (i, clm)) else: print(self._console(client, cmd, opts, msg)) except CallError as e: sys.stderr.write(str(e) + " Try to raise the --timeout value\n") return 1 finally: if endpoint is not None: client.stop() return 0
def __init__(self, endpoint, pubsub_endoint, stats_endpoint, ssh_server, delay=1.): self.topic = 'watcher.' self.delay = delay self.ctx = zmq.Context() self.pubsub_endpoint = pubsub_endoint self.sub_socket = self.ctx.socket(zmq.SUB) self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic) self.sub_socket.connect(self.pubsub_endpoint) self.loop = ioloop.IOLoop.instance() # events coming from circusd self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop) self.substream.on_recv(self.handle_recv) self.client = CircusClient(context=self.ctx, endpoint=endpoint, ssh_server=ssh_server) self.cmds = get_commands() self._pids = defaultdict(list) self._callbacks = dict() self.publisher = StatsPublisher(stats_endpoint, self.ctx) self.running = False # should the streamer be running? self.stopped = False # did the collect started yet? self.circus_pids = {} self.sockets = []
def handle_dealer(self, command, opts, msg, endpoint, timeout, ssh_server, ssh_keyfile): if endpoint is not None: client = CircusClient(endpoint=endpoint, timeout=timeout, ssh_server=ssh_server, ssh_keyfile=ssh_keyfile) else: client = self.client try: if isinstance(msg, list): for i, c in enumerate(msg): clm = self._console(client, c['cmd'], opts, c['msg']) print("%s: %s" % (i, clm)) else: print(self._console(client, command, opts, msg)) except CallError as e: msg = str(e) if 'timed out' in str(e).lower(): msg += TIMEOUT_MSG sys.stderr.write(msg) return 1 finally: if endpoint is not None: client.stop() return 0
def circus_status(endpoint=None, process=None): default = { 'pid': 'unknown', 'status': 'unknown', 'uptime': 'unknown' } if endpoint and process: client = CircusClient(endpoint=endpoint, timeout=2) try: status = client.send_message('status') stats = client.send_message('stats') # Assuming here there's only a process pid = stats['infos'][process].keys()[0] try: uptime = int(stats['infos'][process][pid]['age']) default['uptime'] = humanize.naturaltime(datetime.datetime.now() - datetime.timedelta(seconds=uptime)) default['pid'] = pid except: # circus running but process stopped pass default['status'] = status['statuses'][process].lower() except Exception as exc: if'TIMED OUT' in exc.message.upper(): # circus stopped default['status'] = 'unknown' return default
def setUp(self): super(TestWatcher, self).setUp() self.stream = QueueStream() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus( dummy_process, stdout_stream={'stream': self.stream}) self.cli = CircusClient()
class TestCircus(unittest.TestCase): def setUp(self): self.arbiters = [] self.files = [] self.tmpfiles = [] self.cli = CircusClient() def tearDown(self): self._stop_runners() for file in self.files + self.tmpfiles: if os.path.exists(file): os.remove(file) self.cli.stop() def get_tmpfile(self, content=None): fd, file = mkstemp() os.close(fd) self.tmpfiles.append(file) if content is not None: with open(file, 'w') as f: f.write(content) return file @classmethod def _create_circus(cls, callable, plugins=None, stats=False, **kw): resolve_name(callable) # used to check the callable fd, testfile = mkstemp() os.close(fd) wdir = os.path.dirname(__file__) args = ['generic.py', callable, testfile] worker = {'cmd': _CMD, 'args': args, 'working_dir': wdir, 'name': 'test', 'graceful_timeout': 4} worker.update(kw) if stats: arbiter = get_arbiter([worker], background=True, plugins=plugins, stats_endpoint=DEFAULT_ENDPOINT_STATS, debug=kw.get('debug', False)) else: arbiter = get_arbiter([worker], background=True, plugins=plugins, debug=kw.get('debug', False)) arbiter.start() return testfile, arbiter def _run_circus(self, callable, plugins=None, stats=False, **kw): testfile, arbiter = TestCircus._create_circus(callable, plugins, stats, **kw) self.arbiters.append(arbiter) self.files.append(testfile) return testfile def _stop_runners(self): for arbiter in self.arbiters: arbiter.stop() self.arbiters = [] def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg)
def handle_dealer(self, cmd, opts, msg, endpoint, timeout, ssh_server, ssh_keyfile): if endpoint is not None: client = CircusClient(endpoint=endpoint, timeout=timeout, ssh_server=ssh_server, ssh_keyfile=ssh_keyfile) else: client = self.client try: if isinstance(msg, list): for i, command in enumerate(msg): clm = self._console(client, command['cmd'], opts, command['msg']) print("%s: %s" % (i, clm)) else: print(self._console(client, cmd, opts, msg)) except CallError as e: msg = str(e) if 'timed out' in str(e).lower(): msg += TIMEOUT_MSG sys.stderr.write(msg) return 1 finally: if endpoint is not None: client.stop() return 0
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() self.stream = QueueStream() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus( dummy_process, stdout_stream={'stream': self.stream}) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() def status(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('status') def numprocesses(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('numprocesses') def test_signal(self): self.assertEquals(self.numprocesses('incr', name='test'), 2) def get_pids(): return self.call('list', name='test').get('pids') pids = get_pids() self.assertEquals(len(pids), 2) to_kill = pids[0] self.assertEquals( self.status('signal', name='test', pid=to_kill, signum=signal.SIGKILL), 'ok') time.sleep(1) # wait for the process to die # we still should have two processes, but not the same pids for them pids = get_pids() self.assertEquals(len(pids), 2) self.assertTrue(to_kill not in pids) def test_stats(self): resp = self.call("stats").get('infos') self.assertTrue("test" in resp) watchers = resp['test'] self.assertEqual(watchers[watchers.keys()[0]]['cmdline'], sys.executable.split(os.sep)[-1]) def test_streams(self): time.sleep(1.) # let's see what we got self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() self.stream = QueueStream() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus( dummy_process, stdout_stream={'stream': self.stream}) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() def status(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('status') def numprocesses(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('numprocesses') def testSignal(self): self.assertEquals(self.numprocesses("incr", name="test"), 2) self.assertEquals( self.call("list", name="test").get('processes'), [1, 2]) self.assertEquals( self.status("signal", name="test", process=2, signum=signal.SIGKILL), "ok") time.sleep(1.0) self.assertEquals( self.call("list", name="test").get('processes'), [1, 3]) processes = self.call("list", name="test").get('processes') self.assertEquals( self.status("signal", name="test", signum=signal.SIGKILL), "ok") time.sleep(1.0) self.assertNotEqual( self.call("list", name="test").get('processes'), processes) def testStats(self): resp = self.call("stats").get('infos') self.assertTrue("test" in resp) self.assertEqual(resp['test']['1']['cmdline'], sys.executable.split(os.sep)[-1]) def test_streams(self): time.sleep(2.) # let's see what we got self.assertTrue(self.stream.qsize() > 1)
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() self.stream = QueueStream() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus(dummy_process, stdout_stream={'stream': self.stream}) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() def status(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('status') def numprocesses(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('numprocesses') def test_signal(self): self.assertEquals(self.numprocesses('incr', name='test'), 2) def get_pids(): return self.call('list', name='test').get('pids') pids = get_pids() self.assertEquals(len(pids), 2) to_kill = pids[0] self.assertEquals(self.status('signal', name='test', pid=to_kill, signum=signal.SIGKILL), 'ok') time.sleep(1) # wait for the process to die # we still should have two processes, but not the same pids for them pids = get_pids() self.assertEquals(len(pids), 2) self.assertTrue(to_kill not in pids) def test_stats(self): resp = self.call("stats").get('infos') self.assertTrue("test" in resp) watchers = resp['test'] self.assertEqual(watchers[watchers.keys()[0]]['cmdline'], sys.executable.split(os.sep)[-1]) def test_streams(self): time.sleep(1.) # let's see what we got self.assertTrue(self.stream.qsize() > 1)
def _send_message(command, **properties): # check if circusct.endpoint is in minion config endpoint = __salt__['config.get']('circusctl.endpoint') or \ DEFAULT_ENDPOINT_DEALER # sending keys with None values in the message to circus will result # an error. removing them from properties props = dict((k, v) for k, v in properties.iteritems() if v) client = CircusClient(endpoint=endpoint) return client.send_message(command, **props)
def __init__(self, endpoint): self.endpoint = str(endpoint) self.stats_endpoint = None self.client = CircusClient(endpoint=self.endpoint) self.connected = False self.watchers = [] self.stats = defaultdict(list) self.refresher = Refresher(self) self.dstats = []
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() self.stream = QueueStream() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus(dummy_process, stdout_stream={'stream': self.stream}) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() def status(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('status') def numprocesses(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('numprocesses') def testSignal(self): self.assertEquals(self.numprocesses("incr", name="test"), 2) self.assertEquals(self.call("list", name="test").get('processes'), [1, 2]) self.assertEquals(self.status("signal", name="test", process=2, signum=signal.SIGKILL), "ok") time.sleep(1.0) self.assertEquals(self.call("list", name="test").get('processes'), [1, 3]) processes = self.call("list", name="test").get('processes') self.assertEquals(self.status("signal", name="test", signum=signal.SIGKILL), "ok") time.sleep(1.0) self.assertNotEqual(self.call("list", name="test").get('processes'), processes) def testStats(self): resp = self.call("stats").get('infos') self.assertTrue("test" in resp) self.assertEqual(resp['test']['1']['cmdline'], sys.executable.split(os.sep)[-1]) def test_streams(self): time.sleep(2.) # let's see what we got self.assertTrue(self.stream.qsize() > 1)
def _rm_process(self, name): client = CircusClient(endpoint=self.circus_endpoint) # https://github.com/circus-tent/circus/issues/927 name = name.lower() cmd_rm = {"command": "rm", "properties": {"name": name}} try: call_rm = client.call(cmd_rm) self.log.debug("_rm_device circus client call: %s", str(call_rm)) except CallError: self.log.debug("Could not rm process: %s", name, exc_info=True)
def test_singleton(self): self._stop_runners() dummy_process = 'circus.tests.support.run_process' self._run_circus(dummy_process, singleton=True) cli = CircusClient() # adding more than one process should fail res = cli.send_message('incr', name='test') self.assertEqual(res['numprocesses'], 1)
def _test_singleton(self): yield self._stop_runners() dummy_process = 'circus.tests.support.run_process' self._run_circus(dummy_process, singleton=True) cli = CircusClient() # adding more than one process should fail res = cli.send_message('incr', name='test') self.assertEqual(res['numprocesses'], 1)
def test_singleton(self): self._stop_runners() dummy_process = "circus.tests.test_arbiter.run_dummy" self._run_circus(dummy_process, singleton=True) cli = CircusClient() # adding more than one process should fail res = cli.send_message("incr", name="test") self.assertEqual(res["numprocesses"], 1)
def test_handler(self): self._run_circus('circus.tests.test_client.run_process') time.sleep(.5) # playing around with the watcher client = CircusClient() def call(cmd, **props): msg = make_message(cmd, **props) return client.call(msg) def status(cmd, **props): resp = call(cmd, **props) return resp.get('status') def numprocesses(cmd, **props): resp = call(cmd, **props) return resp.get('numprocesses') def numwatchers(cmd, **props): resp = call(cmd, **props) return resp.get('numwatchers') def set(name, **opts): return status("set", name=name, options=opts) self.assertEquals(set("test", numprocesses=10), 'ok') self.assertEquals(numprocesses("numprocesses"), 10) self.assertEquals(set("test", numprocesses=1), 'ok') self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(numwatchers("numwatchers"), 1) self.assertEquals(call("list").get('watchers'), ['test']) self.assertEquals(call("list", name="test").get('processes'), [10]) self.assertEquals(numprocesses("incr", name="test"), 2) self.assertEquals(numprocesses("numprocesses"), 2) self.assertEquals(numprocesses("decr", name="test"), 1) self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error') self.assertEquals(set("test", env={"test": '1', "test": '2'}), 'ok') resp = call('get', name='test', keys=['env']) options = resp.get('options', {}) self.assertEquals(options.get('env'), {'test': '1', 'test': '2'}) resp = call('stats', name='test') self.assertEqual(resp['status'], 'ok') resp = call('globaloptions', name='test') self.assertEqual(resp['options']['pubsub_endpoint'], 'tcp://127.0.0.1:5556') client.stop()
def _client_test(self, ssh_server): test_file = self._run_circus('circus.tests.support.run_process') self.assertTrue(poll_for(test_file, 'START')) # process started # playing around with the watcher client = CircusClient(ssh_server=ssh_server) def call(cmd, **props): msg = make_message(cmd, **props) return client.call(msg) def status(cmd, **props): resp = call(cmd, **props) return resp.get('status') def numprocesses(cmd, **props): resp = call(cmd, **props) return resp.get('numprocesses') def numwatchers(cmd, **props): resp = call(cmd, **props) return resp.get('numwatchers') def set(name, **opts): return status("set", name=name, options=opts) self.assertEquals(set("test", numprocesses=10), 'ok') self.assertEquals(numprocesses("numprocesses"), 10) self.assertEquals(set("test", numprocesses=1), 'ok') self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(numwatchers("numwatchers"), 1) self.assertEquals(call("list").get('watchers'), ['test']) self.assertEquals(numprocesses("incr", name="test"), 2) self.assertEquals(numprocesses("numprocesses"), 2) self.assertEquals(numprocesses("incr", name="test", nb=2), 4) self.assertEquals(numprocesses("decr", name="test", nb=3), 1) self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error') self.assertEquals(set("test", env={"test": '1', "test": '2'}), 'ok') resp = call('get', name='test', keys=['env']) options = resp.get('options', {}) self.assertEquals(options.get('env'), {'test': '1', 'test': '2'}) resp = call('stats', name='test') self.assertEqual(resp['status'], 'ok') resp = call('globaloptions', name='test') self.assertEqual(resp['options']['pubsub_endpoint'], 'tcp://127.0.0.1:5556') client.stop()
def _running(self): """Return Brewpi instances running using suffix as filter""" client = CircusClient(endpoint=self.circus_endpoint) try: call = client.call({"command": "list", "properties": {}}) except CallError: self.log.error("Could not get running processes", exc_info=True) return [] running_devices = [ x for x in call['watchers'] if x.startswith(self.prefix) ] return running_devices
def _test_stop(self): resp = self.cli.call(make_message("quit")) self.assertEqual(resp.get("status"), "ok") self.assertRaises(CallError, self.cli.call, make_message("list")) self._stop_runners() cli = CircusClient() dummy_process = 'circus.tests.support.run_process' self.test_file = self._run_circus(dummy_process) msg = make_message("numprocesses") resp = cli.call(msg) self.assertEqual(resp.get("status"), "ok")
def __init__(self, host, port, timeout=15): assert type(host) == str assert type(port) == int and port >= 0 and port <= 65535 assert type(timeout) == int and timeout > 0 self._host = host self._port = port self._timeout = timeout self._arbiter = get_arbiter([]) self._arbiter.start() self._client = CircusClient(timeout=self._timeout, endpoint='tcp://{0}:{1}'.format( self._host, self._port))
def _client_test(self, ssh_server): self._run_circus('circus.tests.support.run_process') time.sleep(.5) # playing around with the watcher client = CircusClient(ssh_server=ssh_server) def call(cmd, **props): msg = make_message(cmd, **props) return client.call(msg) def status(cmd, **props): resp = call(cmd, **props) return resp.get('status') def numprocesses(cmd, **props): resp = call(cmd, **props) return resp.get('numprocesses') def numwatchers(cmd, **props): resp = call(cmd, **props) return resp.get('numwatchers') def set(name, **opts): return status("set", name=name, options=opts) self.assertEquals(set("test", numprocesses=10), 'ok') self.assertEquals(numprocesses("numprocesses"), 10) self.assertEquals(set("test", numprocesses=1), 'ok') self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(numwatchers("numwatchers"), 1) self.assertEquals(call("list").get('watchers'), ['test']) self.assertEquals(numprocesses("incr", name="test"), 2) self.assertEquals(numprocesses("numprocesses"), 2) self.assertEquals(numprocesses("incr", name="test", nb=2), 4) self.assertEquals(numprocesses("decr", name="test", nb=3), 1) self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error') self.assertEquals(set("test", env={"test": '1', "test": '2'}), 'ok') resp = call('get', name='test', keys=['env']) options = resp.get('options', {}) self.assertEquals(options.get('env'), {'test': '1', 'test': '2'}) resp = call('stats', name='test') self.assertEqual(resp['status'], 'ok') resp = call('globaloptions', name='test') self.assertEqual(resp['options']['pubsub_endpoint'], 'tcp://127.0.0.1:5556') client.stop()
def setUp(self): super(TestWatcher, self).setUp() dummy_process = 'circus.tests.test_stream.run_process' fd, self.stdout = tempfile.mkstemp() os.close(fd) fd, self.stderr = tempfile.mkstemp() os.close(fd) self.test_file = self._run_circus( dummy_process, stdout_stream={'stream': FileStream(self.stdout)}, stderr_stream={'stream': FileStream(self.stderr)}, debug=True) self.cli = CircusClient()
def __init__(self, endpoint, ssh_server=None): self.endpoint = str(endpoint) self.stats_endpoint = None self.client = CircusClient(endpoint=self.endpoint, ssh_server=ssh_server) self.connected = False self.watchers = [] self.plugins = [] self.stats = defaultdict(list) self.dstats = [] self.sockets = None self.use_sockets = False self.embed_httpd = False
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus(dummy_process) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() def status(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('status') def numprocesses(self, cmd, **props): resp = self.call(cmd, **props) return resp.get('numprocesses') def testSignal(self): self.assertEquals(self.numprocesses("incr", name="test"), 2) self.assertEquals(self.call("list", name="test").get('processes'), [1, 2]) self.assertEquals(self.status("signal", name="test", process=2, signum=signal.SIGKILL), "ok") time.sleep(1.0) self.assertEquals(self.call("list", name="test").get('processes'), [1, 3]) processes = self.call("list", name="test").get('processes') self.assertEquals(self.status("signal", name="test", signum=signal.SIGKILL), "ok") time.sleep(1.0) self.assertNotEqual(self.call("list", name="test").get('processes'), processes) def testStats(self): resp = self.call("stats").get('infos') self.assertTrue("test" in resp) self.assertEqual(resp['test']['1']['cmdline'], 'python')
def handle_dealer(self, cmd, opts, msg, endpoint, timeout): client = CircusClient(endpoint=endpoint, timeout=timeout) try: if isinstance(msg, list): for i, cmd in enumerate(msg): clm = self._console(client, cmd, opts, msg) print("%s: %s" % (i, clm)) else: print(self._console(client, cmd, opts, msg)) except CallError as e: sys.stderr.write(str(e)) return 1 finally: client.stop() return 0
def __init__(self, endpoint): self.endpoint = str(endpoint) self.client = CircusClient(endpoint=self.endpoint) self.connected = False self.watchers = [] self.stats = defaultdict(list) self.refresher = Refresher(self)
def test_plugins(self): # killing the setUp runner self._stop_runners() self.cli.stop() fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin dummy_process = 'circus.tests.support.run_process' plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin, 'file': datafile}] self._run_circus(dummy_process, plugins=plugins) # doing a few operations def nb_processes(): return len(cli.send_message('list', name='test').get('pids')) def incr_processes(): return cli.send_message('incr', name='test') # wait for the plugin to be started self.assertTrue(poll_for(datafile, 'PLUGIN STARTED')) cli = CircusClient() self.assertEqual(nb_processes(), 1) incr_processes() self.assertEqual(nb_processes(), 2) # wait for the plugin to receive the signal self.assertTrue(poll_for(datafile, 'test:spawn')) truncate_file(datafile) incr_processes() self.assertEqual(nb_processes(), 3) # wait for the plugin to receive the signal self.assertTrue(poll_for(datafile, 'test:spawn'))
def setUp(self): super(TestWatcher, self).setUp() self.stream = QueueStream() dummy_process = 'circus.tests.test_watcher.run_process' self.test_file = self._run_circus(dummy_process, stdout_stream={'stream': self.stream}) self.cli = CircusClient()
def handle_dealer(self, cmd, opts, msg, endpoint, timeout): client = CircusClient(endpoint=endpoint, timeout=timeout) try: if isinstance(msg, list): for i, command in enumerate(msg): clm = self._console(client, command['cmd'], opts, command['msg']) print("%s: %s" % (i, clm)) else: print(self._console(client, cmd, opts, msg)) except CallError as e: sys.stderr.write(str(e)) return 1 finally: client.stop() return 0
def test_handler(self): self._run_circus('circus.tests.test_client.run_process') time.sleep(.5) # playing around with the watcher client = CircusClient() def call(cmd, **props): msg = make_message(cmd, **props) return client.call(msg) def status(cmd, **props): resp = call(cmd, **props) return resp.get('status') def numprocesses(cmd, **props): resp = call(cmd, **props) return resp.get('numprocesses') def numwatchers(cmd, **props): resp = call(cmd, **props) return resp.get('numwatchers') def set(name, **opts): return status("set", name=name, options=opts) self.assertEquals(set("test", numprocesses=10), 'ok') self.assertEquals(numprocesses("numprocesses"), 10) self.assertEquals(set("test", numprocesses=1), 'ok') self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(numwatchers("numwatchers"), 1) self.assertEquals(call("list").get('watchers'), ['test']) self.assertEquals(call("list", name="test").get('processes'), [10]) self.assertEquals(numprocesses("incr", name="test"), 2) self.assertEquals(numprocesses("numprocesses"), 2) self.assertEquals(numprocesses("decr", name="test"), 1) self.assertEquals(numprocesses("numprocesses"), 1) self.assertEquals(set("test", env={"test": 1, "test": 2}), 'error') self.assertEquals(set("test", env={"test": '1', "test": '2'}), 'ok') resp = call('get', name='test', keys=['env']) options = resp.get('options', {}) self.assertEquals(options.get('env'), {'test': '1', 'test': '2'}) client.stop()
def __init__(self, endpoint, pubsub_endoint, stats_endpoint, ssh_server=None, delay=1., loop=None): self.topic = b'watcher.' self.delay = delay self.ctx = zmq.Context() self.pubsub_endpoint = pubsub_endoint self.sub_socket = self.ctx.socket(zmq.SUB) self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic) self.sub_socket.connect(self.pubsub_endpoint) self.loop = loop or ioloop.IOLoop.current() self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop) self.substream.on_recv(self.handle_recv) self.client = CircusClient(context=self.ctx, endpoint=endpoint, ssh_server=ssh_server) self.cmds = get_commands() self.publisher = StatsPublisher(stats_endpoint, self.ctx) self._initialize()
def test_handler(self): log = self._get_file() stream = {'stream': FileStream(log)} self._run_circus('circus.tests.test_stats_client.run_process', stdout_stream=stream, stderr_stream=stream, stats=True) time.sleep(.5) # checking that our system is live and running client = CircusClient() res = client.send_message('list') watchers = res['watchers'] watchers.sort() self.assertEqual(['circusd-stats', 'test'], watchers) # making sure the stats process run res = client.send_message('status', name='test') self.assertEqual(res['status'], 'active') res = client.send_message('status', name='circusd-stats') self.assertEqual(res['status'], 'active') # playing around with the stats now: we should get some ! from circus.stats.client import StatsClient client = StatsClient() next = client.iter_messages().next for i in range(10): watcher, pid, stat = next() self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'), watcher)
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() dummy_process = 'circus.tests.test_stream.run_process' fd, log = tempfile.mkstemp() self.log = log os.close(fd) stream = {'stream': FileStream(log)} self.test_file = self._run_circus(dummy_process, stdout_stream=stream, stderr_stream=stream, debug=True) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() os.remove(self.log) def test_stream(self): time.sleep(2.) self.call("stats").get('infos') # let's see what we got in the file with open(self.log) as f: data = f.read() self.assertTrue('stderr' in data) self.assertTrue('stdout' in data) # restarting self.call('restart') time.sleep(1.) current = time.time() # should be running with open(self.log) as f: data = f.readlines() # last log should be less than one second old last = data[-1] delta = abs(current - int(last.split('-')[0])) self.assertTrue(delta < 1, delta)
def _stop_process(self, name): client = CircusClient(endpoint=self.circus_endpoint) # https://github.com/circus-tent/circus/issues/927 name = name.lower() cmd_stop = { "command": "stop", "properties": { "waiting": False, "name": name, "match": "glob" } } try: call_stop = client.call(cmd_stop) self.log.debug("_stop_process circus client call: %s", str(call_stop)) except CallError: self.log.debug("Could not stop process: %s", name, exc_info=True)
def __init__(self, endpoint): self.endpoint = str(endpoint) self.stats_endpoint = None self.client = CircusClient(endpoint=self.endpoint) self.connected = False self.watchers = [] self.plugins = [] self.stats = defaultdict(list) self.dstats = []
def client(self): """ Return an instance of the CircusClient with the endpoint defined by the controller endpoint, which used the port that was written to the port file upon starting of the daemon :return: CircucClient instance """ from circus.client import CircusClient return CircusClient(endpoint=self.get_controller_endpoint(), timeout=self._DAEMON_TIMEOUT)
def setUp(self): super(TestWatcher, self).setUp() dummy_process = 'circus.tests.test_stream.run_process' fd, log = tempfile.mkstemp() self.log = log os.close(fd) stream = {'stream': FileStream(log)} self.test_file = self._run_circus(dummy_process, stdout_stream=stream, stderr_stream=stream, debug=True) self.cli = CircusClient()
def __init__(self, endpoint, pubsub_endoint, stats_endpoint): self.topic = 'watcher.' self.ctx = zmq.Context() self.pubsub_endpoint = pubsub_endoint self.sub_socket = self.ctx.socket(zmq.SUB) self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic) self.sub_socket.connect(self.pubsub_endpoint) self.loop = ioloop.IOLoop() self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop) self.substream.on_recv(self.handle_recv) self.client = CircusClient(context=self.ctx, endpoint=endpoint) self.cmds = get_commands() self.watchers = defaultdict(list) self._pids = defaultdict(list) self.running = False self.stopped = False self.lock = threading.RLock() self.results = Queue.Queue() self.stats = StatsCollector(self) self.publisher = StatsPublisher(self, stats_endpoint, context=self.ctx)
def __init__(self, *args, **config): super(ProcfileWatcher, self).__init__(*args, **config) self.loop_rate = config.get("loop_rate", 3) # in seconds self.procfile_path = config.get("app_path", "/home/application/current/Procfile") self.working_dir = config.get("working_dir", "/home/application/current") self.apprc = config.get("apprc", "/home/application/apprc") self.port = config.get("port", "8888") self.uid = config.get("uid", "ubuntu") self.stderr_stream = { "class": config.get("stderr_stream", "tsuru.stream.Stream") } self.stdout_stream = { "class": config.get("stdout_stream", "tsuru.stream.Stream") } file_watcher = FileWatcher(self.procfile_path, self.reload_procfile) self.period = ioloop.PeriodicCallback(file_watcher, self.loop_rate * 1000, self.loop) self.circus_client = CircusClient()
def __init__(self, *args, **config): super(ProcfileWatcher, self).__init__(*args, **config) self.loop_rate = config.get("loop_rate", 60) # in seconds self.procfile_path = config.get("app_path", "/home/application/current/Procfile") self.working_dir = config.get("working_dir", "/home/application/current") self.port = config.get("port", "8888") self.uid = config.get("uid", "ubuntu") self.stderr_stream = {"class": config.get("stderr_stream", "tsuru.stream.Stream")} self.stdout_stream = {"class": config.get("stdout_stream", "tsuru.stream.Stream")} self.period = ioloop.PeriodicCallback(self.look_after, self.loop_rate * 1000, self.loop) self.circus_client = CircusClient()
def __init__(self, host, port, timeout=15): assert type(host) == str assert type(port) == int and port >= 0 and port <= 65535 assert type(timeout) == int and timeout > 0 self._host = host self._port = port self._timeout = timeout self._arbiter = get_arbiter([]) self._arbiter.start() self._client = CircusClient(timeout=self._timeout, endpoint='tcp://{0}:{1}'.format(self._host, self._port))
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() dummy_process = "circus.tests.test_stream.run_process" fd, self.stdout = tempfile.mkstemp() os.close(fd) fd, self.stderr = tempfile.mkstemp() os.close(fd) self.test_file = self._run_circus( dummy_process, stdout_stream={"stream": FileStream(self.stdout)}, stderr_stream={"stream": FileStream(self.stderr)}, debug=True, ) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() os.remove(self.stdout) os.remove(self.stderr) def test_stream(self): # wait for the process to be started self.assertTrue(poll_for(self.stdout, "stdout")) self.assertTrue(poll_for(self.stderr, "stderr")) # clean slate truncate_file(self.stdout) truncate_file(self.stderr) # restart and make sure streams are still working self.call("restart") # wait for the process to be restarted self.assertTrue(poll_for(self.stdout, "stdout")) self.assertTrue(poll_for(self.stderr, "stderr"))
class TestWatcher(TestCircus): def setUp(self): super(TestWatcher, self).setUp() dummy_process = 'circus.tests.test_stream.run_process' fd, self.stdout = tempfile.mkstemp() os.close(fd) fd, self.stderr = tempfile.mkstemp() os.close(fd) self.test_file = self._run_circus( dummy_process, stdout_stream={'stream': FileStream(self.stdout)}, stderr_stream={'stream': FileStream(self.stderr)}, debug=True) self.cli = CircusClient() def call(self, cmd, **props): msg = make_message(cmd, **props) return self.cli.call(msg) def tearDown(self): super(TestWatcher, self).tearDown() self.cli.stop() os.remove(self.stdout) os.remove(self.stderr) def test_stream(self): # wait for the process to be started self.assertTrue(poll_for(self.stdout, 'stdout')) self.assertTrue(poll_for(self.stderr, 'stderr')) # clean slate truncate_file(self.stdout) truncate_file(self.stderr) # restart and make sure streams are still working self.call('restart') # wait for the process to be restarted self.assertTrue(poll_for(self.stdout, 'stdout')) self.assertTrue(poll_for(self.stderr, 'stderr'))
def test_handler(self): if os.getenv('TRAVIS', False): return log = self._get_file() stream = {'stream': FileStream(log)} self._run_circus('circus.tests.test_stats_client.run_process', stdout_stream=stream, stderr_stream=stream, stats=True) time.sleep(.5) # checking that our system is live and running client = CircusClient() res = client.send_message('list') watchers = res['watchers'] watchers.sort() self.assertEqual(['circusd-stats', 'test'], watchers) # making sure the stats process run res = client.send_message('status', name='test') self.assertEqual(res['status'], 'active') res = client.send_message('status', name='circusd-stats') self.assertEqual(res['status'], 'active') # playing around with the stats now: we should get some ! from circus.stats.client import StatsClient client = StatsClient() next = client.iter_messages().next for i in range(10): watcher, pid, stat = next() self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'), watcher)
def client(self): """ Return an instance of the CircusClient with the endpoint defined by the controller endpoint, which used the port that was written to the port file upon starting of the daemon N.B. This is quite slow the first time it is run due to the import of zmq.ssh in circus/utils.py in circus 0.15.0, which ultimately follows the import of CircusClient. :return: CircusClient instance """ from circus.client import CircusClient return CircusClient(endpoint=self.get_controller_endpoint(), timeout=self._DAEMON_TIMEOUT)
def main(): # TODO, we should ask the server for its command list commands = get_commands() globalopts = parse_arguments(sys.argv[1:], commands) if globalopts['endpoint'] is None: globalopts['endpoint'] = os.environ.get('CIRCUSCTL_ENDPOINT', DEFAULT_ENDPOINT_DEALER) client = CircusClient(endpoint=globalopts['endpoint'], timeout=globalopts['timeout'], ssh_server=globalopts['ssh'], ssh_keyfile=globalopts['ssh_keyfile']) CircusCtl(client, commands).start(globalopts)
def setUp(self): super(TestWatcher, self).setUp() dummy_process = "circus.tests.test_stream.run_process" fd, self.stdout = tempfile.mkstemp() os.close(fd) fd, self.stderr = tempfile.mkstemp() os.close(fd) self.test_file = self._run_circus( dummy_process, stdout_stream={"stream": FileStream(self.stdout)}, stderr_stream={"stream": FileStream(self.stderr)}, debug=True, ) self.cli = CircusClient()
def _add_process(self, name): """Spawn a new brewpi.py process via circus, 'dev-' is appended to the name to keep brewpi devices seperated from other devices """ client = CircusClient(endpoint=self.circus_endpoint) proc_name = self.prefix + name # https://github.com/circus-tent/circus/issues/927 proc_name = proc_name.lower() try: call = client.call({ "command": "add", "properties": { "cmd": self.command_tmpl % name, "name": proc_name, "options": { "copy_env": True, "stdout_stream": { "class": "FileStream", "filename": "%s/%s-stdout.log" % (self.logfilepath, proc_name), }, "stderr_stream": { "class": "FileStream", "filename": "%s/%s-stderr.log" % (self.logfilepath, proc_name), } }, "start": True } }) self.log.debug("_add_process circus client call: %s", str(call)) except CallError: self.log.error("Could not spawn process: %s", proc_name, exc_info=True)
def test_plugins(self): # killing the setUp runner self._stop_runners() self.cli.stop() # setting up a circusd with a plugin dummy_process = 'circus.tests.test_arbiter.run_dummy' plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin}] self._run_circus(dummy_process, plugins=plugins) # doing a few operations cli = CircusClient() msg1 = make_message("list", name="test") resp = cli.call(msg1) self.assertEqual(resp.get('processes'), [1]) msg2 = make_message("incr", name="test") cli.call(msg2) resp = cli.call(msg1) self.assertEqual(resp.get('processes'), [1, 2]) # checking what the plugin did wanted = [('test', 'spawn'), ('test', 'start'), ('test', 'spawn')] self.assertEqual(Plugin.events, wanted)
def __init__(self, endpoint, pubsub_endoint, stats_endpoint, ssh_server=None, delay=1., loop=None): self.topic = b'watcher.' self.delay = delay self.ctx = zmq.Context() self.pubsub_endpoint = pubsub_endoint self.sub_socket = self.ctx.socket(zmq.SUB) self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic) self.sub_socket.connect(self.pubsub_endpoint) self.loop = loop or ioloop.IOLoop.instance() self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop) self.substream.on_recv(self.handle_recv) self.client = CircusClient(context=self.ctx, endpoint=endpoint, ssh_server=ssh_server) self.cmds = get_commands() self.publisher = StatsPublisher(stats_endpoint, self.ctx) self._initialize()
class CircusMgr(object): """Fermentrack Circus Handler, It is a simple wrapper around circus client, any errors raised as CircusException""" def __init__(self, connection_timeout=2, circus_endpoint=DEFAULT_ENDPOINT_DEALER): self._client = CircusClient( timeout=connection_timeout, endpoint=circus_endpoint) def _call(self, command, **props): message = {"command": command, "properties": props or {}} try: res = self._client.call(message) except CallError, callerr: LOG.debug("Error from circus", exc_info=True) raise CircusException("Could send message to circus: {}".format(callerr)) if res['status'] == u'error': raise CircusException("Error: {}".format(res['reason'])) return res
def test_plugins(self): # killing the setUp runner self._stop_runners() self.cli.stop() fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin dummy_process = 'circus.tests.test_arbiter.run_dummy' plugin = 'circus.tests.test_arbiter.Plugin' plugins = [{'use': plugin, 'file': datafile}] self._run_circus(dummy_process, plugins=plugins) # doing a few operations cli = CircusClient() msg1 = make_message("list", name="test") resp = cli.call(msg1) self.assertEqual(resp.get('processes'), [1]) msg2 = make_message("incr", name="test") cli.call(msg2) resp = cli.call(msg1) self.assertEqual(resp.get('processes'), [1, 2]) cli.call(msg2) resp = cli.call(msg1) self.assertEqual(resp.get('processes'), [1, 2, 3]) # wait a bit time.sleep(.2) # checking what the plugin did with open(datafile) as f: data = [line for line in f.read().split('\n') if line != ''] wanted = ['test:spawn', 'test:spawn'] self.assertEqual(data, wanted)
def __init__(self, endpoint, pubsub_endoint, stats_endpoint, delay=1.): self.topic = 'watcher.' self.delay = delay self.ctx = zmq.Context() self.pubsub_endpoint = pubsub_endoint self.sub_socket = self.ctx.socket(zmq.SUB) self.sub_socket.setsockopt(zmq.SUBSCRIBE, self.topic) self.sub_socket.connect(self.pubsub_endpoint) self.loop = ioloop.IOLoop() # events coming from circusd self.substream = zmqstream.ZMQStream(self.sub_socket, self.loop) self.substream.on_recv(self.handle_recv) self.client = CircusClient(context=self.ctx, endpoint=endpoint) self.cmds = get_commands() self._pids = defaultdict(list) self._callbacks = dict() self.collector = StatsCollector() self.publisher = StatsPublisher(stats_endpoint, self.ctx) self.running = False # should the streamer be running? self.stopped = False # did the collect started yet? self.circus_pids = {}
def test_plugins(self): # killing the setUp runner self._stop_runners() self.cli.stop() fd, datafile = mkstemp() os.close(fd) # setting up a circusd with a plugin dummy_process = "circus.tests.test_arbiter.run_dummy" plugin = "circus.tests.test_arbiter.Plugin" plugins = [{"use": plugin, "file": datafile}] self._run_circus(dummy_process, plugins=plugins) # doing a few operations cli = CircusClient() msg1 = make_message("list", name="test") resp = cli.call(msg1) self.assertEqual(resp.get("processes"), [1]) msg2 = make_message("incr", name="test") cli.call(msg2) resp = cli.call(msg1) self.assertEqual(resp.get("processes"), [1, 2]) cli.call(msg2) resp = cli.call(msg1) self.assertEqual(resp.get("processes"), [1, 2, 3]) # wait a bit time.sleep(0.2) # checking what the plugin did with open(datafile) as f: data = [line for line in f.read().split("\n") if line != ""] wanted = ["test:spawn", "test:spawn"] self.assertEqual(data, wanted)