def spawn_processes(self): """Spawn processes. """ # when an on_demand process dies, do not restart it until # the next event if self.pending_socket_event: self._status = "stopped" return for i in self._found_wids: self.spawn_process(i) yield tornado_sleep(0) self._found_wids = {} for i in range(self.numprocesses - len(self.processes)): res = self.spawn_process() if res is False: if len(self.async_killing_futures) == 0: yield self._stop(True) break delay = self.warmup_delay if isinstance(res, float): delay -= (time.time() - res) if delay < 0: delay = 0 yield tornado_sleep(delay)
def test_message_id(self): hooks = {'before_stop': ('circus.tests.test_client.long_hook', False)} testfile, arbiter = self.run_with_hooks(hooks) yield arbiter.start() try: self.assertTrue(os.path.exists(tmp_filename)) msg = make_message("numwatchers") resp = yield self.cli.call(msg) self.assertEqual(resp.get("numwatchers"), 1) # this should timeout resp = yield self.cli.call(make_message("stop")) self.assertEqual(resp.get('status'), 'ok') while arbiter.watchers[0].status() != 'stopped': yield tornado_sleep(.1) resp = yield self.cli.call(make_message("numwatchers")) self.assertEqual(resp.get("numwatchers"), 1) self.assertFalse(os.path.exists(tmp_filename)) finally: if os.path.exists(tmp_filename): os.unlink(tmp_filename) arbiter.stop()
def kill_process(self, process): """Kill process (stop_signal, graceful_timeout then SIGKILL) """ if process.stopping: raise gen.Return(False) logger.debug("%s: kill process %s", self.name, process.pid) if self.stop_children: self.send_signal_process(process, self.stop_signal) else: self.send_signal(process.pid, self.stop_signal) self.notify_event("kill", {"process_pid": process.pid, "time": time.time()}) process.stopping = True waited = 0 while waited < self.graceful_timeout: yield tornado_sleep(1) waited += 1 if not process.is_alive(): break if waited >= self.graceful_timeout: # We are not smart anymore self.send_signal_process(process, signal.SIGKILL) self._process_remove_redirections(process) process.stopping = False process.stop() raise gen.Return(True)
def kill_process(self, process): """Kill process (stop_signal, graceful_timeout then SIGKILL) """ if process.stopping: raise gen.Return(False) try: logger.debug("%s: kill process %s", self.name, process.pid) if self.stop_children: self.send_signal_process(process, self.stop_signal) else: self.send_signal(process.pid, self.stop_signal) self.notify_event("kill", {"process_pid": process.pid, "time": time.time()}) except NoSuchProcess: raise gen.Return(False) process.stopping = True waited = 0 while waited < self.graceful_timeout: if not process.is_alive(): break yield tornado_sleep(0.1) waited += 0.1 if waited >= self.graceful_timeout: # On Windows we can't send a SIGKILL signal, but the # process.stop function will terminate the process # later anyway if hasattr(signal, 'SIGKILL'): # We are not smart anymore self.send_signal_process(process, signal.SIGKILL) if self.stream_redirector: self.stream_redirector.remove_redirections(process) process.stopping = False process.stop() raise gen.Return(True)
def start_arbiter(self, cmd='support.run_process', stdout_stream=None, debug=True, **kw): testfile, arbiter = self._create_circus(cmd, stdout_stream=stdout_stream, debug=debug, use_async=True, **kw) self.test_file = testfile self.arbiter = arbiter self.arbiters.append(arbiter) for i in range(self.ADDRESS_IN_USE_TRY_TIMES): try: yield self.arbiter.start() except ZMQError as e: if e.strerror == 'Address already in use': # One more try to wait for the port being released by the OS yield tornado_sleep(0.1) continue else: raise e else: # Everything goes well, just break break else: # Cannot start after tries raise RuntimeError('Cannot start arbiter after %s times try' % self.ADDRESS_IN_USE_TRY_TIMES)
def _reload(self, graceful=True, sequential=False): """ reload """ if not graceful and sequential: logger.warn("with graceful=False, sequential=True is ignored") if self.prereload_fn is not None: self.prereload_fn(self) if not graceful: yield self._restart() return if self.is_stopped(): yield self._start() elif self.send_hup: for process in self.processes.values(): logger.info("SENDING HUP to %s" % process.pid) process.send_signal(signal.SIGHUP) else: if sequential: active_processes = self.get_active_processes() for process in active_processes: yield self.kill_process(process) self.reap_process(process.pid) self.spawn_process() yield tornado_sleep(self.warmup_delay) else: for i in range(self.numprocesses): self.spawn_process() yield self.manage_processes() self.notify_event("reload", {"time": time.time()}) logger.info('%s reloaded', self.name)
def kill_process(self, process): """Kill process (stop_signal, graceful_timeout then SIGKILL) """ if process.stopping: raise gen.Return(False) logger.debug("%s: kill process %s", self.name, process.pid) if self.stop_children: self.send_signal_process(process, self.stop_signal) else: self.send_signal(process.pid, self.stop_signal) self.notify_event("kill", { "process_pid": process.pid, "time": time.time() }) process.stopping = True waited = 0 while waited < self.graceful_timeout: yield tornado_sleep(1) waited += 1 if not process.is_alive(): break if waited >= self.graceful_timeout: # We are not smart anymore self.send_signal_process(process, signal.SIGKILL) self._process_remove_redirections(process) process.stopping = False process.stop() raise gen.Return(True)
def _start_watchers(self, watcher_iter_func=None): if watcher_iter_func is None: watchers = self.iter_watchers() else: watchers = watcher_iter_func() for watcher in watchers: if watcher.autostart: yield watcher._start() yield tornado_sleep(self.warmup_delay)
def async_run_plugin(klass, config, plugin_info_callback, duration=300): queue = multiprocessing.Queue() plugin_info_callback = functools.partial(plugin_info_callback, queue) circusctl_process = multiprocessing.Process(target=run_plugin, args=(klass, config, plugin_info_callback, duration)) circusctl_process.start() while queue.empty(): yield tornado_sleep(0.1) result = queue.get() raise tornado.gen.Return(result)
def read_from_stream(stream, timeout=5): start = time.time() while time.time() - start < timeout: try: data = stream.get_nowait() raise tornado.gen.Return(u(data['data'])) except Empty: yield tornado_sleep(0.1) raise TimeoutException('Timeout reading queue')
def async_run_plugin(klass, config, plugin_info_callback, duration=300): queue = multiprocessing.Queue() plugin_info_callback = functools.partial(plugin_info_callback, queue) circusctl_process = multiprocessing.Process(target=run_plugin, args=(klass, config, plugin_info_callback, duration)) circusctl_process.start() while queue.empty(): yield tornado_sleep(.1) result = queue.get() raise tornado.gen.Return(result)
def reload(self, graceful=True, sequential=False): """Reloads everything. Run the :func:`prereload_fn` callable if any, then gracefuly reload all watchers. """ if self._stopping: return if self.prereload_fn is not None: self.prereload_fn(self) # reopen log files for handler in logger.handlers: if isinstance(handler, logging.FileHandler): handler.acquire() handler.stream.close() handler.stream = open(handler.baseFilename, handler.mode) handler.release() # gracefully reload watchers for watcher in self.iter_watchers(): yield watcher._reload(graceful=graceful, sequential=sequential) tornado_sleep(self.warmup_delay)
def spawn_processes(self): """Spawn processes. """ # when an on_demand process dies, do not restart it until # the next event if self.pending_socket_event: self._status = "stopped" return for i in range(self.numprocesses - len(self.processes)): res = self.spawn_process() if res is False: yield self._stop() break yield tornado_sleep(self.warmup_delay)
def spawn_processes(self): """Spawn processes. """ # when an on_demand process dies, do not restart it until # the next event if self.on_demand and not self.arbiter.socket_event: self._status = "stopped" return for i in range(self.numprocesses - len(self.processes)): res = self.spawn_process() if res is False: yield self._stop() break yield tornado_sleep(self.warmup_delay)
def async_run_plugin(cls, config, plugin_info_callback, duration=300, endpoint=DEFAULT_ENDPOINT_DEALER, pubsub_endpoint=DEFAULT_ENDPOINT_SUB): queue = multiprocessing.Queue() plugin_info_callback = functools.partial(plugin_info_callback, queue) circusctl_process = multiprocessing.Process( target=run_plugin, args=(cls, config, plugin_info_callback, duration, endpoint, pubsub_endpoint)) circusctl_process.start() while queue.empty(): yield tornado_sleep(.1) result = queue.get() raise tornado.gen.Return(result)
def async_run_ctl(args, stdin=''): """ Start a process that will start the actual circusctl process and poll its ouput, via a queue, without blocking the I/O loop. We do this to avoid blocking the main thread while waiting for circusctl output, so that the arbiter will be able to respond to requests coming from circusctl. """ queue = Queue() circusctl_process = Process(target=run_ctl, args=(args, queue, stdin)) circusctl_process.start() while queue.empty(): yield tornado_sleep(.1) stderr = queue.get() stdout = queue.get() raise Return((stdout, stderr))
def spawn_processes(self): """Spawn processes. """ # when an on_demand process dies, do not restart it until # the next event if self.pending_socket_event: self._status = "stopped" return for i in self._found_wids: self.spawn_process(i) yield tornado_sleep(0) self._found_wids = {} for i in range(self.numprocesses - len(self.processes)): res = self.spawn_process() if res is False: yield self._stop() break delay = self.warmup_delay if isinstance(res, float): delay -= (time.time() - res) if delay < 0: delay = 0 yield tornado_sleep(delay)
def read(self, timeout=None): timeout = timeout or self._timeout if self._buffer: raise tornado.gen.Return(self._buffer.pop(0)) start = time.time() while time.time() - start < timeout: try: msg = self._stream.get_nowait() lines = [l for l in s(msg["data"]).split("\n") if l] self._buffer.extend(lines) raise tornado.gen.Return(self._buffer.pop(0)) except Empty: yield tornado_sleep(0.1) raise TimeoutException("Timeout reading queue")
def async_poll_for(filename, needles, timeout=5): """Async version of poll_for """ if isinstance(needles, str): needles = [needles] start = time() while time() - start < timeout: with open(filename) as f: content = f.read() for needle in needles: if needle in content: raise tornado.gen.Return(True) yield tornado_sleep(0.1) raise TimeoutException('Timeout polling "%s" for "%s". Content: %s' % ( filename, needle, content))
def test_venv_py_ver(self): py_ver = "my_py_ver" venv = os.path.join(os.path.dirname(__file__), 'venv') wanted = os.path.join(venv, 'lib', 'python%s' % py_ver, 'site-packages') if not os.path.exists(wanted): os.makedirs(wanted) watcher = SomeWatcher(virtualenv=venv, virtualenv_py_ver=py_ver) yield watcher.run() try: yield tornado_sleep(1) ppath = watcher.watcher.env['PYTHONPATH'] finally: yield watcher.stop() self.assertTrue(wanted in ppath.split(os.pathsep))
def test_venv_site_packages(self): venv = os.path.join(os.path.dirname(__file__), 'venv') watcher = SomeWatcher(virtualenv=venv) yield watcher.run() try: yield tornado_sleep(1) py_version = get_python_version() major = py_version[0] minor = py_version[1] wanted = os.path.join(venv, 'lib', 'python%d.%d' % (major, minor), 'site-packages') ppath = watcher.watcher.env['PYTHONPATH'] finally: yield watcher.stop() self.assertTrue(wanted in ppath.split(os.pathsep))
def async_run_plugin(klass, config, plugin_info_callback, duration=300, endpoint=DEFAULT_ENDPOINT_DEALER, pubsub_endpoint=DEFAULT_ENDPOINT_SUB): queue = multiprocessing.Queue() plugin_info_callback = functools.partial(plugin_info_callback, queue) circusctl_process = multiprocessing.Process( target=run_plugin, args=(klass, config, plugin_info_callback, duration, endpoint, pubsub_endpoint)) circusctl_process.start() while queue.empty(): yield tornado_sleep(.1) result = queue.get() raise tornado.gen.Return(result)
def read(self, timeout=None): timeout = timeout or self._timeout if self._buffer: raise tornado.gen.Return(self._buffer.pop(0)) start = time.time() while time.time() - start < timeout: try: msg = self._stream.get_nowait() lines = [l for l in s(msg['data']).split('\n') if l] self._buffer.extend(lines) raise tornado.gen.Return(self._buffer.pop(0)) except Empty: yield tornado_sleep(0.1) raise TimeoutException('Timeout reading queue')
def async_poll_for(filename, needles, timeout=5): """Async version of poll_for """ if isinstance(needles, str): needles = [needles] start = time() needle = content = None while time() - start < timeout: with open(filename) as f: content = f.read() for needle in needles: if needle in content: raise tornado.gen.Return(True) yield tornado_sleep(0.1) raise TimeoutException('Timeout polling "%s" for "%s". Content: %s' % ( filename, needle, content))
def test_handler(self): log = self._get_file() stream = {'stream': FileStream(log)} cmd = 'circus.tests.test_stats_client.run_process' stdout_stream = stream stderr_stream = stream yield self.start_arbiter(cmd=cmd, stdout_stream=stdout_stream, stderr_stream=stderr_stream, stats=True, debug=False) # waiting for data to appear in the file stream empty = True while empty: with open(log) as f: empty = f.read() == '' yield tornado_sleep(.1) # checking that our system is live and running client = AsyncCircusClient(endpoint=self.arbiter.endpoint) res = yield client.send_message('list') watchers = sorted(res['watchers']) self.assertEqual(['circusd-stats', 'test'], watchers) # making sure the stats process run res = yield client.send_message('status', name='test') self.assertEqual(res['status'], 'active') res = yield client.send_message('status', name='circusd-stats') self.assertEqual(res['status'], 'active') # playing around with the stats now: we should get some ! from circus.stats.client import StatsClient client = StatsClient(endpoint=self.arbiter.stats_endpoint) message_iterator = client.iter_messages() for i in range(10): watcher, pid, stat = next(message_iterator) self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'), watcher) yield self.stop_arbiter()
def test_handler(self): log = self._get_file() stream = {'stream': FileStream(log)} cmd = 'circus.tests.test_stats_client.run_process' stdout_stream = stream stderr_stream = stream yield self.start_arbiter(cmd=cmd, stdout_stream=stdout_stream, stderr_stream=stderr_stream, stats=True, debug=False) # waiting for data to appear in the file stream empty = True while empty: with open(log) as f: empty = f.read() == '' yield tornado_sleep(.1) # checking that our system is live and running client = AsyncCircusClient(endpoint=self.arbiter.endpoint) res = yield client.send_message('list') watchers = sorted(res['watchers']) self.assertEqual(['circusd-stats', 'test'], watchers) # making sure the stats process run res = yield client.send_message('status', name='test') self.assertEqual(res['status'], 'active') res = yield client.send_message('status', name='circusd-stats') self.assertEqual(res['status'], 'active') # playing around with the stats now: we should get some ! from circus.stats.client import StatsClient client = StatsClient(endpoint=self.arbiter.stats_endpoint) next = get_next(client.iter_messages()) for i in range(10): watcher, pid, stat = next() self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'), watcher) yield self.stop_arbiter()
def read_from_stream(stream, desired_channel, timeout=5): start = time.time() accumulator = '' while not channels[desired_channel] and time.time() - start < timeout: try: data = stream.get_nowait() data = s(data['data']).split('\n') accumulator += data.pop(0) if data: data.insert(0, accumulator) accumulator = data.pop() for line in data: if len(line) > 1 and line[1] == ':': channel, string = line.partition(':')[::2] channels[int(channel)].append(string) except Empty: yield tornado_sleep(0.1) if channels[desired_channel]: raise tornado.gen.Return(channels[desired_channel].pop(0)) raise TimeoutException('Timeout reading queue')
def kill_process(self, process): """Kill process (SIGTERM, graceful_timeout then SIGKILL) """ if process.stopping: raise gen.Return(False) logger.debug("%s: kill process %s", self.name, process.pid) self.send_signal_process(process, signal.SIGTERM) process.stopping = True waited = 0 while waited < self.graceful_timeout: yield tornado_sleep(1) waited = waited + 1 if not process.is_alive(): break if waited >= self.graceful_timeout: # We are not smart anymore self.send_signal_process(process, signal.SIGKILL) self._process_remove_redirections(process) process.stopping = False process.stop() raise gen.Return(True)
def test_copy_path(self): watcher = SomeWatcher(stream=True) yield watcher.run() # wait for watcher data at most 5s messages = [] resp = False start_time = time.time() while (time.time() - start_time) <= 5: yield tornado_sleep(0.5) # More than one Queue.get call is needed to get full # output from a watcher in an environment with rich sys.path. try: m = watcher.stream.get(block=False) messages.append(m) except Queue.Empty: pass data = ''.join(s(m['data']) for m in messages) if 'XYZ' in data: resp = True break self.assertTrue(resp) yield watcher.stop()
def test_handler(self): log = self._get_file() stream = {"stream": FileStream(log)} cmd = "circus.tests.test_stats_client.run_process" stdout_stream = stream stderr_stream = stream yield self.start_arbiter(cmd=cmd, stdout_stream=stdout_stream, stderr_stream=stderr_stream, stats=True) # waiting for data to appear in the file stream empty = True while empty: with open(log) as f: empty = f.read() == "" yield tornado_sleep(0.1) # checking that our system is live and running client = AsyncCircusClient() res = yield client.send_message("list") watchers = sorted(res["watchers"]) self.assertEqual(["circusd-stats", "test"], watchers) # making sure the stats process run res = yield client.send_message("status", name="test") self.assertEqual(res["status"], "active") res = yield client.send_message("status", name="circusd-stats") self.assertEqual(res["status"], "active") # playing around with the stats now: we should get some ! from circus.stats.client import StatsClient client = StatsClient() next = get_next(client.iter_messages()) for i in range(10): watcher, pid, stat = next() self.assertTrue(watcher in ("test", "circusd-stats", "circus"), watcher) yield self.stop_arbiter()
def start_watcher(self, watcher): """Aska a specific watcher to start and wait for the specified warmup delay.""" if watcher.autostart: yield watcher._start() yield tornado_sleep(self.warmup_delay)
def _start_watchers(self): for watcher in self.iter_watchers(): if watcher.autostart: yield watcher._start() yield tornado_sleep(self.warmup_delay)