def spawn_processes(self):
"""Spawn processes.
"""
# when an on_demand process dies, do not restart it until
# the next event
if self.pending_socket_event:
self._status = "stopped"
return
for i in self._found_wids:
self.spawn_process(i)
yield tornado_sleep(0)
self._found_wids = {}
for i in range(self.numprocesses - len(self.processes)):
res = self.spawn_process()
if res is False:
if len(self.async_killing_futures) == 0:
yield self._stop(True)
break
delay = self.warmup_delay
if isinstance(res, float):
delay -= (time.time() - res)
if delay < 0:
delay = 0
yield tornado_sleep(delay)
def test_message_id(self):
hooks = {'before_stop': ('circus.tests.test_client.long_hook', False)}
testfile, arbiter = self.run_with_hooks(hooks)
yield arbiter.start()
try:
self.assertTrue(os.path.exists(tmp_filename))
msg = make_message("numwatchers")
resp = yield self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
# this should timeout
resp = yield self.cli.call(make_message("stop"))
self.assertEqual(resp.get('status'), 'ok')
while arbiter.watchers[0].status() != 'stopped':
yield tornado_sleep(.1)
resp = yield self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
self.assertFalse(os.path.exists(tmp_filename))
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
arbiter.stop()
def kill_process(self, process):
"""Kill process (stop_signal, graceful_timeout then SIGKILL)
"""
if process.stopping:
raise gen.Return(False)
logger.debug("%s: kill process %s", self.name, process.pid)
if self.stop_children:
self.send_signal_process(process, self.stop_signal)
else:
self.send_signal(process.pid, self.stop_signal)
self.notify_event("kill", {"process_pid": process.pid,
"time": time.time()})
process.stopping = True
waited = 0
while waited < self.graceful_timeout:
yield tornado_sleep(1)
waited += 1
if not process.is_alive():
break
if waited >= self.graceful_timeout:
# We are not smart anymore
self.send_signal_process(process, signal.SIGKILL)
self._process_remove_redirections(process)
process.stopping = False
process.stop()
raise gen.Return(True)
def kill_process(self, process):
"""Kill process (stop_signal, graceful_timeout then SIGKILL)
"""
if process.stopping:
raise gen.Return(False)
try:
logger.debug("%s: kill process %s", self.name, process.pid)
if self.stop_children:
self.send_signal_process(process, self.stop_signal)
else:
self.send_signal(process.pid, self.stop_signal)
self.notify_event("kill", {"process_pid": process.pid,
"time": time.time()})
except NoSuchProcess:
raise gen.Return(False)
process.stopping = True
waited = 0
while waited < self.graceful_timeout:
if not process.is_alive():
break
yield tornado_sleep(0.1)
waited += 0.1
if waited >= self.graceful_timeout:
# On Windows we can't send a SIGKILL signal, but the
# process.stop function will terminate the process
# later anyway
if hasattr(signal, 'SIGKILL'):
# We are not smart anymore
self.send_signal_process(process, signal.SIGKILL)
if self.stream_redirector:
self.stream_redirector.remove_redirections(process)
process.stopping = False
process.stop()
raise gen.Return(True)
def start_arbiter(self,
cmd='support.run_process',
stdout_stream=None,
debug=True,
**kw):
testfile, arbiter = self._create_circus(cmd,
stdout_stream=stdout_stream,
debug=debug,
use_async=True,
**kw)
self.test_file = testfile
self.arbiter = arbiter
self.arbiters.append(arbiter)
for i in range(self.ADDRESS_IN_USE_TRY_TIMES):
try:
yield self.arbiter.start()
except ZMQError as e:
if e.strerror == 'Address already in use':
# One more try to wait for the port being released by the OS
yield tornado_sleep(0.1)
continue
else:
raise e
else:
# Everything goes well, just break
break
else:
# Cannot start after tries
raise RuntimeError('Cannot start arbiter after %s times try' %
self.ADDRESS_IN_USE_TRY_TIMES)
def kill_process(self, process):
"""Kill process (stop_signal, graceful_timeout then SIGKILL)
"""
if process.stopping:
raise gen.Return(False)
try:
logger.debug("%s: kill process %s", self.name, process.pid)
if self.stop_children:
self.send_signal_process(process, self.stop_signal)
else:
self.send_signal(process.pid, self.stop_signal)
self.notify_event("kill", {"process_pid": process.pid,
"time": time.time()})
except NoSuchProcess:
raise gen.Return(False)
process.stopping = True
waited = 0
while waited < self.graceful_timeout:
if not process.is_alive():
break
yield tornado_sleep(0.1)
waited += 0.1
if waited >= self.graceful_timeout:
# On Windows we can't send a SIGKILL signal, but the
# process.stop function will terminate the process
# later anyway
if hasattr(signal, 'SIGKILL'):
# We are not smart anymore
self.send_signal_process(process, signal.SIGKILL)
if self.stream_redirector:
self.stream_redirector.remove_redirections(process)
process.stopping = False
process.stop()
raise gen.Return(True)
def _reload(self, graceful=True, sequential=False):
""" reload
"""
if not graceful and sequential:
logger.warn("with graceful=False, sequential=True is ignored")
if self.prereload_fn is not None:
self.prereload_fn(self)
if not graceful:
yield self._restart()
return
if self.is_stopped():
yield self._start()
elif self.send_hup:
for process in self.processes.values():
logger.info("SENDING HUP to %s" % process.pid)
process.send_signal(signal.SIGHUP)
else:
if sequential:
active_processes = self.get_active_processes()
for process in active_processes:
yield self.kill_process(process)
self.reap_process(process.pid)
self.spawn_process()
yield tornado_sleep(self.warmup_delay)
else:
for i in range(self.numprocesses):
self.spawn_process()
yield self.manage_processes()
self.notify_event("reload", {"time": time.time()})
logger.info('%s reloaded', self.name)
def test_message_id(self):
hooks = {'before_stop': ('circus.tests.test_client.long_hook', False)}
testfile, arbiter = self.run_with_hooks(hooks)
yield arbiter.start()
try:
self.assertTrue(os.path.exists(tmp_filename))
msg = make_message("numwatchers")
resp = yield self.cli.call(msg)
self.assertEqual(resp.get("numwatchers"), 1)
# this should timeout
resp = yield self.cli.call(make_message("stop"))
self.assertEqual(resp.get('status'), 'ok')
while arbiter.watchers[0].status() != 'stopped':
yield tornado_sleep(.1)
resp = yield self.cli.call(make_message("numwatchers"))
self.assertEqual(resp.get("numwatchers"), 1)
self.assertFalse(os.path.exists(tmp_filename))
finally:
if os.path.exists(tmp_filename):
os.unlink(tmp_filename)
arbiter.stop()
def kill_process(self, process):
"""Kill process (stop_signal, graceful_timeout then SIGKILL)
"""
if process.stopping:
raise gen.Return(False)
logger.debug("%s: kill process %s", self.name, process.pid)
if self.stop_children:
self.send_signal_process(process, self.stop_signal)
else:
self.send_signal(process.pid, self.stop_signal)
self.notify_event("kill", {
"process_pid": process.pid,
"time": time.time()
})
process.stopping = True
waited = 0
while waited < self.graceful_timeout:
yield tornado_sleep(1)
waited += 1
if not process.is_alive():
break
if waited >= self.graceful_timeout:
# We are not smart anymore
self.send_signal_process(process, signal.SIGKILL)
self._process_remove_redirections(process)
process.stopping = False
process.stop()
raise gen.Return(True)
def _reload(self, graceful=True, sequential=False):
""" reload
"""
if not graceful and sequential:
logger.warn("with graceful=False, sequential=True is ignored")
if self.prereload_fn is not None:
self.prereload_fn(self)
if not graceful:
yield self._restart()
return
if self.is_stopped():
yield self._start()
elif self.send_hup:
for process in self.processes.values():
logger.info("SENDING HUP to %s" % process.pid)
process.send_signal(signal.SIGHUP)
else:
if sequential:
active_processes = self.get_active_processes()
for process in active_processes:
yield self.kill_process(process)
self.reap_process(process.pid)
self.spawn_process()
yield tornado_sleep(self.warmup_delay)
else:
for i in range(self.numprocesses):
self.spawn_process()
yield self.manage_processes()
self.notify_event("reload", {"time": time.time()})
logger.info('%s reloaded', self.name)
def _start_watchers(self, watcher_iter_func=None):
if watcher_iter_func is None:
watchers = self.iter_watchers()
else:
watchers = watcher_iter_func()
for watcher in watchers:
if watcher.autostart:
yield watcher._start()
yield tornado_sleep(self.warmup_delay)
def async_run_plugin(klass, config, plugin_info_callback, duration=300):
queue = multiprocessing.Queue()
plugin_info_callback = functools.partial(plugin_info_callback, queue)
circusctl_process = multiprocessing.Process(target=run_plugin, args=(klass, config, plugin_info_callback, duration))
circusctl_process.start()
while queue.empty():
yield tornado_sleep(0.1)
result = queue.get()
raise tornado.gen.Return(result)
def _start_watchers(self, watcher_iter_func=None):
if watcher_iter_func is None:
watchers = self.iter_watchers()
else:
watchers = watcher_iter_func()
for watcher in watchers:
if watcher.autostart:
yield watcher._start()
yield tornado_sleep(self.warmup_delay)
def read_from_stream(stream, timeout=5):
start = time.time()
while time.time() - start < timeout:
try:
data = stream.get_nowait()
raise tornado.gen.Return(u(data['data']))
except Empty:
yield tornado_sleep(0.1)
raise TimeoutException('Timeout reading queue')
def async_run_plugin(klass, config, plugin_info_callback, duration=300):
queue = multiprocessing.Queue()
plugin_info_callback = functools.partial(plugin_info_callback, queue)
circusctl_process = multiprocessing.Process(target=run_plugin,
args=(klass, config,
plugin_info_callback,
duration))
circusctl_process.start()
while queue.empty():
yield tornado_sleep(.1)
result = queue.get()
raise tornado.gen.Return(result)
def reload(self, graceful=True, sequential=False):
"""Reloads everything.
Run the :func:`prereload_fn` callable if any, then gracefuly
reload all watchers.
"""
if self._stopping:
return
if self.prereload_fn is not None:
self.prereload_fn(self)
# reopen log files
for handler in logger.handlers:
if isinstance(handler, logging.FileHandler):
handler.acquire()
handler.stream.close()
handler.stream = open(handler.baseFilename, handler.mode)
handler.release()
# gracefully reload watchers
for watcher in self.iter_watchers():
yield watcher._reload(graceful=graceful, sequential=sequential)
tornado_sleep(self.warmup_delay)
def spawn_processes(self):
"""Spawn processes.
"""
# when an on_demand process dies, do not restart it until
# the next event
if self.pending_socket_event:
self._status = "stopped"
return
for i in range(self.numprocesses - len(self.processes)):
res = self.spawn_process()
if res is False:
yield self._stop()
break
yield tornado_sleep(self.warmup_delay)
def spawn_processes(self):
"""Spawn processes.
"""
# when an on_demand process dies, do not restart it until
# the next event
if self.on_demand and not self.arbiter.socket_event:
self._status = "stopped"
return
for i in range(self.numprocesses - len(self.processes)):
res = self.spawn_process()
if res is False:
yield self._stop()
break
yield tornado_sleep(self.warmup_delay)
def reload(self, graceful=True, sequential=False):
"""Reloads everything.
Run the :func:`prereload_fn` callable if any, then gracefuly
reload all watchers.
"""
if self._stopping:
return
if self.prereload_fn is not None:
self.prereload_fn(self)
# reopen log files
for handler in logger.handlers:
if isinstance(handler, logging.FileHandler):
handler.acquire()
handler.stream.close()
handler.stream = open(handler.baseFilename, handler.mode)
handler.release()
# gracefully reload watchers
for watcher in self.iter_watchers():
yield watcher._reload(graceful=graceful, sequential=sequential)
tornado_sleep(self.warmup_delay)
def async_run_plugin(cls, config, plugin_info_callback, duration=300,
endpoint=DEFAULT_ENDPOINT_DEALER, pubsub_endpoint=DEFAULT_ENDPOINT_SUB):
queue = multiprocessing.Queue()
plugin_info_callback = functools.partial(plugin_info_callback, queue)
circusctl_process = multiprocessing.Process(
target=run_plugin,
args=(cls, config, plugin_info_callback, duration, endpoint, pubsub_endpoint))
circusctl_process.start()
while queue.empty():
yield tornado_sleep(.1)
result = queue.get()
raise tornado.gen.Return(result)
def async_run_ctl(args, stdin=''):
"""
Start a process that will start the actual circusctl process and poll its
ouput, via a queue, without blocking the I/O loop. We do this to avoid
blocking the main thread while waiting for circusctl output, so that the
arbiter will be able to respond to requests coming from circusctl.
"""
queue = Queue()
circusctl_process = Process(target=run_ctl, args=(args, queue, stdin))
circusctl_process.start()
while queue.empty():
yield tornado_sleep(.1)
stderr = queue.get()
stdout = queue.get()
raise Return((stdout, stderr))
def spawn_processes(self):
"""Spawn processes.
"""
# when an on_demand process dies, do not restart it until
# the next event
if self.pending_socket_event:
self._status = "stopped"
return
for i in self._found_wids:
self.spawn_process(i)
yield tornado_sleep(0)
self._found_wids = {}
for i in range(self.numprocesses - len(self.processes)):
res = self.spawn_process()
if res is False:
yield self._stop()
break
delay = self.warmup_delay
if isinstance(res, float):
delay -= (time.time() - res)
if delay < 0:
delay = 0
yield tornado_sleep(delay)
def async_run_ctl(args, stdin=''):
"""
Start a process that will start the actual circusctl process and poll its
ouput, via a queue, without blocking the I/O loop. We do this to avoid
blocking the main thread while waiting for circusctl output, so that the
arbiter will be able to respond to requests coming from circusctl.
"""
queue = Queue()
circusctl_process = Process(target=run_ctl, args=(args, queue, stdin))
circusctl_process.start()
while queue.empty():
yield tornado_sleep(.1)
stderr = queue.get()
stdout = queue.get()
raise Return((stdout, stderr))
def read(self, timeout=None):
timeout = timeout or self._timeout
if self._buffer:
raise tornado.gen.Return(self._buffer.pop(0))
start = time.time()
while time.time() - start < timeout:
try:
msg = self._stream.get_nowait()
lines = [l for l in s(msg["data"]).split("\n") if l]
self._buffer.extend(lines)
raise tornado.gen.Return(self._buffer.pop(0))
except Empty:
yield tornado_sleep(0.1)
raise TimeoutException("Timeout reading queue")
def async_poll_for(filename, needles, timeout=5):
"""Async version of poll_for
"""
if isinstance(needles, str):
needles = [needles]
start = time()
while time() - start < timeout:
with open(filename) as f:
content = f.read()
for needle in needles:
if needle in content:
raise tornado.gen.Return(True)
yield tornado_sleep(0.1)
raise TimeoutException('Timeout polling "%s" for "%s". Content: %s' % (
filename, needle, content))
def test_venv_py_ver(self):
py_ver = "my_py_ver"
venv = os.path.join(os.path.dirname(__file__), 'venv')
wanted = os.path.join(venv, 'lib', 'python%s' % py_ver,
'site-packages')
if not os.path.exists(wanted):
os.makedirs(wanted)
watcher = SomeWatcher(virtualenv=venv, virtualenv_py_ver=py_ver)
yield watcher.run()
try:
yield tornado_sleep(1)
ppath = watcher.watcher.env['PYTHONPATH']
finally:
yield watcher.stop()
self.assertTrue(wanted in ppath.split(os.pathsep))
def test_venv_site_packages(self):
venv = os.path.join(os.path.dirname(__file__), 'venv')
watcher = SomeWatcher(virtualenv=venv)
yield watcher.run()
try:
yield tornado_sleep(1)
py_version = get_python_version()
major = py_version[0]
minor = py_version[1]
wanted = os.path.join(venv, 'lib', 'python%d.%d' % (major, minor),
'site-packages')
ppath = watcher.watcher.env['PYTHONPATH']
finally:
yield watcher.stop()
self.assertTrue(wanted in ppath.split(os.pathsep))
def async_run_plugin(klass, config, plugin_info_callback, duration=300,
endpoint=DEFAULT_ENDPOINT_DEALER,
pubsub_endpoint=DEFAULT_ENDPOINT_SUB):
queue = multiprocessing.Queue()
plugin_info_callback = functools.partial(plugin_info_callback, queue)
circusctl_process = multiprocessing.Process(
target=run_plugin,
args=(klass, config, plugin_info_callback, duration,
endpoint, pubsub_endpoint))
circusctl_process.start()
while queue.empty():
yield tornado_sleep(.1)
result = queue.get()
raise tornado.gen.Return(result)
def read(self, timeout=None):
timeout = timeout or self._timeout
if self._buffer:
raise tornado.gen.Return(self._buffer.pop(0))
start = time.time()
while time.time() - start < timeout:
try:
msg = self._stream.get_nowait()
lines = [l for l in s(msg['data']).split('\n') if l]
self._buffer.extend(lines)
raise tornado.gen.Return(self._buffer.pop(0))
except Empty:
yield tornado_sleep(0.1)
raise TimeoutException('Timeout reading queue')
def async_poll_for(filename, needles, timeout=5):
"""Async version of poll_for
"""
if isinstance(needles, str):
needles = [needles]
start = time()
needle = content = None
while time() - start < timeout:
with open(filename) as f:
content = f.read()
for needle in needles:
if needle in content:
raise tornado.gen.Return(True)
yield tornado_sleep(0.1)
raise TimeoutException('Timeout polling "%s" for "%s". Content: %s' % (
filename, needle, content))
def test_handler(self):
log = self._get_file()
stream = {'stream': FileStream(log)}
cmd = 'circus.tests.test_stats_client.run_process'
stdout_stream = stream
stderr_stream = stream
yield self.start_arbiter(cmd=cmd,
stdout_stream=stdout_stream,
stderr_stream=stderr_stream,
stats=True,
debug=False)
# waiting for data to appear in the file stream
empty = True
while empty:
with open(log) as f:
empty = f.read() == ''
yield tornado_sleep(.1)
# checking that our system is live and running
client = AsyncCircusClient(endpoint=self.arbiter.endpoint)
res = yield client.send_message('list')
watchers = sorted(res['watchers'])
self.assertEqual(['circusd-stats', 'test'], watchers)
# making sure the stats process run
res = yield client.send_message('status', name='test')
self.assertEqual(res['status'], 'active')
res = yield client.send_message('status', name='circusd-stats')
self.assertEqual(res['status'], 'active')
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient(endpoint=self.arbiter.stats_endpoint)
message_iterator = client.iter_messages()
for i in range(10):
watcher, pid, stat = next(message_iterator)
self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'),
watcher)
yield self.stop_arbiter()
def test_handler(self):
log = self._get_file()
stream = {'stream': FileStream(log)}
cmd = 'circus.tests.test_stats_client.run_process'
stdout_stream = stream
stderr_stream = stream
yield self.start_arbiter(cmd=cmd, stdout_stream=stdout_stream,
stderr_stream=stderr_stream, stats=True,
debug=False)
# waiting for data to appear in the file stream
empty = True
while empty:
with open(log) as f:
empty = f.read() == ''
yield tornado_sleep(.1)
# checking that our system is live and running
client = AsyncCircusClient(endpoint=self.arbiter.endpoint)
res = yield client.send_message('list')
watchers = sorted(res['watchers'])
self.assertEqual(['circusd-stats', 'test'], watchers)
# making sure the stats process run
res = yield client.send_message('status', name='test')
self.assertEqual(res['status'], 'active')
res = yield client.send_message('status', name='circusd-stats')
self.assertEqual(res['status'], 'active')
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient(endpoint=self.arbiter.stats_endpoint)
next = get_next(client.iter_messages())
for i in range(10):
watcher, pid, stat = next()
self.assertTrue(watcher in ('test', 'circusd-stats', 'circus'),
watcher)
yield self.stop_arbiter()
def read_from_stream(stream, desired_channel, timeout=5):
start = time.time()
accumulator = ''
while not channels[desired_channel] and time.time() - start < timeout:
try:
data = stream.get_nowait()
data = s(data['data']).split('\n')
accumulator += data.pop(0)
if data:
data.insert(0, accumulator)
accumulator = data.pop()
for line in data:
if len(line) > 1 and line[1] == ':':
channel, string = line.partition(':')[::2]
channels[int(channel)].append(string)
except Empty:
yield tornado_sleep(0.1)
if channels[desired_channel]:
raise tornado.gen.Return(channels[desired_channel].pop(0))
raise TimeoutException('Timeout reading queue')
def read_from_stream(stream, desired_channel, timeout=5):
start = time.time()
accumulator = ''
while not channels[desired_channel] and time.time() - start < timeout:
try:
data = stream.get_nowait()
data = s(data['data']).split('\n')
accumulator += data.pop(0)
if data:
data.insert(0, accumulator)
accumulator = data.pop()
for line in data:
if len(line) > 1 and line[1] == ':':
channel, string = line.partition(':')[::2]
channels[int(channel)].append(string)
except Empty:
yield tornado_sleep(0.1)
if channels[desired_channel]:
raise tornado.gen.Return(channels[desired_channel].pop(0))
raise TimeoutException('Timeout reading queue')
def kill_process(self, process):
"""Kill process (SIGTERM, graceful_timeout then SIGKILL)
"""
if process.stopping:
raise gen.Return(False)
logger.debug("%s: kill process %s", self.name, process.pid)
self.send_signal_process(process, signal.SIGTERM)
process.stopping = True
waited = 0
while waited < self.graceful_timeout:
yield tornado_sleep(1)
waited = waited + 1
if not process.is_alive():
break
if waited >= self.graceful_timeout:
# We are not smart anymore
self.send_signal_process(process, signal.SIGKILL)
self._process_remove_redirections(process)
process.stopping = False
process.stop()
raise gen.Return(True)
def test_copy_path(self):
watcher = SomeWatcher(stream=True)
yield watcher.run()
# wait for watcher data at most 5s
messages = []
resp = False
start_time = time.time()
while (time.time() - start_time) <= 5:
yield tornado_sleep(0.5)
# More than one Queue.get call is needed to get full
# output from a watcher in an environment with rich sys.path.
try:
m = watcher.stream.get(block=False)
messages.append(m)
except Queue.Empty:
pass
data = ''.join(s(m['data']) for m in messages)
if 'XYZ' in data:
resp = True
break
self.assertTrue(resp)
yield watcher.stop()
def test_handler(self):
log = self._get_file()
stream = {"stream": FileStream(log)}
cmd = "circus.tests.test_stats_client.run_process"
stdout_stream = stream
stderr_stream = stream
yield self.start_arbiter(cmd=cmd, stdout_stream=stdout_stream, stderr_stream=stderr_stream, stats=True)
# waiting for data to appear in the file stream
empty = True
while empty:
with open(log) as f:
empty = f.read() == ""
yield tornado_sleep(0.1)
# checking that our system is live and running
client = AsyncCircusClient()
res = yield client.send_message("list")
watchers = sorted(res["watchers"])
self.assertEqual(["circusd-stats", "test"], watchers)
# making sure the stats process run
res = yield client.send_message("status", name="test")
self.assertEqual(res["status"], "active")
res = yield client.send_message("status", name="circusd-stats")
self.assertEqual(res["status"], "active")
# playing around with the stats now: we should get some !
from circus.stats.client import StatsClient
client = StatsClient()
next = get_next(client.iter_messages())
for i in range(10):
watcher, pid, stat = next()
self.assertTrue(watcher in ("test", "circusd-stats", "circus"), watcher)
yield self.stop_arbiter()
def start_watcher(self, watcher):
"""Aska a specific watcher to start and wait for the specified
warmup delay."""
if watcher.autostart:
yield watcher._start()
yield tornado_sleep(self.warmup_delay)
def _start_watchers(self):
for watcher in self.iter_watchers():
if watcher.autostart:
yield watcher._start()
yield tornado_sleep(self.warmup_delay)
def _start_watchers(self):
for watcher in self.iter_watchers():
if watcher.autostart:
yield watcher._start()
yield tornado_sleep(self.warmup_delay)
def start_watcher(self, watcher):
"""Aska a specific watcher to start and wait for the specified
warmup delay."""
if watcher.autostart:
yield watcher._start()
yield tornado_sleep(self.warmup_delay)
