def run(self):
# crude check for inplace build:
try:
import gevent_zeromq
except ImportError:
print_exc()
print("Could not import gevent_zeromq!")
print(
"You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work."
)
print(
"If you did build gevent_zeromq in-place, then this is a real error."
)
sys.exit(1)
gevent_zeromq.monkey_patch(test_suite=True) # monkey patch
import zmq
self._zmq_dir = os.path.dirname(zmq.__file__)
if nose is None:
print(
"nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs."
)
return self.run_unittest()
else:
return self.run_nose()
def run(self, handler):
try:
from socketio.server import SocketIOServer
except ImportError:
raise ImportError('You need to install gevent_socketio')
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
namespace = self.options.get('namespace', 'socket.io')
policy_server = self.options.get('policy_server', False)
if self.fd is not None:
sock = socket.fromfd(self.fd, socket.AF_INET, socket.SOCK_STREAM)
else:
sock = (self.host, self.port)
socket_server = SocketIOServer(sock,
handler,
namespace=namespace,
policy_server=policy_server)
handler.socket_server = socket_server
socket_server.serve_forever()
def run(self):
# crude check for inplace build:
try:
import gevent_zeromq
except ImportError:
print_exc()
print("Could not import gevent_zeromq!")
print(
"You must build gevent_zeromq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work."
)
print(
"If you did build gevent_zeromq in-place, then this is a real error."
)
sys.exit(1)
gevent_zeromq.monkey_patch(test_suite=True) # monkey patch
import zmq
self._zmq_dir = os.path.dirname(zmq.__file__)
if nose is None:
print("nose unavailable, skipping tests.")
else:
return nose.core.TestProgram(
argv=["", '-vvs',
os.path.join(self._zmq_dir, 'tests')])
def setup_ipython_embed(shell_api=None):
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(
appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile #, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch(
"IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
):
embed_kernel(local_ns=shell_api,
config=ipy_config) # blocks until INT
def setup_ipython_embed(shell_api=None):
_assert_ipython_dir()
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile#, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
embed_kernel(local_ns=shell_api, config=ipy_config) # blocks until INT
def configure(self, options):
self.options = options
if self.use_gevent:
import gevent_zeromq
from gevent import monkey
monkey.patch_all()
gevent_zeromq.monkey_patch()
self._configure_logging()
# These imports need to happen *after* monkey patching
from SimpleSeer.Session import Session
from SimpleSeer import models as M
self.session = Session(options.config)
if self.remote_seer:
from SimpleSeer.SimpleSeer import SimpleSeer as SS
SS(disable=True)
if self.session.mongo.get('is_slave'):
M.base.SimpleDoc.meta['auto_create_index'] = False
if options.profile_heap: self._start_profile_heap()
def run(self, handler):
try:
from socketio.server import SocketIOServer
except ImportError:
raise ImportError('You need to install gevent_socketio')
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
namespace = self.options.get('namespace', 'socket.io')
policy_server = self.options.get('policy_server', False)
socket_server = SocketIOServer((self.host, self.port), handler,
namespace=namespace,
policy_server=policy_server)
handler.socket_server = socket_server
socket_server.serve_forever()
def __init__(self, session):
monkey.patch_socket()
import gevent_zeromq
gevent_zeromq.monkey_patch()
#we do use greenlets, but only patch sock stuff
#other stuff messes up the
config = session.arduino
boardglob = config['board']
boards = glob(boardglob)
if not len(boards):
raise Exception("No Arduino found")
self.board = Arduino(boards[0])
#self.iterator = util.Iterator(self.board)
#self.iterator.daemon = True
#self.iterator.start()
#initialize servo objects
self.servos = {}
if "servos" in config:
for servo in config["servos"]:
self.servos[servo['name']] = Servo(servo['pin'], self.board)
#initialize light objects
self.digitalouts = {}
if "digitalouts" in config:
for do in config["digitalouts"]:
self.digitalouts[do['name']] = DigitalOut(do['pin'], self.board)
if "digitalouts" in config or "servos" in config:
self.subsock = ChannelManager().subscribe("ControlOutput/")
self.buttons = []
if "buttons" in config:
for button in config["buttons"]:
self.buttons.append(Button(button['pin'], button['message'], self.board))
self.potentiometers = []
if "potentiometers" in config:
for pot in config["potentiometers"]:
self.buttons.append(Potentiometer(pot['pin'], pot['name'], self.board))
def run(self, handler):
try:
from socketio.server import SocketIOServer
except ImportError:
raise ImportError('You need to install gevent_socketio')
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
namespace = self.options.get('namespace', 'socket.io')
policy_server = self.options.get('policy_server', False)
socket_server = SocketIOServer((self.host, self.port),
handler,
namespace=namespace,
policy_server=policy_server)
handler.socket_server = socket_server
socket_server.serve_forever()
def run(self):
# crude check for inplace build:
try:
import gevent_zeromq
except ImportError:
print_exc()
print ("Could not import gevent_zeromq!")
print ("You must build gevent_zeromq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.")
print ("If you did build gevent_zeromq in-place, then this is a real error.")
sys.exit(1)
gevent_zeromq.monkey_patch(test_suite=True) # monkey patch
import zmq
self._zmq_dir = os.path.dirname(zmq.__file__)
if nose is None:
print ("nose unavailable, skipping tests.")
else:
return nose.core.TestProgram(argv=["", '-vvs', os.path.join(self._zmq_dir, 'tests')])
def run(self):
# crude check for inplace build:
try:
import gevent_zeromq
except ImportError:
print_exc()
print ("Could not import gevent_zeromq!")
print ("You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.")
print ("If you did build gevent_zeromq in-place, then this is a real error.")
sys.exit(1)
gevent_zeromq.monkey_patch(test_suite=True) # monkey patch
import zmq
self._zmq_dir = os.path.dirname(zmq.__file__)
if nose is None:
print ("nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs.")
return self.run_unittest()
else:
return self.run_nose()
def monkey_patch():
"""Hook to monkey-patch the interpreter for this IO module.
This calls the standard gevent monkey-patching routines. Don't worry,
it's not called by default unless you're running from the command line.
"""
gevent.monkey.patch_all()
gevent_zeromq.monkey_patch()
# Patch signal module for gevent compatability.
# Courtesy of http://code.google.com/p/gevent/issues/detail?id=49
import signal
_orig_signal = signal.signal
def gevent_signal_wrapper(signum,*args,**kwds):
handler = signal.getsignal(signum)
if callable(handler):
handler(signum,None)
def gevent_signal(signum,handler):
_orig_signal(signum,handler)
return gevent.hub.signal(signum,gevent_signal_wrapper,signum)
signal.signal = gevent_signal
def setUp():
try:
from gevent import monkey # NOQA
try:
from gevent_zeromq import monkey_patch, IOLOOP_IS_MONKEYPATCHED # NOQA
monkey.patch_all()
monkey_patch()
except ImportError:
msg = """We have detected that you have gevent in your
environment. In order to have Circus working, you *must*
install gevent_zmq from :
https://github.com/tarekziade/gevent-zeromq
Circus will not need this in the future once
pyzmq gets a green poller:
https://github.com/zeromq/pyzmq/issues/197
"""
raise ImportError(msg)
except ImportError:
pass
def setUp():
from circus import _patch # NOQA
try:
import gevent # NOQA
from gevent import monkey # NOQA
try:
import zmq.eventloop as old_io
import zmq.green as zmq # NOQA
old_io.ioloop.Poller = zmq.Poller
except ImportError:
# older version
try:
from gevent_zeromq import ( # NOQA
monkey_patch, IOLOOP_IS_MONKEYPATCHED) # NOQA
monkey_patch()
warnings.warn("gevent_zeromq is deprecated, please "
"use PyZMQ >= 2.2.0.1")
except ImportError:
raise ImportError(_MSG)
monkey.patch_all()
except ImportError:
pass
def monkey_patch():
"""Hook to monkey-patch the interpreter for this IO module.
This calls the standard gevent monkey-patching routines. Don't worry,
it's not called by default unless you're running from the command line.
"""
gevent.monkey.patch_all()
gevent_zeromq.monkey_patch()
# Patch signal module for gevent compatability.
# Courtesy of http://code.google.com/p/gevent/issues/detail?id=49
import signal
_orig_signal = signal.signal
def gevent_signal_wrapper(signum, *args, **kwds):
handler = signal.getsignal(signum)
if callable(handler):
handler(signum, None)
def gevent_signal(signum, handler):
_orig_signal(signum, handler)
return gevent.hub.signal(signum, gevent_signal_wrapper, signum)
signal.signal = gevent_signal
def setUp():
from circus import _patch # NOQA
try:
from gevent import monkey # NOQA
try:
from gevent_zeromq import monkey_patch, IOLOOP_IS_MONKEYPATCHED # NOQA
monkey.patch_all()
monkey_patch()
except ImportError:
msg = """We have detected that you have gevent in your
environment. In order to have Circus working, you *must*
install gevent_zmq from :
https://github.com/tarekziade/gevent-zeromq
Circus will not need this in the future once
pyzmq gets a green poller:
https://github.com/zeromq/pyzmq/issues/197
"""
raise ImportError(msg)
except ImportError:
pass
def run(self, handler):
try:
from socketio.server import SocketIOServer
except ImportError:
raise ImportError('You need to install gevent_socketio')
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
namespace = self.options.get('namespace', 'socket.io')
policy_server = self.options.get('policy_server', False)
if self.fd is not None:
sock = socket.fromfd(self.fd, socket.AF_INET, socket.SOCK_STREAM)
else:
sock = (self.host, self.port)
socket_server = SocketIOServer(sock, handler,
namespace=namespace,
policy_server=policy_server)
handler.socket_server = socket_server
socket_server.serve_forever()
def get_arbiter(watchers,
controller='tcp://127.0.0.1:5555',
pubsub_endpoint='tcp://127.0.0.1:5556',
stats_endpoint=None,
env=None,
name=None,
context=None,
background=False,
stream_backend="thread",
plugins=None):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from the
actual executable name. It becomes the display name for the executing
program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn (default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes (default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes (default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for configuring
the stderr stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **stats_endpoint** -- the stats endpoint. If not provided,
the *circusd-stats* process will not be launched. (default: None)
- **context** -- the zmq context (default: None)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
- **plugins** -- a list of plugins. Each item is a mapping with:
- **use** -- Fully qualified name that points to the plugin class
- every other value is passed to the plugin in the **config** option
"""
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
from circus.watcher import Watcher
if background:
from circus.arbiter import ThreadedArbiter as Arbiter # NOQA
else:
from circus.arbiter import Arbiter # NOQA
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers,
controller,
pubsub_endpoint,
stats_endpoint=stats_endpoint,
context=context,
plugins=plugins)
def get_config(config_file):
if not os.path.exists(config_file):
sys.stderr.write("the configuration file %r does not exist\n" % config_file)
sys.stderr.write("Exiting...\n")
sys.exit(1)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config["check"] = dget("circus", "check_delay", 5, int)
config["endpoint"] = dget("circus", "endpoint", "tcp://127.0.0.1:5555")
config["pubsub_endpoint"] = dget("circus", "pubsub_endpoint", "tcp://127.0.0.1:5556")
config["stats_endpoint"] = dget("circus", "stats_endpoint", None, str)
config["warmup_delay"] = dget("circus", "warmup_delay", 0, int)
config["httpd"] = dget("circus", "httpd", False, bool)
config["httpd_host"] = dget("circus", "httpd_host", "localhost", str)
config["httpd_port"] = dget("circus", "httpd_port", 8080, int)
stream_backend = dget("circus", "stream_backend", "thread")
if stream_backend == "gevent":
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " + "but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config["stream_backend"] = stream_backend
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock["name"] = section.split("socket:")[-1].lower()
sockets.append(sock)
if section.startswith("plugin:"):
plugins.append(dict(cfg.items(section)))
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher["name"] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == "cmd":
watcher["cmd"] = val
elif opt == "args":
watcher["args"] = val
elif opt == "numprocesses":
watcher["numprocesses"] = dget(section, "numprocesses", 1, int)
elif opt == "warmup_delay":
watcher["warmup_delay"] = dget(section, "warmup_delay", 0, int)
elif opt == "executable":
watcher["executable"] = dget(section, "executable", None, str)
elif opt == "working_dir":
watcher["working_dir"] = val
elif opt == "shell":
watcher["shell"] = dget(section, "shell", False, bool)
elif opt == "uid":
watcher["uid"] = val
elif opt == "gid":
watcher["gid"] = val
elif opt == "send_hup":
watcher["send_hup"] = dget(section, "send_hup", False, bool)
elif opt == "check_flapping":
watcher["check_flapping"] = dget(section, "check_flapping", True, bool)
elif opt == "max_retry":
watcher["max_retry"] = dget(section, "max_retry", 5, int)
elif opt == "graceful_timout":
watcher["graceful_timeout"] = dget(section, "graceful_timeout", 30, int)
elif opt.startswith("stderr_stream") or opt.startswith("stdout_stream"):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith("rlimit_"):
limit = opt[7:]
watcher["rlimits"][limit] = int(val)
elif opt == "priority":
watcher["priority"] = dget(section, "priority", 0, int)
elif opt == "use_sockets":
watcher["use_sockets"] = dget(section, "use_sockets", False, bool)
elif opt == "singleton":
watcher["singleton"] = dget(section, "singleton", False, bool)
else:
# freeform
watcher[opt] = val
# set the stream backend
watcher["stream_backend"] = stream_backend
watchers.append(watcher)
config["watchers"] = watchers
config["plugins"] = plugins
config["sockets"] = sockets
return config
import uuid
import gevent
import gevent.monkey
gevent.monkey.patch_all()
import gevent_zeromq
gevent_zeromq.monkey_patch()
import zmq
import arrayserver.server.tests
import arrayserver.server.redisutils as redisutils
import arrayserver.server.arrayserverconfig as arrayserverconfig
import arrayserver.server.arrayserverconfig.orderedyaml as orderedyaml
import arrayserver.server.arrayservernode as arrayservernode
import arrayserver.server.arrayserverbroker as arrayserverbroker
import arrayserver.server.tests.test_utils as test_utils
import redis
import collections
import sys
import yaml
import os
import logging
import time
import atexit
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
def start_redis(datapath, redis_port):
proc = redisutils.RedisProcess(redis_port, datapath,
import logging
import os
import sys
try:
import gevent # NOQA
try:
from gevent_zeromq import monkey_patch, IOLOOP_IS_MONKEYPATCHED # NOQA
monkey_patch()
except ImportError:
msg = """We have detected that you have gevent in your
environment. In order to have Circus working, you *must*
install gevent_zmq from :
https://github.com/tarekziade/gevent-zeromq
Circus will not need this in the future once
pyzmq gets a green poller:
https://github.com/zeromq/pyzmq/issues/197
"""
raise ImportError(msg)
except ImportError:
pass
version_info = (0, 5)
__version__ = ".".join(map(str, version_info))
logger = logging.getLogger('circus')
def includeme(config):
try:
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
except ImportError:
pass
config.include("cornice")
config.include("mozsvc")
config.scan("tokenserver.views")
settings = config.registry.settings
# default metlog setup
if 'metlog.backend' not in settings:
settings['metlog.backend'] = 'mozsvc.metrics.MetlogPlugin'
settings['metlog.enabled'] = True
settings['metlog.sender_class'] = \
'metlog.senders.StdOutSender'
metlog_wrapper = load_from_settings('metlog', settings)
if settings['metlog.enabled']:
for logger in ('tokenserver', 'mozsvc', 'powerhose'):
hook_logger(logger, metlog_wrapper.client)
config.registry['metlog'] = metlog_wrapper.client
# initializes the assignment backend
load_and_register("tokenserver", config)
# initialize the powerhose and browserid backends if they exist
for section in ("powerhose", "browserid"):
try:
load_and_register(section, config)
except NoSectionError:
pass
# load apps and set them up back in the setting
key = 'tokenserver.applications'
applications = defaultdict(list)
for element in settings.get(key, '').split(','):
element = element.strip()
if element == '':
continue
element = element.split('-')
if len(element) != 2:
continue
app, version = element
applications[app].append(version)
settings[key] = applications
# load the secrets file(s)
key = 'tokenserver.secrets_file'
secret_file = settings[key]
if not isinstance(secret_file, list):
secret_file = [secret_file]
files = []
for line in secret_file:
secret_file = [file for file in [file.strip() for file in line.split()]
if file != '']
files.extend(secret_file)
settings[key] = Secrets(files)
read_endpoints(config)
def get_config(config_file):
if not os.path.exists(config_file):
sys.stderr.write("the configuration file %r does not exist\n" %
config_file)
sys.stderr.write("Exiting...\n")
sys.exit(1)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
DEFAULT_ENDPOINT_SUB)
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
config['httpd'] = dget('circus', 'httpd', False, bool)
config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
config['debug'] = dget('circus', 'debug', False, bool)
stream_backend = dget('circus', 'stream_backend', 'thread')
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config['stream_backend'] = stream_backend
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1].lower()
sockets.append(sock)
if section.startswith("plugin:"):
plugins.append(dict(cfg.items(section)))
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30, int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
elif opt == 'singleton':
watcher['singleton'] = dget(section, "singleton", False,
bool)
elif opt == 'stream_backend':
watcher['stream_backend'] = val
elif opt == 'copy_env':
watcher['copy_env'] = dget(section, "copy_env", False,
bool)
elif opt == 'copy_path':
watcher['copy_path'] = dget(section, "copy_path", False,
bool)
elif opt.startswith('hooks.'):
hook_name = opt[len('hooks.'):]
val = [elmt.strip() for elmt in val.split(',', 1)]
if len(val) == 1:
val.append(False)
else:
val[1] = to_boolean(val[1])
watcher['hooks'][hook_name] = val
else:
# freeform
watcher[opt] = val
# set the stream backend
if 'stream_backend' not in watcher:
watcher['stream_backend'] = stream_backend
watchers.append(watcher)
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
def get_config(config_file):
if not os.path.exists(config_file):
sys.stderr.write("the configuration file %r does not exist\n" %
config_file)
sys.stderr.write("Exiting...\n")
sys.exit(1)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
'tcp://127.0.0.1:5556')
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
stream_backend = dget('circus', 'stream_backend', 'thread')
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config['stream_backend'] = stream_backend
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1]
sockets.append(sock)
if section.startswith("plugin:"):
plugins.append(dict(cfg.items(section)))
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30, int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
else:
# freeform
watcher[opt] = val
# set the stream backend
watcher['stream_backend'] = stream_backend
watchers.append(watcher)
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
also the gevent thread. A proper example is show on green_heart.py where both,
numpy calculation and gevent, are kept responsive.
Authors
-------
* MinRK
* Pedro Algarvio
"""
import os
import time
import numpy
import gevent
import threading
import gevent_zeromq
gevent_zeromq.monkey_patch(zmq_devices=True)
from gevent_zeromq import zmq
from gevent_zeromq.devices import ThreadDevice
def im_alive(t=None):
print "I'm alive!"
if t:
gevent.spawn_later(t, im_alive, t)
def run_blocking_call(A):
print "starting blocking loop"
tic = time.time()
numpy.dot(A,A.transpose())
print "blocked for %.3f s"%(time.time()-tic)
def get_config(config_file):
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
'tcp://127.0.0.1:5556')
stream_backend = dget('circus', 'stream_backend', 'thread')
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config['stream_backend'] = stream_backend
# Initialize watchers to manage
watchers = []
for section in cfg.sections():
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid '] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'flapping_attempts':
watcher['flapping_attempts'] = dget(section,
"flapping_attempts", 2,
int)
elif opt == 'flapping_window':
watcher['flapping_window'] = dget(section,
"flapping_window", 1,
int)
elif opt == 'retry_in':
watcher['retry_in'] = dget(section, "retry_in", 7, int)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30, int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
# second pass, parse stream conf
stdout_conf = watcher.get('stdout_stream', {})
watcher['stdout_stream'] = stream_config(watcher['name'],
stdout_conf)
stderr_conf = watcher.get('stderr_stream', {})
watcher['stderr_stream'] = stream_config(watcher['name'],
stderr_conf)
# set the stream backend
watcher['stream_backend'] = stream_backend
watchers.append(watcher)
config['watchers'] = watchers
return config
# encoding: utf-8
#
# Custom routing Router to Mama (XREP to REQ)
#
# Author: Jeremy Avnet (brainsik) <spork(dash)zmq(at)theory(dot)org>
# Author: Travis Cline <*****@*****.**> - gevent-zeromq adaption
import gevent
import gevent_zeromq
gevent_zeromq.monkey_patch() # patching required because of zmq import in zhelpers
import time
import random
import zhelpers
NBR_WORKERS = 10
def worker_thread(context):
worker = context.socket(zmq.REQ)
# We use a string identity for ease here
zhelpers.set_id(worker)
worker.connect("ipc://routing.ipc")
total = 0
while True:
# Tell the router we're ready for work
def get_config(config_file):
if not os.path.exists(config_file):
sys.stderr.write("the configuration file %r does not exist\n" %
config_file)
sys.stderr.write("Exiting...\n")
sys.exit(1)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
'tcp://127.0.0.1:5556')
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
stream_backend = dget('circus', 'stream_backend', 'thread')
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config['stream_backend'] = stream_backend
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1]
sockets.append(sock)
if section.startswith("plugin:"):
plugins.append(dict(cfg.items(section)))
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30,
int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
else:
# freeform
watcher[opt] = val
# set the stream backend
watcher['stream_backend'] = stream_backend
watchers.append(watcher)
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
def get_config(config_file):
if not os.path.exists(config_file):
sys.stderr.write("the configuration file %r does not exist\n" %
config_file)
sys.stderr.write("Exiting...\n")
sys.exit(1)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
DEFAULT_ENDPOINT_SUB)
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
config['httpd'] = dget('circus', 'httpd', False, bool)
config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
config['debug'] = dget('circus', 'debug', False, bool)
stream_backend = dget('circus', 'stream_backend', 'thread')
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config['stream_backend'] = stream_backend
# Initialize watchers, plugins & sockets to manage
watchers = []
environs = {}
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1].lower()
sockets.append(sock)
if section.startswith("plugin:"):
plugins.append(dict(cfg.items(section)))
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30,
int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
elif opt == 'singleton':
watcher['singleton'] = dget(section, "singleton", False,
bool)
elif opt == 'stream_backend':
watcher['stream_backend'] = val
elif opt == 'copy_env':
watcher['copy_env'] = dget(section, "copy_env", False,
bool)
elif opt == 'copy_path':
watcher['copy_path'] = dget(section, "copy_path", False,
bool)
elif opt.startswith('hooks.'):
hook_name = opt[len('hooks.'):]
val = [elmt.strip() for elmt in val.split(',', 1)]
if len(val) == 1:
val.append(False)
else:
val[1] = to_boolean(val[1])
watcher['hooks'][hook_name] = val
elif opt == 'respawn':
watcher['respawn'] = dget(section, "respawn", True, bool)
elif opt == 'env':
logger.warning('the env option is deprecated the use of '
'env sections is recommended')
watcher['env'] = parse_env_str(val)
else:
# freeform
watcher[opt] = val
# set the stream backend
if 'stream_backend' not in watcher:
watcher['stream_backend'] = stream_backend
watchers.append(watcher)
if section.startswith('env:'):
for watcher in section.split("env:", 1)[1].split(','):
watcher = watcher.strip()
if not watcher in environs:
environs[watcher] = dict()
environs[watcher].update([(k.upper(), v)
for k, v in cfg.items(section)])
for watcher in watchers:
if watcher['name'] in environs:
if not 'env' in watcher:
watcher['env'] = dict()
watcher['env'].update(environs[watcher['name']])
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
numpy calculation and gevent, are kept responsive.
Authors
-------
* MinRK
* Pedro Algarvio
"""
import os
import time
import numpy
import gevent
import threading
import gevent_zeromq
gevent_zeromq.monkey_patch(zmq_devices=True)
from gevent_zeromq import zmq
from gevent_zeromq.devices import ThreadDevice
def im_alive(t=None):
print "I'm alive!"
if t:
gevent.spawn_later(t, im_alive, t)
def run_blocking_call(A):
print "starting blocking loop"
tic = time.time()
numpy.dot(A, A.transpose())
import gevent
import gevent.monkey
gevent.monkey.patch_all()
import gevent_zeromq
gevent_zeromq.monkey_patch()
import zmq
import unittest
import numpy as np
import logging
import time
import test_utils
import os
import shelve
import tables
import pandas
import arrayserver.server.rpc as rpc
import arrayserver.server.rpc.client as client
from arrayserver.server.arrayserverbroker import ArrayServerBroker
from arrayserver.server.arrayservernode import ArrayServerNode
from arrayserver.server.arrayserverconfig import ArrayServerConfig, generate_config_hdf5, generate_config_numpy
import arrayserver.server.redisutils as redisutils
import arrayserver.server.arrayserverconfig as arrayserverconfig
from arrayserver.array_proxy.arrayserver_array_proxy import ArrayServerArrayProxy
import arrayserver.array_proxy.npproxy as npp
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
logging.debug("starting")
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn(
"IPython profile dir not found. Attempting to avoid race condition"
)
import gevent
import random
gevent.sleep(
random.random() *
3.0) # Introduce a random delay to make conflict less likely
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(
appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile #, shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch(
"IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api,
config=ipy_config) # blocks until INT signal
break
except Exception as ex:
log.debug(
"Failed IPython initialize attempt (try #%s): %s",
tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
except:
try:
if os.path.exists(
ipy_config.KernelApp.connection_file):
os.remove(ipy_config.KernelApp.connection_file)
except Exception:
pass
raise
from collections import defaultdict
try:
import gevent # NOQA
from gevent import monkey
monkey.noisy = False
monkey.patch_all()
from gevent_zeromq import monkey_patch
monkey_patch()
except ImportError:
pass
from circus import util
from circus import logger
from zmq.eventloop import ioloop
from iowait import IOWait, SelectIOWait
class BaseStatsCollector(ioloop.PeriodicCallback):
def __init__(self, streamer, name, callback_time=1., io_loop=None):
ioloop.PeriodicCallback.__init__(self, self._callback,
callback_time * 1000, io_loop)
self.streamer = streamer
self.name = name
def _callback(self):
logger.debug('Publishing stats about {0}'.format(self.name))
for stats in self.collect_stats():
if stats is None:
def setup_ipython_embed(shell_api=None):
if not _exists_ipython_dir():
log.warn("IPython profile dir not found. Attempting to avoid race condition")
import gevent
import random
gevent.sleep(random.random() * 3.0) # Introduce a random delay to make conflict less likely
from gevent_zeromq import monkey_patch
monkey_patch()
# patch in device:
# gevent-zeromq does not support devices, which block in the C layer.
# we need to support the "heartbeat" which is a simple bounceback, so we
# simulate it using the following method.
import zmq
orig_device = zmq.device
def device_patch(dev_type, insock, outsock, *args):
if dev_type == zmq.FORWARDER:
while True:
m = insock.recv()
outsock.send(m)
else:
orig_device.device(dev_type, insock, outsock, *args)
zmq.device = device_patch
# patch in auto-completion support
# added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
# we effectively make the same patches via monkeypatching
from IPython.core.interactiveshell import InteractiveShell
from IPython.zmq.ipkernel import IPKernelApp
old_start = IPKernelApp.start
old_set_completer_frame = InteractiveShell.set_completer_frame
def new_start(appself):
# restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
appself.shell.set_completer_frame = bound_scf
appself.shell.set_completer_frame()
old_start(appself)
IPKernelApp.start = new_start
from IPython import embed_kernel
ipy_config = _setup_ipython_config()
# set specific manhole options
import tempfile # , shutil
from mock import patch
temp_dir = tempfile.mkdtemp()
ipy_config.Application.ipython_dir = temp_dir
with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
for tries in range(3):
try:
embed_kernel(local_ns=shell_api, config=ipy_config) # blocks until INT
break
except Exception as ex:
log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
import gevent
import random
gevent.sleep(random.random() * 0.5)
def get_config(config_file):
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
'tcp://127.0.0.1:5556')
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
stream_backend = dget('circus', 'stream_backend', 'thread')
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
config['stream_backend'] = stream_backend
# Initialize watchers to manage
watchers = []
for section in cfg.sections():
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid '] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'flapping_attempts':
watcher['flapping_attempts'] = dget(
section, "flapping_attempts", 2, int)
elif opt == 'flapping_window':
watcher['flapping_window'] = dget(section,
"flapping_window", 1,
int)
elif opt == 'retry_in':
watcher['retry_in'] = dget(section, "retry_in", 7, int)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30,
int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
# second pass, parse stream conf
stdout_conf = watcher.get('stdout_stream', {})
watcher['stdout_stream'] = stream_config(watcher['name'],
stdout_conf)
stderr_conf = watcher.get('stderr_stream', {})
watcher['stderr_stream'] = stream_config(watcher['name'],
stderr_conf)
# set the stream backend
watcher['stream_backend'] = stream_backend
watchers.append(watcher)
config['watchers'] = watchers
return config
def get_arbiter(watchers, controller=None,
pubsub_endpoint=None,
stats_endpoint=None,
env=None, name=None, context=None,
background=False, stream_backend="thread",
plugins=None, debug=False):
"""Creates a Arbiter and a single watcher in it.
Options:
- **watchers** -- a list of watchers. A watcher in that case is a
dict containing:
- **name** -- the name of the watcher (default: None)
- **cmd** -- the command line used to run the Watcher.
- **args** -- the args for the command (list or string).
- **executable** -- When executable is given, the first item in
the args sequence obtained from **cmd** is still treated by most
programs as the command name, which can then be different from the
actual executable name. It becomes the display name for the executing
program in utilities such as **ps**.
- **numprocesses** -- the number of processes to spawn (default: 1).
- **warmup_delay** -- the delay in seconds between two spawns
(default: 0)
- **shell** -- if True, the processes are run in the shell
(default: False)
- **working_dir** - the working dir for the processes (default: None)
- **uid** -- the user id used to run the processes (default: None)
- **gid** -- the group id used to run the processes (default: None)
- **env** -- the environment passed to the processes (default: None)
- **send_hup**: if True, a process reload will be done by sending
the SIGHUP signal. (default: False)
- **stdout_stream**: a mapping containing the options for configuring
the stdout stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **stderr_stream**: a mapping containing the options for configuring
the stderr stream. Default to None. When provided, may contain:
- **class**: the fully qualified name of the class to use for
streaming. Defaults to circus.stream.FileStream
- **refresh_time**: the delay between two stream checks. Defaults
to 0.3 seconds.
- any other key will be passed the class constructor.
- **max_retry**: the number of times we attempt to start a process,
before we abandon and stop the whole watcher. (default: 5)
- **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
- **pubsub_endpoint** -- the zmq entry point for the pubsub
(default: 'tcp://127.0.0.1:5556')
- **stats_endpoint** -- the stats endpoint. If not provided,
the *circusd-stats* process will not be launched. (default: None)
- **context** -- the zmq context (default: None)
- **background** -- If True, the arbiter is launched in a thread in the
background (default: False)
- **stream_backend** -- the backend that will be used for the streaming
process. Can be *thread* or *gevent*. When set to *gevent* you need
to have *gevent* and *gevent_zmq* installed. (default: thread)
- **plugins** -- a list of plugins. Each item is a mapping with:
- **use** -- Fully qualified name that points to the plugin class
- every other value is passed to the plugin in the **config** option
- **debug** -- If True the arbiter is launched in debug mode
(default: False)
"""
from circus.util import DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB
if controller is None:
controller = DEFAULT_ENDPOINT_DEALER
if pubsub_endpoint is None:
pubsub_endpoint = DEFAULT_ENDPOINT_SUB
if stream_backend == 'gevent':
try:
import gevent # NOQA
import gevent_zeromq # NOQA
except ImportError:
sys.stderr.write("stream_backend set to gevent, " +
"but gevent or gevent_zeromq isn't installed\n")
sys.stderr.write("Exiting...")
sys.exit(1)
from gevent import monkey
from gevent_zeromq import monkey_patch
monkey.patch_all()
monkey_patch()
from circus.watcher import Watcher
if background:
from circus.arbiter import ThreadedArbiter as Arbiter # NOQA
else:
from circus.arbiter import Arbiter # NOQA
_watchers = []
for watcher in watchers:
cmd = watcher['cmd']
watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
watcher['stream_backend'] = stream_backend
_watchers.append(Watcher.load_from_config(watcher))
return Arbiter(_watchers, controller, pubsub_endpoint,
stats_endpoint=stats_endpoint,
context=context, plugins=plugins, debug=debug)