Exemplo n.º 1
0
    def run(self):
        # crude check for inplace build:
        try:
            import gevent_zeromq
        except ImportError:
            print_exc()
            print("Could not import gevent_zeromq!")
            print(
                "You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work."
            )
            print(
                "If you did build gevent_zeromq in-place, then this is a real error."
            )
            sys.exit(1)

        gevent_zeromq.monkey_patch(test_suite=True)  # monkey patch
        import zmq
        self._zmq_dir = os.path.dirname(zmq.__file__)

        if nose is None:
            print(
                "nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs."
            )
            return self.run_unittest()
        else:
            return self.run_nose()
Exemplo n.º 2
0
    def run(self, handler):
        try:
            from socketio.server import SocketIOServer
        except ImportError:
            raise ImportError('You need to install gevent_socketio')

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

        namespace = self.options.get('namespace', 'socket.io')
        policy_server = self.options.get('policy_server', False)

        if self.fd is not None:
            sock = socket.fromfd(self.fd, socket.AF_INET, socket.SOCK_STREAM)
        else:
            sock = (self.host, self.port)

        socket_server = SocketIOServer(sock,
                                       handler,
                                       namespace=namespace,
                                       policy_server=policy_server)
        handler.socket_server = socket_server
        socket_server.serve_forever()
Exemplo n.º 3
0
    def run(self):
        # crude check for inplace build:
        try:
            import gevent_zeromq
        except ImportError:
            print_exc()
            print("Could not import gevent_zeromq!")
            print(
                "You must build gevent_zeromq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work."
            )
            print(
                "If you did build gevent_zeromq in-place, then this is a real error."
            )
            sys.exit(1)

        gevent_zeromq.monkey_patch(test_suite=True)  # monkey patch
        import zmq
        self._zmq_dir = os.path.dirname(zmq.__file__)

        if nose is None:
            print("nose unavailable, skipping tests.")
        else:
            return nose.core.TestProgram(
                argv=["", '-vvs',
                      os.path.join(self._zmq_dir, 'tests')])
Exemplo n.º 4
0
    def setup_ipython_embed(shell_api=None):
        from gevent_zeromq import monkey_patch
        monkey_patch()

        # patch in device:
        # gevent-zeromq does not support devices, which block in the C layer.
        # we need to support the "heartbeat" which is a simple bounceback, so we
        # simulate it using the following method.
        import zmq
        orig_device = zmq.device

        def device_patch(dev_type, insock, outsock, *args):
            if dev_type == zmq.FORWARDER:
                while True:
                    m = insock.recv()
                    outsock.send(m)
            else:
                orig_device.device(dev_type, insock, outsock, *args)

        zmq.device = device_patch

        # patch in auto-completion support
        # added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
        # we effectively make the same patches via monkeypatching
        from IPython.core.interactiveshell import InteractiveShell
        from IPython.zmq.ipkernel import IPKernelApp
        old_start = IPKernelApp.start
        old_set_completer_frame = InteractiveShell.set_completer_frame

        def new_start(appself):
            # restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
            bound_scf = old_set_completer_frame.__get__(
                appself.shell, InteractiveShell)
            appself.shell.set_completer_frame = bound_scf
            appself.shell.set_completer_frame()
            old_start(appself)

        IPKernelApp.start = new_start

        from IPython import embed_kernel
        ipy_config = _setup_ipython_config()

        # set specific manhole options
        import tempfile  #, shutil
        from mock import patch
        temp_dir = tempfile.mkdtemp()
        ipy_config.Application.ipython_dir = temp_dir

        with patch(
                "IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
        ):
            embed_kernel(local_ns=shell_api,
                         config=ipy_config)  # blocks until INT
Exemplo n.º 5
0
    def setup_ipython_embed(shell_api=None):
        _assert_ipython_dir()
        from gevent_zeromq import monkey_patch
        monkey_patch()

        # patch in device:
        # gevent-zeromq does not support devices, which block in the C layer.
        # we need to support the "heartbeat" which is a simple bounceback, so we
        # simulate it using the following method.
        import zmq
        orig_device = zmq.device

        def device_patch(dev_type, insock, outsock, *args):
            if dev_type == zmq.FORWARDER:
                while True:
                    m = insock.recv()
                    outsock.send(m)
            else:
                orig_device.device(dev_type, insock, outsock, *args)

        zmq.device = device_patch

        # patch in auto-completion support
        # added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
        # we effectively make the same patches via monkeypatching
        from IPython.core.interactiveshell import InteractiveShell
        from IPython.zmq.ipkernel import IPKernelApp
        old_start = IPKernelApp.start
        old_set_completer_frame = InteractiveShell.set_completer_frame

        def new_start(appself):
            # restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
            bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
            appself.shell.set_completer_frame = bound_scf
            appself.shell.set_completer_frame()
            old_start(appself)

        IPKernelApp.start = new_start

        from IPython import embed_kernel
        ipy_config = _setup_ipython_config()

        # set specific manhole options
        import tempfile#, shutil
        from mock import patch
        temp_dir = tempfile.mkdtemp()
        ipy_config.Application.ipython_dir = temp_dir

        with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
            embed_kernel(local_ns=shell_api, config=ipy_config)      # blocks until INT
Exemplo n.º 6
0
 def configure(self, options):
     self.options = options
     if self.use_gevent:
         import gevent_zeromq
         from gevent import monkey
         monkey.patch_all()
         gevent_zeromq.monkey_patch()
     self._configure_logging()
     # These imports need to happen *after* monkey patching
     from SimpleSeer.Session import Session
     from SimpleSeer import models as M
     self.session = Session(options.config)
     if self.remote_seer:
         from SimpleSeer.SimpleSeer import SimpleSeer as SS
         SS(disable=True)
     if self.session.mongo.get('is_slave'):
         M.base.SimpleDoc.meta['auto_create_index'] = False
     if options.profile_heap: self._start_profile_heap()
Exemplo n.º 7
0
    def run(self, handler):
        try:
            from socketio.server import SocketIOServer
        except ImportError:
            raise ImportError('You need to install gevent_socketio')

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

        namespace = self.options.get('namespace', 'socket.io')
        policy_server = self.options.get('policy_server', False)
        socket_server = SocketIOServer((self.host, self.port), handler,
                                       namespace=namespace,
                                       policy_server=policy_server)
        handler.socket_server = socket_server
        socket_server.serve_forever()
Exemplo n.º 8
0
    def __init__(self, session):

        monkey.patch_socket() 
        import gevent_zeromq
        gevent_zeromq.monkey_patch()
         #we do use greenlets, but only patch sock stuff
        #other stuff messes up the

        config = session.arduino 
        boardglob = config['board']
        boards = glob(boardglob)
        if not len(boards):
              raise Exception("No Arduino found")
        
        self.board = Arduino(boards[0])
        #self.iterator = util.Iterator(self.board)
        #self.iterator.daemon = True
        #self.iterator.start()

        #initialize servo objects
        self.servos = {}
        if "servos" in config:
            for servo in config["servos"]:
                self.servos[servo['name']] = Servo(servo['pin'], self.board)

        #initialize light objects
        self.digitalouts = {}
        if "digitalouts" in config:
            for do in config["digitalouts"]:
                self.digitalouts[do['name']] = DigitalOut(do['pin'], self.board)

        if "digitalouts" in config or "servos" in config:
            self.subsock = ChannelManager().subscribe("ControlOutput/")

        self.buttons = []
        if "buttons" in config:
            for button in config["buttons"]:
                self.buttons.append(Button(button['pin'], button['message'], self.board))

        self.potentiometers = []
        if "potentiometers" in config:
            for pot in config["potentiometers"]:
                self.buttons.append(Potentiometer(pot['pin'], pot['name'], self.board))                
Exemplo n.º 9
0
    def run(self, handler):
        try:
            from socketio.server import SocketIOServer
        except ImportError:
            raise ImportError('You need to install gevent_socketio')

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

        namespace = self.options.get('namespace', 'socket.io')
        policy_server = self.options.get('policy_server', False)
        socket_server = SocketIOServer((self.host, self.port),
                                       handler,
                                       namespace=namespace,
                                       policy_server=policy_server)
        handler.socket_server = socket_server
        socket_server.serve_forever()
Exemplo n.º 10
0
    def run(self):
        # crude check for inplace build:
        try:
            import gevent_zeromq
        except ImportError:
            print_exc()
            print ("Could not import gevent_zeromq!")
            print ("You must build gevent_zeromq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.")
            print ("If you did build gevent_zeromq in-place, then this is a real error.")
            sys.exit(1)

        gevent_zeromq.monkey_patch(test_suite=True) # monkey patch
        import zmq
        self._zmq_dir = os.path.dirname(zmq.__file__)

        if nose is None:
            print ("nose unavailable, skipping tests.")
        else:
            return nose.core.TestProgram(argv=["", '-vvs', os.path.join(self._zmq_dir, 'tests')])
Exemplo n.º 11
0
    def run(self):
        # crude check for inplace build:
        try:
            import gevent_zeromq
        except ImportError:
            print_exc()
            print ("Could not import gevent_zeromq!")
            print ("You must build pyzmq with 'python setup.py build_ext --inplace' for 'python setup.py test' to work.")
            print ("If you did build gevent_zeromq in-place, then this is a real error.")
            sys.exit(1)

        gevent_zeromq.monkey_patch(test_suite=True) # monkey patch
        import zmq
        self._zmq_dir = os.path.dirname(zmq.__file__)

        if nose is None:
            print ("nose unavailable, falling back on unittest. Skipped tests will appear as ERRORs.")
            return self.run_unittest()
        else:
            return self.run_nose()
Exemplo n.º 12
0
def monkey_patch():
    """Hook to monkey-patch the interpreter for this IO module.

    This calls the standard gevent monkey-patching routines.  Don't worry,
    it's not called by default unless you're running from the command line.
    """
    gevent.monkey.patch_all()
    gevent_zeromq.monkey_patch()
    #  Patch signal module for gevent compatability.
    #  Courtesy of http://code.google.com/p/gevent/issues/detail?id=49
    import signal
    _orig_signal = signal.signal
    def gevent_signal_wrapper(signum,*args,**kwds):
        handler = signal.getsignal(signum)
        if callable(handler):
            handler(signum,None)
    def gevent_signal(signum,handler):
        _orig_signal(signum,handler)
        return gevent.hub.signal(signum,gevent_signal_wrapper,signum)
    signal.signal = gevent_signal
Exemplo n.º 13
0
def setUp():
    try:
        from gevent import monkey       # NOQA
        try:
            from gevent_zeromq import monkey_patch, IOLOOP_IS_MONKEYPATCHED  # NOQA
            monkey.patch_all()
            monkey_patch()
        except ImportError:
            msg = """We have detected that you have gevent in your
            environment. In order to have Circus working, you *must*
            install gevent_zmq from :

            https://github.com/tarekziade/gevent-zeromq

            Circus will not need this in the future once
            pyzmq gets a green poller:

            https://github.com/zeromq/pyzmq/issues/197
            """
            raise ImportError(msg)
    except ImportError:
        pass
Exemplo n.º 14
0
def setUp():
    from circus import _patch   # NOQA
    try:
        import gevent                   # NOQA
        from gevent import monkey       # NOQA
        try:
            import zmq.eventloop as old_io
            import zmq.green as zmq         # NOQA
            old_io.ioloop.Poller = zmq.Poller
        except ImportError:
            # older version
            try:
                from gevent_zeromq import (                     # NOQA
                        monkey_patch, IOLOOP_IS_MONKEYPATCHED)  # NOQA
                monkey_patch()
                warnings.warn("gevent_zeromq is deprecated, please "
                            "use PyZMQ >= 2.2.0.1")
            except ImportError:
                raise ImportError(_MSG)

        monkey.patch_all()
    except ImportError:
        pass
Exemplo n.º 15
0
def monkey_patch():
    """Hook to monkey-patch the interpreter for this IO module.

    This calls the standard gevent monkey-patching routines.  Don't worry,
    it's not called by default unless you're running from the command line.
    """
    gevent.monkey.patch_all()
    gevent_zeromq.monkey_patch()
    #  Patch signal module for gevent compatability.
    #  Courtesy of http://code.google.com/p/gevent/issues/detail?id=49
    import signal
    _orig_signal = signal.signal

    def gevent_signal_wrapper(signum, *args, **kwds):
        handler = signal.getsignal(signum)
        if callable(handler):
            handler(signum, None)

    def gevent_signal(signum, handler):
        _orig_signal(signum, handler)
        return gevent.hub.signal(signum, gevent_signal_wrapper, signum)

    signal.signal = gevent_signal
Exemplo n.º 16
0
def setUp():
    from circus import _patch  # NOQA
    try:
        from gevent import monkey  # NOQA
        try:
            from gevent_zeromq import monkey_patch, IOLOOP_IS_MONKEYPATCHED  # NOQA
            monkey.patch_all()
            monkey_patch()
        except ImportError:
            msg = """We have detected that you have gevent in your
            environment. In order to have Circus working, you *must*
            install gevent_zmq from :

            https://github.com/tarekziade/gevent-zeromq

            Circus will not need this in the future once
            pyzmq gets a green poller:

            https://github.com/zeromq/pyzmq/issues/197
            """
            raise ImportError(msg)
    except ImportError:
        pass
Exemplo n.º 17
0
    def run(self, handler):
        try:
            from socketio.server import SocketIOServer
        except ImportError:
            raise ImportError('You need to install gevent_socketio')

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

        namespace = self.options.get('namespace', 'socket.io')
        policy_server = self.options.get('policy_server', False)

        if self.fd is not None:
            sock = socket.fromfd(self.fd, socket.AF_INET, socket.SOCK_STREAM)
        else:
            sock = (self.host, self.port)

        socket_server = SocketIOServer(sock, handler,
                                       namespace=namespace,
                                       policy_server=policy_server)
        handler.socket_server = socket_server
        socket_server.serve_forever()
Exemplo n.º 18
0
def get_arbiter(watchers,
                controller='tcp://127.0.0.1:5555',
                pubsub_endpoint='tcp://127.0.0.1:5556',
                stats_endpoint=None,
                env=None,
                name=None,
                context=None,
                background=False,
                stream_backend="thread",
                plugins=None):
    """Creates a Arbiter and a single watcher in it.

    Options:

    - **watchers** -- a list of watchers. A watcher in that case is a
      dict containing:

        - **name** -- the name of the watcher (default: None)
        - **cmd** -- the command line used to run the Watcher.
        - **args** -- the args for the command (list or string).
        - **executable** -- When executable is given, the first item in
          the args sequence obtained from **cmd** is still treated by most
          programs as the command name, which can then be different from the
          actual executable name. It becomes the display name for the executing
          program in utilities such as **ps**.

        - **numprocesses** -- the number of processes to spawn (default: 1).
        - **warmup_delay** -- the delay in seconds between two spawns
          (default: 0)
        - **shell** -- if True, the processes are run in the shell
          (default: False)
        - **working_dir** - the working dir for the processes (default: None)
        - **uid** -- the user id used to run the processes (default: None)
        - **gid** -- the group id used to run the processes (default: None)
        - **env** -- the environment passed to the processes (default: None)
        - **send_hup**: if True, a process reload will be done by sending
          the SIGHUP signal. (default: False)
        - **stdout_stream**: a mapping containing the options for configuring
          the stdout stream. Default to None. When provided, may contain:

            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **stderr_stream**: a mapping containing the options for configuring
          the stderr stream. Default to None. When provided, may contain:

            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **max_retry**: the number of times we attempt to start a process,
          before we abandon and stop the whole watcher. (default: 5)

    - **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
    - **pubsub_endpoint** -- the zmq entry point for the pubsub
      (default: 'tcp://127.0.0.1:5556')
    - **stats_endpoint** -- the stats endpoint. If not provided,
      the *circusd-stats* process will not be launched. (default: None)
    - **context** -- the zmq context (default: None)
    - **background** -- If True, the arbiter is launched in a thread in the
      background (default: False)
    - **stream_backend** -- the backend that will be used for the streaming
      process. Can be *thread* or *gevent*. When set to *gevent* you need
      to have *gevent* and *gevent_zmq* installed. (default: thread)
    - **plugins** -- a list of plugins. Each item is a mapping with:

        - **use** -- Fully qualified name that points to the plugin class
        - every other value is passed to the plugin in the **config** option

    """
    if stream_backend == 'gevent':
        try:
            import gevent  # NOQA
            import gevent_zeromq  # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    from circus.watcher import Watcher
    if background:
        from circus.arbiter import ThreadedArbiter as Arbiter  # NOQA
    else:
        from circus.arbiter import Arbiter  # NOQA

    _watchers = []

    for watcher in watchers:
        cmd = watcher['cmd']
        watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
        watcher['stream_backend'] = stream_backend
        _watchers.append(Watcher.load_from_config(watcher))

    return Arbiter(_watchers,
                   controller,
                   pubsub_endpoint,
                   stats_endpoint=stats_endpoint,
                   context=context,
                   plugins=plugins)
Exemplo n.º 19
0
def get_config(config_file):
    if not os.path.exists(config_file):
        sys.stderr.write("the configuration file %r does not exist\n" % config_file)
        sys.stderr.write("Exiting...\n")
        sys.exit(1)

    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config["check"] = dget("circus", "check_delay", 5, int)
    config["endpoint"] = dget("circus", "endpoint", "tcp://127.0.0.1:5555")
    config["pubsub_endpoint"] = dget("circus", "pubsub_endpoint", "tcp://127.0.0.1:5556")
    config["stats_endpoint"] = dget("circus", "stats_endpoint", None, str)
    config["warmup_delay"] = dget("circus", "warmup_delay", 0, int)
    config["httpd"] = dget("circus", "httpd", False, bool)
    config["httpd_host"] = dget("circus", "httpd_host", "localhost", str)
    config["httpd_port"] = dget("circus", "httpd_port", 8080, int)

    stream_backend = dget("circus", "stream_backend", "thread")

    if stream_backend == "gevent":
        try:
            import gevent  # NOQA
            import gevent_zeromq  # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " + "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch

        monkey.patch_all()
        monkey_patch()

    config["stream_backend"] = stream_backend

    # Initialize watchers, plugins & sockets to manage
    watchers = []
    plugins = []
    sockets = []

    for section in cfg.sections():
        if section.startswith("socket:"):
            sock = dict(cfg.items(section))
            sock["name"] = section.split("socket:")[-1].lower()
            sockets.append(sock)

        if section.startswith("plugin:"):
            plugins.append(dict(cfg.items(section)))

        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher["name"] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == "cmd":
                    watcher["cmd"] = val
                elif opt == "args":
                    watcher["args"] = val
                elif opt == "numprocesses":
                    watcher["numprocesses"] = dget(section, "numprocesses", 1, int)
                elif opt == "warmup_delay":
                    watcher["warmup_delay"] = dget(section, "warmup_delay", 0, int)
                elif opt == "executable":
                    watcher["executable"] = dget(section, "executable", None, str)
                elif opt == "working_dir":
                    watcher["working_dir"] = val
                elif opt == "shell":
                    watcher["shell"] = dget(section, "shell", False, bool)
                elif opt == "uid":
                    watcher["uid"] = val
                elif opt == "gid":
                    watcher["gid"] = val
                elif opt == "send_hup":
                    watcher["send_hup"] = dget(section, "send_hup", False, bool)
                elif opt == "check_flapping":
                    watcher["check_flapping"] = dget(section, "check_flapping", True, bool)
                elif opt == "max_retry":
                    watcher["max_retry"] = dget(section, "max_retry", 5, int)
                elif opt == "graceful_timout":
                    watcher["graceful_timeout"] = dget(section, "graceful_timeout", 30, int)
                elif opt.startswith("stderr_stream") or opt.startswith("stdout_stream"):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith("rlimit_"):
                    limit = opt[7:]
                    watcher["rlimits"][limit] = int(val)
                elif opt == "priority":
                    watcher["priority"] = dget(section, "priority", 0, int)
                elif opt == "use_sockets":
                    watcher["use_sockets"] = dget(section, "use_sockets", False, bool)
                elif opt == "singleton":
                    watcher["singleton"] = dget(section, "singleton", False, bool)
                else:
                    # freeform
                    watcher[opt] = val

            # set the stream backend
            watcher["stream_backend"] = stream_backend

            watchers.append(watcher)

    config["watchers"] = watchers
    config["plugins"] = plugins
    config["sockets"] = sockets
    return config
Exemplo n.º 20
0
import uuid

import gevent
import gevent.monkey
gevent.monkey.patch_all()
import gevent_zeromq
gevent_zeromq.monkey_patch()
import zmq

import arrayserver.server.tests
import arrayserver.server.redisutils as redisutils
import arrayserver.server.arrayserverconfig as arrayserverconfig
import arrayserver.server.arrayserverconfig.orderedyaml as orderedyaml
import arrayserver.server.arrayservernode as arrayservernode
import arrayserver.server.arrayserverbroker as arrayserverbroker
import arrayserver.server.tests.test_utils as test_utils

import redis
import collections
import sys
import yaml
import os
import logging
import time
import atexit

log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

def start_redis(datapath, redis_port):
    proc = redisutils.RedisProcess(redis_port, datapath,
Exemplo n.º 21
0
import logging
import os
import sys

try:
    import gevent  # NOQA
    try:
        from gevent_zeromq import monkey_patch, IOLOOP_IS_MONKEYPATCHED  # NOQA
        monkey_patch()
    except ImportError:
        msg = """We have detected that you have gevent in your
        environment. In order to have Circus working, you *must*
        install gevent_zmq from :

          https://github.com/tarekziade/gevent-zeromq

        Circus will not need this in the future once
        pyzmq gets a green poller:

          https://github.com/zeromq/pyzmq/issues/197
        """
        raise ImportError(msg)
except ImportError:
    pass

version_info = (0, 5)
__version__ = ".".join(map(str, version_info))

logger = logging.getLogger('circus')

Exemplo n.º 22
0
def includeme(config):
    try:
        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()
    except ImportError:
        pass

    config.include("cornice")
    config.include("mozsvc")
    config.scan("tokenserver.views")
    settings = config.registry.settings

    # default metlog setup
    if 'metlog.backend' not in settings:
        settings['metlog.backend'] = 'mozsvc.metrics.MetlogPlugin'
        settings['metlog.enabled'] = True
        settings['metlog.sender_class'] = \
                'metlog.senders.StdOutSender'

    metlog_wrapper = load_from_settings('metlog', settings)

    if settings['metlog.enabled']:
        for logger in ('tokenserver', 'mozsvc', 'powerhose'):
            hook_logger(logger, metlog_wrapper.client)

    config.registry['metlog'] = metlog_wrapper.client

    # initializes the assignment backend
    load_and_register("tokenserver", config)

    # initialize the powerhose and browserid backends if they exist
    for section in ("powerhose", "browserid"):
        try:
            load_and_register(section, config)
        except NoSectionError:
            pass

    # load apps and set them up back in the setting
    key = 'tokenserver.applications'
    applications = defaultdict(list)
    for element in settings.get(key, '').split(','):
        element = element.strip()
        if element == '':
            continue
        element = element.split('-')
        if len(element) != 2:
            continue
        app, version = element
        applications[app].append(version)

    settings[key] = applications

    # load the secrets file(s)
    key = 'tokenserver.secrets_file'
    secret_file = settings[key]
    if not isinstance(secret_file, list):
        secret_file = [secret_file]

    files = []
    for line in secret_file:
        secret_file = [file for file in [file.strip() for file in line.split()]
                       if file != '']
        files.extend(secret_file)

    settings[key] = Secrets(files)
    read_endpoints(config)
Exemplo n.º 23
0
def get_config(config_file):
    if not os.path.exists(config_file):
        sys.stderr.write("the configuration file %r does not exist\n" %
                config_file)
        sys.stderr.write("Exiting...\n")
        sys.exit(1)

    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config['check'] = dget('circus', 'check_delay', 5, int)
    config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
    config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                     DEFAULT_ENDPOINT_SUB)
    config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
    config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
    config['httpd'] = dget('circus', 'httpd', False, bool)
    config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
    config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
    config['debug'] = dget('circus', 'debug', False, bool)
    stream_backend = dget('circus', 'stream_backend', 'thread')
    if stream_backend == 'gevent':
        try:
            import gevent           # NOQA
            import gevent_zeromq    # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    config['stream_backend'] = stream_backend

    # Initialize watchers, plugins & sockets to manage
    watchers = []
    plugins = []
    sockets = []

    for section in cfg.sections():
        if section.startswith("socket:"):
            sock = dict(cfg.items(section))
            sock['name'] = section.split("socket:")[-1].lower()
            sockets.append(sock)

        if section.startswith("plugin:"):
            plugins.append(dict(cfg.items(section)))

        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher['name'] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == 'cmd':
                    watcher['cmd'] = val
                elif opt == 'args':
                    watcher['args'] = val
                elif opt == 'numprocesses':
                    watcher['numprocesses'] = dget(section, 'numprocesses', 1,
                            int)
                elif opt == 'warmup_delay':
                    watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
                            int)
                elif opt == 'executable':
                    watcher['executable'] = dget(section, 'executable', None,
                            str)
                elif opt == 'working_dir':
                    watcher['working_dir'] = val
                elif opt == 'shell':
                    watcher['shell'] = dget(section, 'shell', False, bool)
                elif opt == 'uid':
                    watcher['uid'] = val
                elif opt == 'gid':
                    watcher['gid'] = val
                elif opt == 'send_hup':
                    watcher['send_hup'] = dget(section, 'send_hup', False,
                            bool)
                elif opt == 'check_flapping':
                    watcher['check_flapping'] = dget(section, 'check_flapping',
                                                     True, bool)
                elif opt == 'max_retry':
                    watcher['max_retry'] = dget(section, "max_retry", 5, int)
                elif opt == 'graceful_timout':
                    watcher['graceful_timeout'] = dget(section,
                            "graceful_timeout", 30, int)
                elif opt.startswith('stderr_stream') or \
                        opt.startswith('stdout_stream'):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith('rlimit_'):
                    limit = opt[7:]
                    watcher['rlimits'][limit] = int(val)
                elif opt == 'priority':
                    watcher['priority'] = dget(section, "priority", 0, int)
                elif opt == 'use_sockets':
                    watcher['use_sockets'] = dget(section, "use_sockets",
                                                  False, bool)
                elif opt == 'singleton':
                    watcher['singleton'] = dget(section, "singleton", False,
                                                bool)
                elif opt == 'stream_backend':
                    watcher['stream_backend'] = val
                elif opt == 'copy_env':
                    watcher['copy_env'] = dget(section, "copy_env", False,
                                                bool)
                elif opt == 'copy_path':
                    watcher['copy_path'] = dget(section, "copy_path", False,
                                                bool)
                elif opt.startswith('hooks.'):
                    hook_name = opt[len('hooks.'):]
                    val = [elmt.strip() for elmt in val.split(',', 1)]
                    if len(val) == 1:
                        val.append(False)
                    else:
                        val[1] = to_boolean(val[1])

                    watcher['hooks'][hook_name] = val

                else:
                    # freeform
                    watcher[opt] = val

            # set the stream backend
            if 'stream_backend' not in watcher:
                watcher['stream_backend'] = stream_backend
            watchers.append(watcher)

    config['watchers'] = watchers
    config['plugins'] = plugins
    config['sockets'] = sockets
    return config
Exemplo n.º 24
0
def get_config(config_file):
    if not os.path.exists(config_file):
        sys.stderr.write("the configuration file %r does not exist\n" %
                config_file)
        sys.stderr.write("Exiting...\n")
        sys.exit(1)

    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config['check'] = dget('circus', 'check_delay', 5, int)
    config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
    config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                     'tcp://127.0.0.1:5556')
    config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
    stream_backend = dget('circus', 'stream_backend', 'thread')

    if stream_backend == 'gevent':
        try:
            import gevent           # NOQA
            import gevent_zeromq    # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    config['stream_backend'] = stream_backend

    # Initialize watchers, plugins & sockets to manage
    watchers = []
    plugins = []
    sockets = []

    for section in cfg.sections():
        if section.startswith("socket:"):
            sock = dict(cfg.items(section))
            sock['name'] = section.split("socket:")[-1]
            sockets.append(sock)

        if section.startswith("plugin:"):
            plugins.append(dict(cfg.items(section)))

        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher['name'] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == 'cmd':
                    watcher['cmd'] = val
                elif opt == 'args':
                    watcher['args'] = val
                elif opt == 'numprocesses':
                    watcher['numprocesses'] = dget(section, 'numprocesses', 1,
                            int)
                elif opt == 'warmup_delay':
                    watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
                            int)
                elif opt == 'executable':
                    watcher['executable'] = dget(section, 'executable', None,
                            str)
                elif opt == 'working_dir':
                    watcher['working_dir'] = val
                elif opt == 'shell':
                    watcher['shell'] = dget(section, 'shell', False, bool)
                elif opt == 'uid':
                    watcher['uid'] = val
                elif opt == 'gid':
                    watcher['gid'] = val
                elif opt == 'send_hup':
                    watcher['send_hup'] = dget(section, 'send_hup', False,
                            bool)
                elif opt == 'check_flapping':
                    watcher['check_flapping'] = dget(section, 'check_flapping',
                                                     True, bool)
                elif opt == 'max_retry':
                    watcher['max_retry'] = dget(section, "max_retry", 5, int)
                elif opt == 'graceful_timout':
                    watcher['graceful_timeout'] = dget(section,
                            "graceful_timeout", 30, int)
                elif opt.startswith('stderr_stream') or \
                        opt.startswith('stdout_stream'):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith('rlimit_'):
                    limit = opt[7:]
                    watcher['rlimits'][limit] = int(val)
                elif opt == 'priority':
                    watcher['priority'] = dget(section, "priority", 0, int)
                elif opt == 'use_sockets':
                    watcher['use_sockets'] = dget(section, "use_sockets",
                                                  False, bool)
                else:
                    # freeform
                    watcher[opt] = val

            # set the stream backend
            watcher['stream_backend'] = stream_backend

            watchers.append(watcher)

    config['watchers'] = watchers
    config['plugins'] = plugins
    config['sockets'] = sockets
    return config
Exemplo n.º 25
0
also the gevent thread. A proper example is show on green_heart.py where both,
numpy calculation and gevent, are kept responsive.

Authors
-------
* MinRK
* Pedro Algarvio
"""

import os
import time
import numpy
import gevent
import threading
import gevent_zeromq
gevent_zeromq.monkey_patch(zmq_devices=True)

from gevent_zeromq import zmq
from gevent_zeromq.devices import ThreadDevice


def im_alive(t=None):
    print "I'm alive!"
    if t:
        gevent.spawn_later(t, im_alive, t)

def run_blocking_call(A):
    print "starting blocking loop"
    tic = time.time()
    numpy.dot(A,A.transpose())
    print "blocked for %.3f s"%(time.time()-tic)
Exemplo n.º 26
0
def get_config(config_file):
    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config['check'] = dget('circus', 'check_delay', 5, int)
    config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
    config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                     'tcp://127.0.0.1:5556')

    stream_backend = dget('circus', 'stream_backend', 'thread')
    if stream_backend == 'gevent':
        try:
            import gevent           # NOQA
            import gevent_zeromq    # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    config['stream_backend'] = stream_backend

    # Initialize watchers to manage
    watchers = []
    for section in cfg.sections():
        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher['name'] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == 'cmd':
                    watcher['cmd'] = val
                elif opt == 'args':
                    watcher['args'] = val
                elif opt == 'numprocesses':
                    watcher['numprocesses'] = dget(section, 'numprocesses', 1,
                            int)
                elif opt == 'warmup_delay':
                    watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
                            int)
                elif opt == 'executable':
                    watcher['executable'] = dget(section, 'executable', None,
                            str)
                elif opt == 'working_dir':
                    watcher['working_dir'] = val
                elif opt == 'shell':
                    watcher['shell'] = dget(section, 'shell', False, bool)
                elif opt == 'uid':
                    watcher['uid '] = val
                elif opt == 'gid':
                    watcher['gid'] = val
                elif opt == 'send_hup':
                    watcher['send_hup'] = dget(section, 'send_hup', False,
                            bool)
                elif opt == 'flapping_attempts':
                    watcher['flapping_attempts'] = dget(section,
                                                        "flapping_attempts", 2,
                                                        int)
                elif opt == 'flapping_window':
                    watcher['flapping_window'] = dget(section,
                                                      "flapping_window", 1,
                                                      int)
                elif opt == 'retry_in':
                    watcher['retry_in'] = dget(section, "retry_in", 7, int)
                elif opt == 'max_retry':
                    watcher['max_retry'] = dget(section, "max_retry", 5, int)
                elif opt == 'graceful_timout':
                    watcher['graceful_timeout'] = dget(section,
                            "graceful_timeout", 30, int)
                elif opt.startswith('stderr_stream') or \
                        opt.startswith('stdout_stream'):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith('rlimit_'):
                    limit = opt[7:]
                    watcher['rlimits'][limit] = int(val)
                elif opt == 'priority':
                    watcher['priority'] = dget(section, "priority", 0, int)

            # second pass, parse stream conf
            stdout_conf = watcher.get('stdout_stream', {})
            watcher['stdout_stream'] = stream_config(watcher['name'],
                stdout_conf)

            stderr_conf = watcher.get('stderr_stream', {})
            watcher['stderr_stream'] = stream_config(watcher['name'],
                stderr_conf)

            # set the stream backend
            watcher['stream_backend'] = stream_backend

            watchers.append(watcher)

    config['watchers'] = watchers
    return config
Exemplo n.º 27
0
Arquivo: rtmama.py Projeto: tmc/zguide
# encoding: utf-8
#
#   Custom routing Router to Mama (XREP to REQ)
#
#   Author: Jeremy Avnet (brainsik) <spork(dash)zmq(at)theory(dot)org>
#   Author: Travis Cline <*****@*****.**> - gevent-zeromq adaption


import gevent
import gevent_zeromq
gevent_zeromq.monkey_patch() # patching required because of zmq import in zhelpers


import time
import random

import zhelpers

NBR_WORKERS = 10


def worker_thread(context):
    worker = context.socket(zmq.REQ)

    # We use a string identity for ease here
    zhelpers.set_id(worker)
    worker.connect("ipc://routing.ipc")

    total = 0
    while True:
        # Tell the router we're ready for work
Exemplo n.º 28
0
def get_config(config_file):
    if not os.path.exists(config_file):
        sys.stderr.write("the configuration file %r does not exist\n" %
                         config_file)
        sys.stderr.write("Exiting...\n")
        sys.exit(1)

    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config['check'] = dget('circus', 'check_delay', 5, int)
    config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
    config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                     'tcp://127.0.0.1:5556')
    config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
    stream_backend = dget('circus', 'stream_backend', 'thread')

    if stream_backend == 'gevent':
        try:
            import gevent  # NOQA
            import gevent_zeromq  # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    config['stream_backend'] = stream_backend

    # Initialize watchers, plugins & sockets to manage
    watchers = []
    plugins = []
    sockets = []

    for section in cfg.sections():
        if section.startswith("socket:"):
            sock = dict(cfg.items(section))
            sock['name'] = section.split("socket:")[-1]
            sockets.append(sock)

        if section.startswith("plugin:"):
            plugins.append(dict(cfg.items(section)))

        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher['name'] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == 'cmd':
                    watcher['cmd'] = val
                elif opt == 'args':
                    watcher['args'] = val
                elif opt == 'numprocesses':
                    watcher['numprocesses'] = dget(section, 'numprocesses', 1,
                                                   int)
                elif opt == 'warmup_delay':
                    watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
                                                   int)
                elif opt == 'executable':
                    watcher['executable'] = dget(section, 'executable', None,
                                                 str)
                elif opt == 'working_dir':
                    watcher['working_dir'] = val
                elif opt == 'shell':
                    watcher['shell'] = dget(section, 'shell', False, bool)
                elif opt == 'uid':
                    watcher['uid'] = val
                elif opt == 'gid':
                    watcher['gid'] = val
                elif opt == 'send_hup':
                    watcher['send_hup'] = dget(section, 'send_hup', False,
                                               bool)
                elif opt == 'check_flapping':
                    watcher['check_flapping'] = dget(section, 'check_flapping',
                                                     True, bool)
                elif opt == 'max_retry':
                    watcher['max_retry'] = dget(section, "max_retry", 5, int)
                elif opt == 'graceful_timout':
                    watcher['graceful_timeout'] = dget(section,
                                                       "graceful_timeout", 30,
                                                       int)
                elif opt.startswith('stderr_stream') or \
                        opt.startswith('stdout_stream'):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith('rlimit_'):
                    limit = opt[7:]
                    watcher['rlimits'][limit] = int(val)
                elif opt == 'priority':
                    watcher['priority'] = dget(section, "priority", 0, int)
                elif opt == 'use_sockets':
                    watcher['use_sockets'] = dget(section, "use_sockets",
                                                  False, bool)
                else:
                    # freeform
                    watcher[opt] = val

            # set the stream backend
            watcher['stream_backend'] = stream_backend

            watchers.append(watcher)

    config['watchers'] = watchers
    config['plugins'] = plugins
    config['sockets'] = sockets
    return config
Exemplo n.º 29
0
def get_config(config_file):
    if not os.path.exists(config_file):
        sys.stderr.write("the configuration file %r does not exist\n" %
                         config_file)
        sys.stderr.write("Exiting...\n")
        sys.exit(1)

    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config['check'] = dget('circus', 'check_delay', 5, int)
    config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
    config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                     DEFAULT_ENDPOINT_SUB)
    config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
    config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
    config['httpd'] = dget('circus', 'httpd', False, bool)
    config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
    config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
    config['debug'] = dget('circus', 'debug', False, bool)
    stream_backend = dget('circus', 'stream_backend', 'thread')
    if stream_backend == 'gevent':
        try:
            import gevent  # NOQA
            import gevent_zeromq  # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    config['stream_backend'] = stream_backend

    # Initialize watchers, plugins & sockets to manage
    watchers = []
    environs = {}
    plugins = []
    sockets = []

    for section in cfg.sections():
        if section.startswith("socket:"):
            sock = dict(cfg.items(section))
            sock['name'] = section.split("socket:")[-1].lower()
            sockets.append(sock)

        if section.startswith("plugin:"):
            plugins.append(dict(cfg.items(section)))

        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher['name'] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == 'cmd':
                    watcher['cmd'] = val
                elif opt == 'args':
                    watcher['args'] = val
                elif opt == 'numprocesses':
                    watcher['numprocesses'] = dget(section, 'numprocesses', 1,
                                                   int)
                elif opt == 'warmup_delay':
                    watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
                                                   int)
                elif opt == 'executable':
                    watcher['executable'] = dget(section, 'executable', None,
                                                 str)
                elif opt == 'working_dir':
                    watcher['working_dir'] = val
                elif opt == 'shell':
                    watcher['shell'] = dget(section, 'shell', False, bool)
                elif opt == 'uid':
                    watcher['uid'] = val
                elif opt == 'gid':
                    watcher['gid'] = val
                elif opt == 'send_hup':
                    watcher['send_hup'] = dget(section, 'send_hup', False,
                                               bool)
                elif opt == 'check_flapping':
                    watcher['check_flapping'] = dget(section, 'check_flapping',
                                                     True, bool)
                elif opt == 'max_retry':
                    watcher['max_retry'] = dget(section, "max_retry", 5, int)
                elif opt == 'graceful_timout':
                    watcher['graceful_timeout'] = dget(section,
                                                       "graceful_timeout", 30,
                                                       int)
                elif opt.startswith('stderr_stream') or \
                        opt.startswith('stdout_stream'):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith('rlimit_'):
                    limit = opt[7:]
                    watcher['rlimits'][limit] = int(val)
                elif opt == 'priority':
                    watcher['priority'] = dget(section, "priority", 0, int)
                elif opt == 'use_sockets':
                    watcher['use_sockets'] = dget(section, "use_sockets",
                                                  False, bool)
                elif opt == 'singleton':
                    watcher['singleton'] = dget(section, "singleton", False,
                                                bool)
                elif opt == 'stream_backend':
                    watcher['stream_backend'] = val
                elif opt == 'copy_env':
                    watcher['copy_env'] = dget(section, "copy_env", False,
                                               bool)
                elif opt == 'copy_path':
                    watcher['copy_path'] = dget(section, "copy_path", False,
                                                bool)
                elif opt.startswith('hooks.'):
                    hook_name = opt[len('hooks.'):]
                    val = [elmt.strip() for elmt in val.split(',', 1)]
                    if len(val) == 1:
                        val.append(False)
                    else:
                        val[1] = to_boolean(val[1])

                    watcher['hooks'][hook_name] = val

                elif opt == 'respawn':
                    watcher['respawn'] = dget(section, "respawn", True, bool)

                elif opt == 'env':
                    logger.warning('the env option is deprecated the use of '
                                   'env sections is recommended')
                    watcher['env'] = parse_env_str(val)

                else:
                    # freeform
                    watcher[opt] = val

            # set the stream backend
            if 'stream_backend' not in watcher:
                watcher['stream_backend'] = stream_backend
            watchers.append(watcher)

        if section.startswith('env:'):
            for watcher in section.split("env:", 1)[1].split(','):
                watcher = watcher.strip()
                if not watcher in environs:
                    environs[watcher] = dict()
                environs[watcher].update([(k.upper(), v)
                                          for k, v in cfg.items(section)])

    for watcher in watchers:
        if watcher['name'] in environs:
            if not 'env' in watcher:
                watcher['env'] = dict()
            watcher['env'].update(environs[watcher['name']])

    config['watchers'] = watchers
    config['plugins'] = plugins
    config['sockets'] = sockets
    return config
Exemplo n.º 30
0
numpy calculation and gevent, are kept responsive.

Authors
-------
* MinRK
* Pedro Algarvio
"""

import os
import time
import numpy
import gevent
import threading
import gevent_zeromq

gevent_zeromq.monkey_patch(zmq_devices=True)

from gevent_zeromq import zmq
from gevent_zeromq.devices import ThreadDevice


def im_alive(t=None):
    print "I'm alive!"
    if t:
        gevent.spawn_later(t, im_alive, t)


def run_blocking_call(A):
    print "starting blocking loop"
    tic = time.time()
    numpy.dot(A, A.transpose())
Exemplo n.º 31
0
import gevent
import gevent.monkey
gevent.monkey.patch_all()
import gevent_zeromq
gevent_zeromq.monkey_patch()
import zmq

import unittest
import numpy as np
import logging
import time
import test_utils
import os
import shelve
import tables
import pandas

import arrayserver.server.rpc as rpc
import arrayserver.server.rpc.client as client
from arrayserver.server.arrayserverbroker import ArrayServerBroker
from arrayserver.server.arrayservernode import ArrayServerNode
from arrayserver.server.arrayserverconfig import ArrayServerConfig, generate_config_hdf5, generate_config_numpy
import arrayserver.server.redisutils as redisutils
import arrayserver.server.arrayserverconfig as arrayserverconfig
from arrayserver.array_proxy.arrayserver_array_proxy import ArrayServerArrayProxy
import arrayserver.array_proxy.npproxy as npp

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
logging.debug("starting")
Exemplo n.º 32
0
    def setup_ipython_embed(shell_api=None):
        if not _exists_ipython_dir():
            log.warn(
                "IPython profile dir not found. Attempting to avoid race condition"
            )
            import gevent
            import random
            gevent.sleep(
                random.random() *
                3.0)  # Introduce a random delay to make conflict less likely

        from gevent_zeromq import monkey_patch
        monkey_patch()

        # patch in device:
        # gevent-zeromq does not support devices, which block in the C layer.
        # we need to support the "heartbeat" which is a simple bounceback, so we
        # simulate it using the following method.
        import zmq
        orig_device = zmq.device

        def device_patch(dev_type, insock, outsock, *args):
            if dev_type == zmq.FORWARDER:
                while True:
                    m = insock.recv()
                    outsock.send(m)
            else:
                orig_device.device(dev_type, insock, outsock, *args)

        zmq.device = device_patch

        # patch in auto-completion support
        # added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
        # we effectively make the same patches via monkeypatching
        from IPython.core.interactiveshell import InteractiveShell
        from IPython.zmq.ipkernel import IPKernelApp
        old_start = IPKernelApp.start
        old_set_completer_frame = InteractiveShell.set_completer_frame

        def new_start(appself):
            # restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
            bound_scf = old_set_completer_frame.__get__(
                appself.shell, InteractiveShell)
            appself.shell.set_completer_frame = bound_scf
            appself.shell.set_completer_frame()
            old_start(appself)

        IPKernelApp.start = new_start

        from IPython import embed_kernel
        ipy_config = _setup_ipython_config()

        # set specific manhole options
        import tempfile  #, shutil
        from mock import patch
        temp_dir = tempfile.mkdtemp()
        ipy_config.Application.ipython_dir = temp_dir

        with patch(
                "IPython.core.interactiveshell.InteractiveShell.init_virtualenv"
        ):
            for tries in range(3):
                try:
                    embed_kernel(local_ns=shell_api,
                                 config=ipy_config)  # blocks until INT signal
                    break
                except Exception as ex:
                    log.debug(
                        "Failed IPython initialize attempt (try #%s): %s",
                        tries, str(ex))
                    import gevent
                    import random
                    gevent.sleep(random.random() * 0.5)
                except:
                    try:
                        if os.path.exists(
                                ipy_config.KernelApp.connection_file):
                            os.remove(ipy_config.KernelApp.connection_file)
                    except Exception:
                        pass
                    raise
Exemplo n.º 33
0
from collections import defaultdict
try:
    import gevent       # NOQA
    from gevent import monkey
    monkey.noisy = False
    monkey.patch_all()
    from gevent_zeromq import monkey_patch
    monkey_patch()
except ImportError:
    pass


from circus import util
from circus import logger

from zmq.eventloop import ioloop
from iowait import IOWait, SelectIOWait


class BaseStatsCollector(ioloop.PeriodicCallback):

    def __init__(self, streamer, name, callback_time=1., io_loop=None):
        ioloop.PeriodicCallback.__init__(self, self._callback,
                                         callback_time * 1000, io_loop)
        self.streamer = streamer
        self.name = name

    def _callback(self):
        logger.debug('Publishing stats about {0}'.format(self.name))
        for stats in self.collect_stats():
            if stats is None:
Exemplo n.º 34
0
Arquivo: pycc.py Projeto: ooici/pyon
    def setup_ipython_embed(shell_api=None):
        if not _exists_ipython_dir():
            log.warn("IPython profile dir not found. Attempting to avoid race condition")
            import gevent
            import random

            gevent.sleep(random.random() * 3.0)  # Introduce a random delay to make conflict less likely

        from gevent_zeromq import monkey_patch

        monkey_patch()

        # patch in device:
        # gevent-zeromq does not support devices, which block in the C layer.
        # we need to support the "heartbeat" which is a simple bounceback, so we
        # simulate it using the following method.
        import zmq

        orig_device = zmq.device

        def device_patch(dev_type, insock, outsock, *args):
            if dev_type == zmq.FORWARDER:
                while True:
                    m = insock.recv()
                    outsock.send(m)
            else:
                orig_device.device(dev_type, insock, outsock, *args)

        zmq.device = device_patch

        # patch in auto-completion support
        # added in https://github.com/ipython/ipython/commit/f4be28f06c2b23cd8e4a3653b9e84bde593e4c86
        # we effectively make the same patches via monkeypatching
        from IPython.core.interactiveshell import InteractiveShell
        from IPython.zmq.ipkernel import IPKernelApp

        old_start = IPKernelApp.start
        old_set_completer_frame = InteractiveShell.set_completer_frame

        def new_start(appself):
            # restore old set_completer_frame that gets no-op'd out in ZmqInteractiveShell.__init__
            bound_scf = old_set_completer_frame.__get__(appself.shell, InteractiveShell)
            appself.shell.set_completer_frame = bound_scf
            appself.shell.set_completer_frame()
            old_start(appself)

        IPKernelApp.start = new_start

        from IPython import embed_kernel

        ipy_config = _setup_ipython_config()

        # set specific manhole options
        import tempfile  # , shutil
        from mock import patch

        temp_dir = tempfile.mkdtemp()
        ipy_config.Application.ipython_dir = temp_dir

        with patch("IPython.core.interactiveshell.InteractiveShell.init_virtualenv"):
            for tries in range(3):
                try:
                    embed_kernel(local_ns=shell_api, config=ipy_config)  # blocks until INT
                    break
                except Exception as ex:
                    log.debug("Failed IPython initialize attempt (try #%s): %s", tries, str(ex))
                    import gevent
                    import random

                    gevent.sleep(random.random() * 0.5)
Exemplo n.º 35
0
def get_config(config_file):
    cfg, cfg_files_read = read_config(config_file)
    dget = cfg.dget
    config = {}

    # main circus options
    config['check'] = dget('circus', 'check_delay', 5, int)
    config['endpoint'] = dget('circus', 'endpoint', 'tcp://127.0.0.1:5555')
    config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                     'tcp://127.0.0.1:5556')
    config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)

    stream_backend = dget('circus', 'stream_backend', 'thread')
    if stream_backend == 'gevent':
        try:
            import gevent  # NOQA
            import gevent_zeromq  # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    config['stream_backend'] = stream_backend

    # Initialize watchers to manage
    watchers = []
    for section in cfg.sections():
        if section.startswith("watcher:"):
            watcher = watcher_defaults()
            watcher['name'] = section.split("watcher:", 1)[1]

            # create watcher options
            for opt, val in cfg.items(section):
                if opt == 'cmd':
                    watcher['cmd'] = val
                elif opt == 'args':
                    watcher['args'] = val
                elif opt == 'numprocesses':
                    watcher['numprocesses'] = dget(section, 'numprocesses', 1,
                                                   int)
                elif opt == 'warmup_delay':
                    watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
                                                   int)
                elif opt == 'executable':
                    watcher['executable'] = dget(section, 'executable', None,
                                                 str)
                elif opt == 'working_dir':
                    watcher['working_dir'] = val
                elif opt == 'shell':
                    watcher['shell'] = dget(section, 'shell', False, bool)
                elif opt == 'uid':
                    watcher['uid '] = val
                elif opt == 'gid':
                    watcher['gid'] = val
                elif opt == 'send_hup':
                    watcher['send_hup'] = dget(section, 'send_hup', False,
                                               bool)
                elif opt == 'flapping_attempts':
                    watcher['flapping_attempts'] = dget(
                        section, "flapping_attempts", 2, int)
                elif opt == 'flapping_window':
                    watcher['flapping_window'] = dget(section,
                                                      "flapping_window", 1,
                                                      int)
                elif opt == 'retry_in':
                    watcher['retry_in'] = dget(section, "retry_in", 7, int)
                elif opt == 'max_retry':
                    watcher['max_retry'] = dget(section, "max_retry", 5, int)
                elif opt == 'graceful_timout':
                    watcher['graceful_timeout'] = dget(section,
                                                       "graceful_timeout", 30,
                                                       int)
                elif opt.startswith('stderr_stream') or \
                        opt.startswith('stdout_stream'):
                    stream_name, stream_opt = opt.split(".", 1)
                    watcher[stream_name][stream_opt] = val
                elif opt.startswith('rlimit_'):
                    limit = opt[7:]
                    watcher['rlimits'][limit] = int(val)
                elif opt == 'priority':
                    watcher['priority'] = dget(section, "priority", 0, int)

            # second pass, parse stream conf
            stdout_conf = watcher.get('stdout_stream', {})
            watcher['stdout_stream'] = stream_config(watcher['name'],
                                                     stdout_conf)

            stderr_conf = watcher.get('stderr_stream', {})
            watcher['stderr_stream'] = stream_config(watcher['name'],
                                                     stderr_conf)

            # set the stream backend
            watcher['stream_backend'] = stream_backend

            watchers.append(watcher)

    config['watchers'] = watchers
    return config
Exemplo n.º 36
0
def get_arbiter(watchers, controller=None,
                pubsub_endpoint=None,
                stats_endpoint=None,
                env=None, name=None, context=None,
                background=False, stream_backend="thread",
                plugins=None, debug=False):
    """Creates a Arbiter and a single watcher in it.

    Options:

    - **watchers** -- a list of watchers. A watcher in that case is a
      dict containing:

        - **name** -- the name of the watcher (default: None)
        - **cmd** -- the command line used to run the Watcher.
        - **args** -- the args for the command (list or string).
        - **executable** -- When executable is given, the first item in
          the args sequence obtained from **cmd** is still treated by most
          programs as the command name, which can then be different from the
          actual executable name. It becomes the display name for the executing
          program in utilities such as **ps**.

        - **numprocesses** -- the number of processes to spawn (default: 1).
        - **warmup_delay** -- the delay in seconds between two spawns
          (default: 0)
        - **shell** -- if True, the processes are run in the shell
          (default: False)
        - **working_dir** - the working dir for the processes (default: None)
        - **uid** -- the user id used to run the processes (default: None)
        - **gid** -- the group id used to run the processes (default: None)
        - **env** -- the environment passed to the processes (default: None)
        - **send_hup**: if True, a process reload will be done by sending
          the SIGHUP signal. (default: False)
        - **stdout_stream**: a mapping containing the options for configuring
          the stdout stream. Default to None. When provided, may contain:

            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **stderr_stream**: a mapping containing the options for configuring
          the stderr stream. Default to None. When provided, may contain:

            - **class**: the fully qualified name of the class to use for
              streaming. Defaults to circus.stream.FileStream
            - **refresh_time**: the delay between two stream checks. Defaults
              to 0.3 seconds.
            - any other key will be passed the class constructor.
        - **max_retry**: the number of times we attempt to start a process,
          before we abandon and stop the whole watcher. (default: 5)

    - **controller** -- the zmq entry point (default: 'tcp://127.0.0.1:5555')
    - **pubsub_endpoint** -- the zmq entry point for the pubsub
      (default: 'tcp://127.0.0.1:5556')
    - **stats_endpoint** -- the stats endpoint. If not provided,
      the *circusd-stats* process will not be launched. (default: None)
    - **context** -- the zmq context (default: None)
    - **background** -- If True, the arbiter is launched in a thread in the
      background (default: False)
    - **stream_backend** -- the backend that will be used for the streaming
      process. Can be *thread* or *gevent*. When set to *gevent* you need
      to have *gevent* and *gevent_zmq* installed. (default: thread)
    - **plugins** -- a list of plugins. Each item is a mapping with:

        - **use** -- Fully qualified name that points to the plugin class
        - every other value is passed to the plugin in the **config** option

    - **debug** -- If True the arbiter is launched in debug mode
      (default: False)
    """
    from circus.util import DEFAULT_ENDPOINT_DEALER, DEFAULT_ENDPOINT_SUB
    if controller is None:
        controller = DEFAULT_ENDPOINT_DEALER
    if pubsub_endpoint is None:
        pubsub_endpoint = DEFAULT_ENDPOINT_SUB

    if stream_backend == 'gevent':
        try:
            import gevent           # NOQA
            import gevent_zeromq    # NOQA
        except ImportError:
            sys.stderr.write("stream_backend set to gevent, " +
                             "but gevent or gevent_zeromq isn't installed\n")
            sys.stderr.write("Exiting...")
            sys.exit(1)

        from gevent import monkey
        from gevent_zeromq import monkey_patch
        monkey.patch_all()
        monkey_patch()

    from circus.watcher import Watcher
    if background:
        from circus.arbiter import ThreadedArbiter as Arbiter   # NOQA
    else:
        from circus.arbiter import Arbiter   # NOQA

    _watchers = []

    for watcher in watchers:
        cmd = watcher['cmd']
        watcher['name'] = watcher.get('name', os.path.basename(cmd.split()[0]))
        watcher['stream_backend'] = stream_backend
        _watchers.append(Watcher.load_from_config(watcher))

    return Arbiter(_watchers, controller, pubsub_endpoint,
                   stats_endpoint=stats_endpoint,
                   context=context, plugins=plugins, debug=debug)