예제 #1
0
 def testSystem(self):
     from pulsar import system
     cfg = Config()
     self.assertEqual(cfg.uid, system.get_uid())
     self.assertEqual(cfg.gid, system.get_gid())
     self.assertEqual(cfg.proc_name, 'pulsar')
     cfg.set('process_name', 'bla')
     self.assertEqual(cfg.proc_name, 'bla')
예제 #2
0
 def testFunction(self):
     cfg = Config()
     worker = get_actor()
     self.assertTrue(cfg.post_fork)
     self.assertEqual(cfg.post_fork(worker), None)
     cfg.set('post_fork', post_fork)
     self.assertEqual(cfg.post_fork(worker), worker)
     cfg1 = pickle.loads(pickle.dumps(cfg))
     self.assertEqual(cfg1.post_fork(worker), worker)
예제 #3
0
class FlaskGreen(MultiApp):
    """Multiapp Server configurator
    """
    cfg = Config(bind=':8080', echo_bind=':8060')

    def build(self):
        yield self.new_app(WSGIServer, callable=FlaskSite())
        yield self.new_app(SocketServer, 'echo', callable=EchoServerProtocol,
                           echo_connection_made=log_connection)
예제 #4
0
 def __testFunctionFromConfigFile(self):
     # TODO, fails in pypy for some odd reasons
     worker = get_actor()
     cfg = Config()
     self.assertEqual(cfg.connection_made(worker), None)
     self.assertTrue(cfg.import_from_module(__file__))
     self.assertEqual(cfg.connection_made(worker), worker)
     cfg1 = pickle.loads(pickle.dumps(cfg))
     self.assertEqual(cfg1.connection_made(worker), worker)
예제 #5
0
class MicroAgentApp(Application):
    cfg = Config(apps=['microagent'])

    def worker_start(self, worker, exc=None):
        log = self.cfg.configured_logger()

        signal_bus_dsn = self.cfg.settings['signal_bus'].value
        signal_prefix = self.cfg.settings['signal_prefix'].value

        if signal_bus_dsn.startswith('redis'):
            bus = RedisSignalBus(signal_bus_dsn,
                                 prefix=signal_prefix,
                                 logger=log)

        elif signal_bus_dsn.startswith('aioredis'):
            from .aioredis import AIORedisSignalBus
            bus = AIORedisSignalBus(signal_bus_dsn[3:],
                                    prefix=signal_prefix,
                                    logger=log)

        else:
            bus = None

        queue_broker_dsn = self.cfg.settings.get('queue_broker').value

        if queue_broker_dsn.startswith('amqp'):
            from .amqp import AMQPBroker

            ssl_context, cert_file = None, self.cfg.settings.get(
                'broker_cert').value

            if cert_file:
                ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
                ssl_context.check_hostname = False
                ssl_context.load_verify_locations(cert_file)

            broker = AMQPBroker(queue_broker_dsn, ssl_context=ssl_context)

        elif queue_broker_dsn.startswith('kafka'):
            from .kafka import KafkaBroker
            broker = KafkaBroker(queue_broker_dsn)

        elif queue_broker_dsn.startswith('redis'):
            broker = PulsarRedisBroker(queue_broker_dsn)

        else:
            broker = None

        worker.agent = self.cfg.agent(bus=bus,
                                      broker=broker,
                                      logger=log,
                                      settings=self.cfg.settings)
        asyncio.ensure_future(worker.agent.start())

    def worker_stopping(self, worker, exc=None):
        asyncio.ensure_future(worker.agent.stop())
예제 #6
0
 def test_methods(self):
     cfg = Config()
     self.assertEqual(cfg.get('sdjcbsjkbcd', 'ciao'), 'ciao')
     d = dict(cfg.items())
     self.assertEqual(len(d), len(cfg))
     sett = cfg.get('debug')
     self.assertTrue(str(sett))
     self.assertEqual(cfg.settings['debug'].default, False)
     cfg.set('debug', True, default=True)
     self.assertEqual(cfg.debug, True)
     self.assertEqual(cfg.settings['debug'].default, True)
예제 #7
0
class QueueApp(Application):
    """A pulsar :class:`.Application` for consuming :class:`.Task`.

    This application can also schedule periodic tasks when the
    :ref:`schedule_periodic <setting-schedule_periodic>` flag is ``True``.
    """
    backend_factory = Consumer
    name = 'tasks'
    cfg = Config(apps=('tasks', ),
                 version=__version__,
                 data_store=DEFAULT_MQ_BACKEND)
    _backend = None

    def api(self):
        return Producer(self.cfg, logger=self.logger)

    @property
    def backend(self):
        return self._backend

    async def monitor_start(self, monitor, exc=None):
        if not exc and self.cfg.workers:
            self._backend = await self._start(monitor, False)

    def monitor_task(self, monitor):
        if monitor.is_running():
            self._backend.tick(monitor)

    def monitor_stopping(self, worker, exc=None):
        if self._backend:
            backend = self._backend
            self._backend = None
            return backend.close()

    async def worker_start(self, worker, exc=None):
        if not exc:
            self._backend = await self._start(worker)
            if not worker.is_monitor():
                self._backend.bind_event('close', _close)

    def worker_stopping(self, worker, exc=None):
        if self._backend:
            return self._backend.close()

    def actorparams(self, monitor, params):
        # makes sure workers are only consuming tasks, not scheduling.
        cfg = params['cfg']
        cfg.set('schedule_periodic', False)

    def _start(self, actor, consume=True):
        return self.backend_factory(self.cfg,
                                    logger=self.logger).start(actor, consume)
예제 #8
0
class LuxApp(Application):
    name = 'lux'
    cfg = Config(include=('loglevel', 'loghandlers', 'debug', 'config'))

    def __call__(self, actor=None):
        try:
            return super(LuxApp, self).__call__(actor)
        except ImproperlyConfigured:
            actor = actor or get_actor()
            return self.on_config(actor)

    def on_config(self, actor):
        asyncio.set_event_loop(actor._loop)
        return False
예제 #9
0
def _spawn_actor(kind, monitor, cfg=None, name=None, aid=None, **kw):
    # Internal function which spawns a new Actor and return its
    # ActorProxyMonitor.
    # *cls* is the Actor class
    # *monitor* can be either the arbiter or a monitor
    if monitor:
        params = monitor.actorparams()
        name = params.pop('name', name)
        aid = params.pop('aid', aid)
        cfg = params.pop('cfg', cfg)

    # get config if not available
    if cfg is None:
        if monitor:
            cfg = monitor.cfg.copy()
        else:
            cfg = Config()

    if not aid:
        aid = create_aid()

    if not monitor:  # monitor not available, this is the arbiter
        assert kind == 'arbiter'
        name = kind
        params = {}
        if not cfg.exc_id:
            cfg.set('exc_id', aid)
    #
    for key, value in kw.items():
        if key in cfg.settings:
            cfg.set(key, value)
        else:
            params[key] = value
    #
    if monitor:
        kind = kind or cfg.concurrency
    if not kind:
        raise TypeError('Cannot spawn')

    maker = concurrency_models.get(kind)
    if maker:
        c = maker()
        return c.make(kind, cfg, name, aid, monitor=monitor, **params)
    else:
        raise ValueError('Concurrency %s not supported in pulsar' % kind)
예제 #10
0
    def create_config(cls, params, prefix=None, name=None):
        '''Create a new :class:`.Config` container.

        Invoked during initialisation, it overrides defaults
        with ``params`` and apply the ``prefix`` to non global
        settings.
        '''
        if isinstance(cls.cfg, Config):
            cfg = cls.cfg.copy(name=name, prefix=prefix)
        else:
            cfg = cls.cfg.copy()
            if name:
                cfg[name] = name
            if prefix:
                cfg[prefix] = prefix
            cfg = Config(**cfg)
        cfg.update_settings()
        cfg.update(params, True)
        return cfg
예제 #11
0
class PulsarQueue(MultiApp):
    """Build a multi-app consisting on a taskqueue and a JSON-RPC server.
    """
    cfg = Config('Pulsar Queue')

    def __init__(self, callable=None, **params):
        super().__init__(**params)
        self.manager = callable

    def api(self):
        return self.apps()[0].api()

    def build(self):
        yield self.new_app(QueueApp, callable=self.manager)
        wsgi = self.cfg.params.get('wsgi')
        if wsgi:
            if wsgi is True:
                wsgi = Rpc
            yield self.new_app(RpcServer, prefix='rpc', callable=self)
예제 #12
0
    def testDefaults(self):
        from pulsar.utils import config
        self.assertFalse(config.pass_through(None))
        cfg = Config()
        self.assertEqual(list(sorted(cfg)), list(sorted(cfg.settings)))

        def _():
            cfg.debug = 3
        self.assertRaises(AttributeError, _)
        #
        name = tempfile.mktemp()
        with open(name, 'w') as f:
            f.write('a')
        self.assertRaises(RuntimeError, cfg.import_from_module, name)
        os.remove(name)
        #
        name = '%s.py' % name
        with open(name, 'w') as f:
            f.write('a')
        self.assertRaises(RuntimeError, cfg.import_from_module, name)
        os.remove(name)
예제 #13
0
def config(**kw):
    return Config(**kw)
예제 #14
0
class Configurator(object):
    '''A mixin for configuring and loading a pulsar application server.

    :parameter name: to override the class :attr:`name` attribute.
    :parameter description: to override the class :attr:`cfg.description`
        attribute.
    :parameter epilog: to override the class :attr:`cfg.epilog` attribute.
    :parameter version: Optional version of this application, it overrides
        the class :attr:`cfg.version` attribute.
    :parameter argv: Optional list of command line parameters to parse, if
        not supplied the :attr:`sys.argv` list will be used. The parameter
        is only relevant if ``parse_console`` is ``True``.
    :parameter parse_console: ``True`` (default) if the console parameters
        needs parsing.
    :parameter script: Optional string which set the :attr:`script`
        attribute.
    :parameter params: a dictionary of configuration parameters which
        overrides the defaults and the :attr:`cfg` class attribute.
        They will be overwritten by a :ref:`config file <setting-config>`
        or command line arguments.

    .. attribute:: name

        The name is unique if this is an :class:`Application`. In this
        case it defines the application monitor name as well and can be
        access in the arbiter domain via the :func:`get_application`
        function.

    .. attribute:: argv

        Optional list of command line parameters. If not available the
        :attr:`sys.argv` list will be used when parsing the console.

    .. attribute:: cfg

        The :class:`.Config` for this :class:`Configurator`.
        If set as class attribute it will be replaced during initialisation.

        Default: ``None``.

    .. attribute:: console_parsed

        ``True`` if this application parsed the console before starting.

    .. attribute:: script

        Full path of the script which starts the application or ``None``.
        Evaluated during initialization via the :meth:`python_path` method.
    '''
    argv = None
    name = None
    cfg = Config()

    def __init__(self,
                 name=None,
                 description=None,
                 epilog=None,
                 version=None,
                 argv=None,
                 parse_console=True,
                 script=None,
                 cfg=None,
                 load_config=True,
                 **params):
        cls = self.__class__
        self.name = name or cls.name or cls.__name__.lower()
        if not isinstance(cfg, Config):
            cfg = cfg or {}
            cfg.update(params)
            cfg = cls.create_config(cfg)
        else:
            cfg.update(params)
        self.cfg = cfg
        cfg.description = description or self.cfg.description
        cfg.epilog = epilog or self.cfg.epilog
        cfg.version = version or self.cfg.version
        cfg.name = self.name
        cfg.application = cls
        self.argv = argv
        self.console_parsed = parse_console
        self.cfg.script = self.script = self.python_path(script)

    @property
    def version(self):
        '''Version of this :class:`Application`'''
        return self.cfg.version

    @property
    def root_dir(self):
        '''Root directory of this :class:`Configurator`.

        Evaluated from the :attr:`script` attribute.
        '''
        if self.cfg.script:
            return os.path.dirname(self.cfg.script)

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

    def python_path(self, script):
        '''Called during initialisation to obtain the ``script`` name and
        to add the :attr:`script` directory to the python path if not in the
        path already.
        If ``script`` does not evalueate to ``True`` it is evaluated from
        the ``__main__`` import. Returns the real path of the python
        script which runs the application.
        '''
        if not script:
            try:
                import __main__
                script = getfile(__main__)
            except Exception:  # pragma    nocover
                return
        script = os.path.realpath(script)
        path = os.path.dirname(script)
        if path not in sys.path:
            sys.path.insert(0, path)
        return script

    def on_config(self, arbiter):
        '''Callback when configuration is loaded.

        This is a chance to do applications specific checks before the
        concurrent machinery is put into place.

        If it returns ``False`` the application will abort.
        '''
        pass

    def load_config(self):
        '''Load the application configuration from a file and/or
        from the command line.

        Called during application initialisation. The parameters
        overriding order is the following:

        * default parameters.
        * the key-valued params passed in the initialisation.
        * the parameters in the optional configuration file
        * the parameters passed in the command line.
        '''
        # get the actor if available and override default cfg values with those
        # from the actor
        actor = get_actor()
        if actor and actor.is_running():
            # actor available and running.
            # Unless argv is set, skip parsing
            if self.argv is None:
                self.console_parsed = False
            # copy global settings
            self.cfg.copy_globals(actor.cfg)
        #
        for name in list(self.cfg.params):
            if name in self.cfg.settings:
                value = self.cfg.params.pop(name)
                if value is not None:
                    self.cfg.set(name, value)
        # parse console args
        if self.console_parsed:
            self.cfg.parse_command_line(self.argv)
        else:
            self.cfg.params.update(self.cfg.import_from_module())

    def start(self):
        '''Invoked the application callable method and start
        the ``arbiter`` if it wasn't already started.

        It returns a :class:`~asyncio.Future` called back once the
        application/applications are running. It returns ``None`` if
        called more than once.
        '''
        on_start = self()
        arbiter = pulsar.arbiter()
        if arbiter and on_start:
            arbiter.start()
        return on_start

    @classmethod
    def create_config(cls, params, prefix=None, name=None):
        '''Create a new :class:`.Config` container.

        Invoked during initialisation, it overrides defaults
        with ``params`` and apply the ``prefix`` to non global
        settings.
        '''
        if isinstance(cls.cfg, Config):
            cfg = cls.cfg.copy(name=name, prefix=prefix)
        else:
            cfg = cls.cfg.copy()
            if name:
                cfg[name] = name
            if prefix:
                cfg[prefix] = prefix
            cfg = Config(**cfg)
        cfg.update_settings()
        cfg.update(params, True)
        return cfg
예제 #15
0
 def test_exclude(self):
     cfg = Config(exclude=['config'])
     self.assertEqual(cfg.config, 'config.py')
     self.assertEqual(cfg.params['config'], 'config.py')
     self.assertFalse('config' in cfg.settings)
예제 #16
0
 def testBadConfig(self):
     cfg = Config()
     self.assertEqual(cfg.config, 'config.py')
     self.assertEqual(cfg.import_from_module('foo/bla/cnkjnckjcn.py'), [])
     cfg.set('config', None)
     self.assertEqual(cfg.config, None)
예제 #17
0
        request.actor.logger.info('Setting value')
        self.value = value

    def remote_get_value(self, request):
        request.actor.logger.info('Getting value')
        return self.value


def start(arbiter, **kw):
    ensure_future(app(arbiter))


async def app(arbiter):
    # Spawn a new actor
    calc = await Calculator.spawn(name='calc1')
    print(calc.name)
    # set value in the remote calculator
    await calc.set_value(46)
    # get value from the remote calculator
    value = await calc.get_value()
    print(value)

    # Stop the application
    arbiter.stop()


if __name__ == '__main__':
    cfg = Config()
    cfg.parse_command_line()
    arbiter(cfg=cfg, start=start).start()
예제 #18
0
class MultiWsgi(MultiApp):
    cfg = Config(bind=':0', rpc_bind=':0', bla='foo')

    def build(self):
        yield self.new_app(WSGIServer, callable=dummy)
        yield self.new_app(WSGIServer, 'rpc', callable=dummy)