def testSystem(self): from pulsar import system cfg = Config() self.assertEqual(cfg.uid, system.get_uid()) self.assertEqual(cfg.gid, system.get_gid()) self.assertEqual(cfg.proc_name, 'pulsar') cfg.set('process_name', 'bla') self.assertEqual(cfg.proc_name, 'bla')
def testFunction(self): cfg = Config() worker = get_actor() self.assertTrue(cfg.post_fork) self.assertEqual(cfg.post_fork(worker), None) cfg.set('post_fork', post_fork) self.assertEqual(cfg.post_fork(worker), worker) cfg1 = pickle.loads(pickle.dumps(cfg)) self.assertEqual(cfg1.post_fork(worker), worker)
class FlaskGreen(MultiApp): """Multiapp Server configurator """ cfg = Config(bind=':8080', echo_bind=':8060') def build(self): yield self.new_app(WSGIServer, callable=FlaskSite()) yield self.new_app(SocketServer, 'echo', callable=EchoServerProtocol, echo_connection_made=log_connection)
def __testFunctionFromConfigFile(self): # TODO, fails in pypy for some odd reasons worker = get_actor() cfg = Config() self.assertEqual(cfg.connection_made(worker), None) self.assertTrue(cfg.import_from_module(__file__)) self.assertEqual(cfg.connection_made(worker), worker) cfg1 = pickle.loads(pickle.dumps(cfg)) self.assertEqual(cfg1.connection_made(worker), worker)
class MicroAgentApp(Application): cfg = Config(apps=['microagent']) def worker_start(self, worker, exc=None): log = self.cfg.configured_logger() signal_bus_dsn = self.cfg.settings['signal_bus'].value signal_prefix = self.cfg.settings['signal_prefix'].value if signal_bus_dsn.startswith('redis'): bus = RedisSignalBus(signal_bus_dsn, prefix=signal_prefix, logger=log) elif signal_bus_dsn.startswith('aioredis'): from .aioredis import AIORedisSignalBus bus = AIORedisSignalBus(signal_bus_dsn[3:], prefix=signal_prefix, logger=log) else: bus = None queue_broker_dsn = self.cfg.settings.get('queue_broker').value if queue_broker_dsn.startswith('amqp'): from .amqp import AMQPBroker ssl_context, cert_file = None, self.cfg.settings.get( 'broker_cert').value if cert_file: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.check_hostname = False ssl_context.load_verify_locations(cert_file) broker = AMQPBroker(queue_broker_dsn, ssl_context=ssl_context) elif queue_broker_dsn.startswith('kafka'): from .kafka import KafkaBroker broker = KafkaBroker(queue_broker_dsn) elif queue_broker_dsn.startswith('redis'): broker = PulsarRedisBroker(queue_broker_dsn) else: broker = None worker.agent = self.cfg.agent(bus=bus, broker=broker, logger=log, settings=self.cfg.settings) asyncio.ensure_future(worker.agent.start()) def worker_stopping(self, worker, exc=None): asyncio.ensure_future(worker.agent.stop())
def test_methods(self): cfg = Config() self.assertEqual(cfg.get('sdjcbsjkbcd', 'ciao'), 'ciao') d = dict(cfg.items()) self.assertEqual(len(d), len(cfg)) sett = cfg.get('debug') self.assertTrue(str(sett)) self.assertEqual(cfg.settings['debug'].default, False) cfg.set('debug', True, default=True) self.assertEqual(cfg.debug, True) self.assertEqual(cfg.settings['debug'].default, True)
class QueueApp(Application): """A pulsar :class:`.Application` for consuming :class:`.Task`. This application can also schedule periodic tasks when the :ref:`schedule_periodic <setting-schedule_periodic>` flag is ``True``. """ backend_factory = Consumer name = 'tasks' cfg = Config(apps=('tasks', ), version=__version__, data_store=DEFAULT_MQ_BACKEND) _backend = None def api(self): return Producer(self.cfg, logger=self.logger) @property def backend(self): return self._backend async def monitor_start(self, monitor, exc=None): if not exc and self.cfg.workers: self._backend = await self._start(monitor, False) def monitor_task(self, monitor): if monitor.is_running(): self._backend.tick(monitor) def monitor_stopping(self, worker, exc=None): if self._backend: backend = self._backend self._backend = None return backend.close() async def worker_start(self, worker, exc=None): if not exc: self._backend = await self._start(worker) if not worker.is_monitor(): self._backend.bind_event('close', _close) def worker_stopping(self, worker, exc=None): if self._backend: return self._backend.close() def actorparams(self, monitor, params): # makes sure workers are only consuming tasks, not scheduling. cfg = params['cfg'] cfg.set('schedule_periodic', False) def _start(self, actor, consume=True): return self.backend_factory(self.cfg, logger=self.logger).start(actor, consume)
class LuxApp(Application): name = 'lux' cfg = Config(include=('loglevel', 'loghandlers', 'debug', 'config')) def __call__(self, actor=None): try: return super(LuxApp, self).__call__(actor) except ImproperlyConfigured: actor = actor or get_actor() return self.on_config(actor) def on_config(self, actor): asyncio.set_event_loop(actor._loop) return False
def _spawn_actor(kind, monitor, cfg=None, name=None, aid=None, **kw): # Internal function which spawns a new Actor and return its # ActorProxyMonitor. # *cls* is the Actor class # *monitor* can be either the arbiter or a monitor if monitor: params = monitor.actorparams() name = params.pop('name', name) aid = params.pop('aid', aid) cfg = params.pop('cfg', cfg) # get config if not available if cfg is None: if monitor: cfg = monitor.cfg.copy() else: cfg = Config() if not aid: aid = create_aid() if not monitor: # monitor not available, this is the arbiter assert kind == 'arbiter' name = kind params = {} if not cfg.exc_id: cfg.set('exc_id', aid) # for key, value in kw.items(): if key in cfg.settings: cfg.set(key, value) else: params[key] = value # if monitor: kind = kind or cfg.concurrency if not kind: raise TypeError('Cannot spawn') maker = concurrency_models.get(kind) if maker: c = maker() return c.make(kind, cfg, name, aid, monitor=monitor, **params) else: raise ValueError('Concurrency %s not supported in pulsar' % kind)
def create_config(cls, params, prefix=None, name=None): '''Create a new :class:`.Config` container. Invoked during initialisation, it overrides defaults with ``params`` and apply the ``prefix`` to non global settings. ''' if isinstance(cls.cfg, Config): cfg = cls.cfg.copy(name=name, prefix=prefix) else: cfg = cls.cfg.copy() if name: cfg[name] = name if prefix: cfg[prefix] = prefix cfg = Config(**cfg) cfg.update_settings() cfg.update(params, True) return cfg
class PulsarQueue(MultiApp): """Build a multi-app consisting on a taskqueue and a JSON-RPC server. """ cfg = Config('Pulsar Queue') def __init__(self, callable=None, **params): super().__init__(**params) self.manager = callable def api(self): return self.apps()[0].api() def build(self): yield self.new_app(QueueApp, callable=self.manager) wsgi = self.cfg.params.get('wsgi') if wsgi: if wsgi is True: wsgi = Rpc yield self.new_app(RpcServer, prefix='rpc', callable=self)
def testDefaults(self): from pulsar.utils import config self.assertFalse(config.pass_through(None)) cfg = Config() self.assertEqual(list(sorted(cfg)), list(sorted(cfg.settings))) def _(): cfg.debug = 3 self.assertRaises(AttributeError, _) # name = tempfile.mktemp() with open(name, 'w') as f: f.write('a') self.assertRaises(RuntimeError, cfg.import_from_module, name) os.remove(name) # name = '%s.py' % name with open(name, 'w') as f: f.write('a') self.assertRaises(RuntimeError, cfg.import_from_module, name) os.remove(name)
def config(**kw): return Config(**kw)
class Configurator(object): '''A mixin for configuring and loading a pulsar application server. :parameter name: to override the class :attr:`name` attribute. :parameter description: to override the class :attr:`cfg.description` attribute. :parameter epilog: to override the class :attr:`cfg.epilog` attribute. :parameter version: Optional version of this application, it overrides the class :attr:`cfg.version` attribute. :parameter argv: Optional list of command line parameters to parse, if not supplied the :attr:`sys.argv` list will be used. The parameter is only relevant if ``parse_console`` is ``True``. :parameter parse_console: ``True`` (default) if the console parameters needs parsing. :parameter script: Optional string which set the :attr:`script` attribute. :parameter params: a dictionary of configuration parameters which overrides the defaults and the :attr:`cfg` class attribute. They will be overwritten by a :ref:`config file <setting-config>` or command line arguments. .. attribute:: name The name is unique if this is an :class:`Application`. In this case it defines the application monitor name as well and can be access in the arbiter domain via the :func:`get_application` function. .. attribute:: argv Optional list of command line parameters. If not available the :attr:`sys.argv` list will be used when parsing the console. .. attribute:: cfg The :class:`.Config` for this :class:`Configurator`. If set as class attribute it will be replaced during initialisation. Default: ``None``. .. attribute:: console_parsed ``True`` if this application parsed the console before starting. .. attribute:: script Full path of the script which starts the application or ``None``. Evaluated during initialization via the :meth:`python_path` method. ''' argv = None name = None cfg = Config() def __init__(self, name=None, description=None, epilog=None, version=None, argv=None, parse_console=True, script=None, cfg=None, load_config=True, **params): cls = self.__class__ self.name = name or cls.name or cls.__name__.lower() if not isinstance(cfg, Config): cfg = cfg or {} cfg.update(params) cfg = cls.create_config(cfg) else: cfg.update(params) self.cfg = cfg cfg.description = description or self.cfg.description cfg.epilog = epilog or self.cfg.epilog cfg.version = version or self.cfg.version cfg.name = self.name cfg.application = cls self.argv = argv self.console_parsed = parse_console self.cfg.script = self.script = self.python_path(script) @property def version(self): '''Version of this :class:`Application`''' return self.cfg.version @property def root_dir(self): '''Root directory of this :class:`Configurator`. Evaluated from the :attr:`script` attribute. ''' if self.cfg.script: return os.path.dirname(self.cfg.script) def __repr__(self): return self.name def __str__(self): return self.__repr__() def python_path(self, script): '''Called during initialisation to obtain the ``script`` name and to add the :attr:`script` directory to the python path if not in the path already. If ``script`` does not evalueate to ``True`` it is evaluated from the ``__main__`` import. Returns the real path of the python script which runs the application. ''' if not script: try: import __main__ script = getfile(__main__) except Exception: # pragma nocover return script = os.path.realpath(script) path = os.path.dirname(script) if path not in sys.path: sys.path.insert(0, path) return script def on_config(self, arbiter): '''Callback when configuration is loaded. This is a chance to do applications specific checks before the concurrent machinery is put into place. If it returns ``False`` the application will abort. ''' pass def load_config(self): '''Load the application configuration from a file and/or from the command line. Called during application initialisation. The parameters overriding order is the following: * default parameters. * the key-valued params passed in the initialisation. * the parameters in the optional configuration file * the parameters passed in the command line. ''' # get the actor if available and override default cfg values with those # from the actor actor = get_actor() if actor and actor.is_running(): # actor available and running. # Unless argv is set, skip parsing if self.argv is None: self.console_parsed = False # copy global settings self.cfg.copy_globals(actor.cfg) # for name in list(self.cfg.params): if name in self.cfg.settings: value = self.cfg.params.pop(name) if value is not None: self.cfg.set(name, value) # parse console args if self.console_parsed: self.cfg.parse_command_line(self.argv) else: self.cfg.params.update(self.cfg.import_from_module()) def start(self): '''Invoked the application callable method and start the ``arbiter`` if it wasn't already started. It returns a :class:`~asyncio.Future` called back once the application/applications are running. It returns ``None`` if called more than once. ''' on_start = self() arbiter = pulsar.arbiter() if arbiter and on_start: arbiter.start() return on_start @classmethod def create_config(cls, params, prefix=None, name=None): '''Create a new :class:`.Config` container. Invoked during initialisation, it overrides defaults with ``params`` and apply the ``prefix`` to non global settings. ''' if isinstance(cls.cfg, Config): cfg = cls.cfg.copy(name=name, prefix=prefix) else: cfg = cls.cfg.copy() if name: cfg[name] = name if prefix: cfg[prefix] = prefix cfg = Config(**cfg) cfg.update_settings() cfg.update(params, True) return cfg
def test_exclude(self): cfg = Config(exclude=['config']) self.assertEqual(cfg.config, 'config.py') self.assertEqual(cfg.params['config'], 'config.py') self.assertFalse('config' in cfg.settings)
def testBadConfig(self): cfg = Config() self.assertEqual(cfg.config, 'config.py') self.assertEqual(cfg.import_from_module('foo/bla/cnkjnckjcn.py'), []) cfg.set('config', None) self.assertEqual(cfg.config, None)
request.actor.logger.info('Setting value') self.value = value def remote_get_value(self, request): request.actor.logger.info('Getting value') return self.value def start(arbiter, **kw): ensure_future(app(arbiter)) async def app(arbiter): # Spawn a new actor calc = await Calculator.spawn(name='calc1') print(calc.name) # set value in the remote calculator await calc.set_value(46) # get value from the remote calculator value = await calc.get_value() print(value) # Stop the application arbiter.stop() if __name__ == '__main__': cfg = Config() cfg.parse_command_line() arbiter(cfg=cfg, start=start).start()
class MultiWsgi(MultiApp): cfg = Config(bind=':0', rpc_bind=':0', bla='foo') def build(self): yield self.new_app(WSGIServer, callable=dummy) yield self.new_app(WSGIServer, 'rpc', callable=dummy)