def testSystem(self):
from pulsar import system
cfg = Config()
self.assertEqual(cfg.uid, system.get_uid())
self.assertEqual(cfg.gid, system.get_gid())
self.assertEqual(cfg.proc_name, 'pulsar')
cfg.set('process_name', 'bla')
self.assertEqual(cfg.proc_name, 'bla')
def testFunction(self):
cfg = Config()
worker = get_actor()
self.assertTrue(cfg.post_fork)
self.assertEqual(cfg.post_fork(worker), None)
cfg.set('post_fork', post_fork)
self.assertEqual(cfg.post_fork(worker), worker)
cfg1 = pickle.loads(pickle.dumps(cfg))
self.assertEqual(cfg1.post_fork(worker), worker)
class FlaskGreen(MultiApp):
"""Multiapp Server configurator
"""
cfg = Config(bind=':8080', echo_bind=':8060')
def build(self):
yield self.new_app(WSGIServer, callable=FlaskSite())
yield self.new_app(SocketServer, 'echo', callable=EchoServerProtocol,
echo_connection_made=log_connection)
def __testFunctionFromConfigFile(self):
# TODO, fails in pypy for some odd reasons
worker = get_actor()
cfg = Config()
self.assertEqual(cfg.connection_made(worker), None)
self.assertTrue(cfg.import_from_module(__file__))
self.assertEqual(cfg.connection_made(worker), worker)
cfg1 = pickle.loads(pickle.dumps(cfg))
self.assertEqual(cfg1.connection_made(worker), worker)
class MicroAgentApp(Application):
cfg = Config(apps=['microagent'])
def worker_start(self, worker, exc=None):
log = self.cfg.configured_logger()
signal_bus_dsn = self.cfg.settings['signal_bus'].value
signal_prefix = self.cfg.settings['signal_prefix'].value
if signal_bus_dsn.startswith('redis'):
bus = RedisSignalBus(signal_bus_dsn,
prefix=signal_prefix,
logger=log)
elif signal_bus_dsn.startswith('aioredis'):
from .aioredis import AIORedisSignalBus
bus = AIORedisSignalBus(signal_bus_dsn[3:],
prefix=signal_prefix,
logger=log)
else:
bus = None
queue_broker_dsn = self.cfg.settings.get('queue_broker').value
if queue_broker_dsn.startswith('amqp'):
from .amqp import AMQPBroker
ssl_context, cert_file = None, self.cfg.settings.get(
'broker_cert').value
if cert_file:
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False
ssl_context.load_verify_locations(cert_file)
broker = AMQPBroker(queue_broker_dsn, ssl_context=ssl_context)
elif queue_broker_dsn.startswith('kafka'):
from .kafka import KafkaBroker
broker = KafkaBroker(queue_broker_dsn)
elif queue_broker_dsn.startswith('redis'):
broker = PulsarRedisBroker(queue_broker_dsn)
else:
broker = None
worker.agent = self.cfg.agent(bus=bus,
broker=broker,
logger=log,
settings=self.cfg.settings)
asyncio.ensure_future(worker.agent.start())
def worker_stopping(self, worker, exc=None):
asyncio.ensure_future(worker.agent.stop())
def test_methods(self):
cfg = Config()
self.assertEqual(cfg.get('sdjcbsjkbcd', 'ciao'), 'ciao')
d = dict(cfg.items())
self.assertEqual(len(d), len(cfg))
sett = cfg.get('debug')
self.assertTrue(str(sett))
self.assertEqual(cfg.settings['debug'].default, False)
cfg.set('debug', True, default=True)
self.assertEqual(cfg.debug, True)
self.assertEqual(cfg.settings['debug'].default, True)
class QueueApp(Application):
"""A pulsar :class:`.Application` for consuming :class:`.Task`.
This application can also schedule periodic tasks when the
:ref:`schedule_periodic <setting-schedule_periodic>` flag is ``True``.
"""
backend_factory = Consumer
name = 'tasks'
cfg = Config(apps=('tasks', ),
version=__version__,
data_store=DEFAULT_MQ_BACKEND)
_backend = None
def api(self):
return Producer(self.cfg, logger=self.logger)
@property
def backend(self):
return self._backend
async def monitor_start(self, monitor, exc=None):
if not exc and self.cfg.workers:
self._backend = await self._start(monitor, False)
def monitor_task(self, monitor):
if monitor.is_running():
self._backend.tick(monitor)
def monitor_stopping(self, worker, exc=None):
if self._backend:
backend = self._backend
self._backend = None
return backend.close()
async def worker_start(self, worker, exc=None):
if not exc:
self._backend = await self._start(worker)
if not worker.is_monitor():
self._backend.bind_event('close', _close)
def worker_stopping(self, worker, exc=None):
if self._backend:
return self._backend.close()
def actorparams(self, monitor, params):
# makes sure workers are only consuming tasks, not scheduling.
cfg = params['cfg']
cfg.set('schedule_periodic', False)
def _start(self, actor, consume=True):
return self.backend_factory(self.cfg,
logger=self.logger).start(actor, consume)
class LuxApp(Application):
name = 'lux'
cfg = Config(include=('loglevel', 'loghandlers', 'debug', 'config'))
def __call__(self, actor=None):
try:
return super(LuxApp, self).__call__(actor)
except ImproperlyConfigured:
actor = actor or get_actor()
return self.on_config(actor)
def on_config(self, actor):
asyncio.set_event_loop(actor._loop)
return False
def _spawn_actor(kind, monitor, cfg=None, name=None, aid=None, **kw):
# Internal function which spawns a new Actor and return its
# ActorProxyMonitor.
# *cls* is the Actor class
# *monitor* can be either the arbiter or a monitor
if monitor:
params = monitor.actorparams()
name = params.pop('name', name)
aid = params.pop('aid', aid)
cfg = params.pop('cfg', cfg)
# get config if not available
if cfg is None:
if monitor:
cfg = monitor.cfg.copy()
else:
cfg = Config()
if not aid:
aid = create_aid()
if not monitor: # monitor not available, this is the arbiter
assert kind == 'arbiter'
name = kind
params = {}
if not cfg.exc_id:
cfg.set('exc_id', aid)
#
for key, value in kw.items():
if key in cfg.settings:
cfg.set(key, value)
else:
params[key] = value
#
if monitor:
kind = kind or cfg.concurrency
if not kind:
raise TypeError('Cannot spawn')
maker = concurrency_models.get(kind)
if maker:
c = maker()
return c.make(kind, cfg, name, aid, monitor=monitor, **params)
else:
raise ValueError('Concurrency %s not supported in pulsar' % kind)
def create_config(cls, params, prefix=None, name=None):
'''Create a new :class:`.Config` container.
Invoked during initialisation, it overrides defaults
with ``params`` and apply the ``prefix`` to non global
settings.
'''
if isinstance(cls.cfg, Config):
cfg = cls.cfg.copy(name=name, prefix=prefix)
else:
cfg = cls.cfg.copy()
if name:
cfg[name] = name
if prefix:
cfg[prefix] = prefix
cfg = Config(**cfg)
cfg.update_settings()
cfg.update(params, True)
return cfg
class PulsarQueue(MultiApp):
"""Build a multi-app consisting on a taskqueue and a JSON-RPC server.
"""
cfg = Config('Pulsar Queue')
def __init__(self, callable=None, **params):
super().__init__(**params)
self.manager = callable
def api(self):
return self.apps()[0].api()
def build(self):
yield self.new_app(QueueApp, callable=self.manager)
wsgi = self.cfg.params.get('wsgi')
if wsgi:
if wsgi is True:
wsgi = Rpc
yield self.new_app(RpcServer, prefix='rpc', callable=self)
def testDefaults(self):
from pulsar.utils import config
self.assertFalse(config.pass_through(None))
cfg = Config()
self.assertEqual(list(sorted(cfg)), list(sorted(cfg.settings)))
def _():
cfg.debug = 3
self.assertRaises(AttributeError, _)
#
name = tempfile.mktemp()
with open(name, 'w') as f:
f.write('a')
self.assertRaises(RuntimeError, cfg.import_from_module, name)
os.remove(name)
#
name = '%s.py' % name
with open(name, 'w') as f:
f.write('a')
self.assertRaises(RuntimeError, cfg.import_from_module, name)
os.remove(name)
def config(**kw):
return Config(**kw)
class Configurator(object):
'''A mixin for configuring and loading a pulsar application server.
:parameter name: to override the class :attr:`name` attribute.
:parameter description: to override the class :attr:`cfg.description`
attribute.
:parameter epilog: to override the class :attr:`cfg.epilog` attribute.
:parameter version: Optional version of this application, it overrides
the class :attr:`cfg.version` attribute.
:parameter argv: Optional list of command line parameters to parse, if
not supplied the :attr:`sys.argv` list will be used. The parameter
is only relevant if ``parse_console`` is ``True``.
:parameter parse_console: ``True`` (default) if the console parameters
needs parsing.
:parameter script: Optional string which set the :attr:`script`
attribute.
:parameter params: a dictionary of configuration parameters which
overrides the defaults and the :attr:`cfg` class attribute.
They will be overwritten by a :ref:`config file <setting-config>`
or command line arguments.
.. attribute:: name
The name is unique if this is an :class:`Application`. In this
case it defines the application monitor name as well and can be
access in the arbiter domain via the :func:`get_application`
function.
.. attribute:: argv
Optional list of command line parameters. If not available the
:attr:`sys.argv` list will be used when parsing the console.
.. attribute:: cfg
The :class:`.Config` for this :class:`Configurator`.
If set as class attribute it will be replaced during initialisation.
Default: ``None``.
.. attribute:: console_parsed
``True`` if this application parsed the console before starting.
.. attribute:: script
Full path of the script which starts the application or ``None``.
Evaluated during initialization via the :meth:`python_path` method.
'''
argv = None
name = None
cfg = Config()
def __init__(self,
name=None,
description=None,
epilog=None,
version=None,
argv=None,
parse_console=True,
script=None,
cfg=None,
load_config=True,
**params):
cls = self.__class__
self.name = name or cls.name or cls.__name__.lower()
if not isinstance(cfg, Config):
cfg = cfg or {}
cfg.update(params)
cfg = cls.create_config(cfg)
else:
cfg.update(params)
self.cfg = cfg
cfg.description = description or self.cfg.description
cfg.epilog = epilog or self.cfg.epilog
cfg.version = version or self.cfg.version
cfg.name = self.name
cfg.application = cls
self.argv = argv
self.console_parsed = parse_console
self.cfg.script = self.script = self.python_path(script)
@property
def version(self):
'''Version of this :class:`Application`'''
return self.cfg.version
@property
def root_dir(self):
'''Root directory of this :class:`Configurator`.
Evaluated from the :attr:`script` attribute.
'''
if self.cfg.script:
return os.path.dirname(self.cfg.script)
def __repr__(self):
return self.name
def __str__(self):
return self.__repr__()
def python_path(self, script):
'''Called during initialisation to obtain the ``script`` name and
to add the :attr:`script` directory to the python path if not in the
path already.
If ``script`` does not evalueate to ``True`` it is evaluated from
the ``__main__`` import. Returns the real path of the python
script which runs the application.
'''
if not script:
try:
import __main__
script = getfile(__main__)
except Exception: # pragma nocover
return
script = os.path.realpath(script)
path = os.path.dirname(script)
if path not in sys.path:
sys.path.insert(0, path)
return script
def on_config(self, arbiter):
'''Callback when configuration is loaded.
This is a chance to do applications specific checks before the
concurrent machinery is put into place.
If it returns ``False`` the application will abort.
'''
pass
def load_config(self):
'''Load the application configuration from a file and/or
from the command line.
Called during application initialisation. The parameters
overriding order is the following:
* default parameters.
* the key-valued params passed in the initialisation.
* the parameters in the optional configuration file
* the parameters passed in the command line.
'''
# get the actor if available and override default cfg values with those
# from the actor
actor = get_actor()
if actor and actor.is_running():
# actor available and running.
# Unless argv is set, skip parsing
if self.argv is None:
self.console_parsed = False
# copy global settings
self.cfg.copy_globals(actor.cfg)
#
for name in list(self.cfg.params):
if name in self.cfg.settings:
value = self.cfg.params.pop(name)
if value is not None:
self.cfg.set(name, value)
# parse console args
if self.console_parsed:
self.cfg.parse_command_line(self.argv)
else:
self.cfg.params.update(self.cfg.import_from_module())
def start(self):
'''Invoked the application callable method and start
the ``arbiter`` if it wasn't already started.
It returns a :class:`~asyncio.Future` called back once the
application/applications are running. It returns ``None`` if
called more than once.
'''
on_start = self()
arbiter = pulsar.arbiter()
if arbiter and on_start:
arbiter.start()
return on_start
@classmethod
def create_config(cls, params, prefix=None, name=None):
'''Create a new :class:`.Config` container.
Invoked during initialisation, it overrides defaults
with ``params`` and apply the ``prefix`` to non global
settings.
'''
if isinstance(cls.cfg, Config):
cfg = cls.cfg.copy(name=name, prefix=prefix)
else:
cfg = cls.cfg.copy()
if name:
cfg[name] = name
if prefix:
cfg[prefix] = prefix
cfg = Config(**cfg)
cfg.update_settings()
cfg.update(params, True)
return cfg
def test_exclude(self):
cfg = Config(exclude=['config'])
self.assertEqual(cfg.config, 'config.py')
self.assertEqual(cfg.params['config'], 'config.py')
self.assertFalse('config' in cfg.settings)
def testBadConfig(self):
cfg = Config()
self.assertEqual(cfg.config, 'config.py')
self.assertEqual(cfg.import_from_module('foo/bla/cnkjnckjcn.py'), [])
cfg.set('config', None)
self.assertEqual(cfg.config, None)
request.actor.logger.info('Setting value')
self.value = value
def remote_get_value(self, request):
request.actor.logger.info('Getting value')
return self.value
def start(arbiter, **kw):
ensure_future(app(arbiter))
async def app(arbiter):
# Spawn a new actor
calc = await Calculator.spawn(name='calc1')
print(calc.name)
# set value in the remote calculator
await calc.set_value(46)
# get value from the remote calculator
value = await calc.get_value()
print(value)
# Stop the application
arbiter.stop()
if __name__ == '__main__':
cfg = Config()
cfg.parse_command_line()
arbiter(cfg=cfg, start=start).start()
class MultiWsgi(MultiApp):
cfg = Config(bind=':0', rpc_bind=':0', bla='foo')
def build(self):
yield self.new_app(WSGIServer, callable=dummy)
yield self.new_app(WSGIServer, 'rpc', callable=dummy)
