forked from quantmind/pulsar
-
Notifications
You must be signed in to change notification settings - Fork 0
/
__init__.py
665 lines (548 loc) · 21.8 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
'''
This module implements the main classes for pulsar application framework. The
framework is built on top of pulsar asynchronous engine and allows to
implement servers with very little effort. The main classes here are:
:class:`Application` and :class:`MultiApp` which, has the name suggests, is
a factory of several :class:`Application` running on a single server.
The :class:`Configurator` is a mixin used as base class for both
:class:`Application` and :class:`MultiApp`.
Configurator
===============================
.. autoclass:: Configurator
:members:
:member-order: bysource
Application
===============================
.. autoclass:: Application
:members:
:member-order: bysource
.. automethod:: __call__
Multi App
===============================
.. autoclass:: MultiApp
:members:
:member-order: bysource
Get application
=========================
.. autofunction:: get_application
.. _when-monitor-start:
When monitor start
=================================
The application framework provides a way for adding hooks which are executed
every time a new application starts. A hook is registered by::
from pulsar.apps import when_monitor_start
def myhook(monitor):
...
when_monitor_start.append(myhook)
By default, the list of hooks only contains a callback to start the
:ref:`default data store <setting-data_store>` if it needs to.
'''
import os
import sys
from inspect import getfile
from functools import partial
from collections import namedtuple, OrderedDict
import pulsar
from pulsar import (get_actor, Config, task,
multi_async, Future, ImproperlyConfigured)
__all__ = ['Application', 'MultiApp', 'get_application', 'when_monitor_start']
when_monitor_start = []
new_app = namedtuple('new_app', 'prefix params')
def get_application(name):
'''Fetch an :class:`Application` associated with ``name`` if available.
This function may return an :ref:`asynchronous component <coroutine>`.
The application name is set during initialisation. Check the
:attr:`Configurator.name` attribute for more information.
'''
actor = get_actor()
if actor:
if actor.is_arbiter():
return _get_app(actor, name, False)
else:
return _get_remote_app(actor, name)
def _get_remote_app(actor, name):
cfg = yield from actor.send('arbiter', 'run', _get_app, name)
return cfg.app() if cfg else None
def _get_app(arbiter, name, safe=True):
monitor = arbiter.get_actor(name)
if monitor:
cfg = yield from monitor.start_event
if safe:
return cfg
else:
return monitor.app
@task
def monitor_start(self, exc=None):
start_event = self.start_event
if exc:
start_event.set_exception(exc)
return
app = self.app
try:
self.bind_event('on_params', monitor_params)
self.bind_event('on_info', monitor_info)
self.bind_event('stopping', monitor_stopping)
for callback in when_monitor_start:
coro = callback(self)
if coro:
yield from coro
self.bind_event('periodic_task', app.monitor_task)
coro = app.monitor_start(self)
if coro:
yield from coro
if not self.cfg.workers:
coro = app.worker_start(self)
if coro:
yield from coro
result = self.cfg
except Exception as exc:
coro = self.stop(exc)
if coro:
yield from coro
start_event.set_result(None)
else:
start_event.set_result(result)
@task
def monitor_stopping(self, exc=None):
if not self.cfg.workers:
coro = self.app.worker_stopping(self)
if coro:
yield from coro
coro = self.app.monitor_stopping(self)
if coro:
yield from coro
return self
def monitor_info(self, info=None):
if not self.cfg.workers:
self.app.worker_info(self, info)
else:
self.app.monitor_info(self, info)
def monitor_params(self, params=None):
app = self.app
params.update({'cfg': app.cfg.clone(),
'name': '%s.worker' % app.name,
'start': worker_start})
app.actorparams(self, params)
def worker_start(self, exc=None):
app = getattr(self, 'app', None)
if app is None:
cfg = self.cfg
self.app = app = cfg.application.from_config(cfg, logger=self.logger)
self.bind_event('on_info', app.worker_info)
self.bind_event('stopping', app.worker_stopping)
return app.worker_start(self, exc=exc)
class Configurator(object):
'''A mixin for configuring and loading a pulsar application server.
:parameter name: to override the class :attr:`name` attribute.
:parameter description: to override the class :attr:`cfg.description`
attribute.
:parameter epilog: to override the class :attr:`cfg.epilog` attribute.
:parameter version: Optional version of this application, it overrides
the class :attr:`cfg.version` attribute.
:parameter argv: Optional list of command line parameters to parse, if
not supplied the :attr:`sys.argv` list will be used. The parameter
is only relevant if ``parse_console`` is ``True``.
:parameter parse_console: ``True`` (default) if the console parameters
needs parsing.
:parameter script: Optional string which set the :attr:`script`
attribute.
:parameter params: a dictionary of configuration parameters which
overrides the defaults and the :attr:`cfg` class attribute.
They will be overwritten by a :ref:`config file <setting-config>`
or command line arguments.
.. attribute:: name
The name is unique if this is an :class:`Application`. In this
case it defines the application monitor name as well and can be
access in the arbiter domain via the :func:`get_application`
function.
.. attribute:: argv
Optional list of command line parameters. If not available the
:attr:`sys.argv` list will be used when parsing the console.
.. attribute:: cfg
The :class:`.Config` for this :class:`Configurator`.
If set as class attribute it will be replaced during initialisation.
Default: ``None``.
.. attribute:: console_parsed
``True`` if this application parsed the console before starting.
.. attribute:: script
Full path of the script which starts the application or ``None``.
Evaluated during initialization via the :meth:`python_path` method.
'''
argv = None
name = None
cfg = Config()
def __init__(self,
name=None,
description=None,
epilog=None,
version=None,
argv=None,
parse_console=True,
script=None,
cfg=None,
load_config=True,
**params):
cls = self.__class__
self.name = name or cls.name or cls.__name__.lower()
if not isinstance(cfg, Config):
cfg = cfg or {}
cfg.update(params)
cfg = cls.create_config(cfg)
else:
cfg.update(params)
self.cfg = cfg
cfg.description = description or self.cfg.description
cfg.epilog = epilog or self.cfg.epilog
cfg.version = version or self.cfg.version
cfg.name = self.name
cfg.application = cls
self.argv = argv
self.console_parsed = parse_console
self.cfg.script = self.script = self.python_path(script)
@property
def version(self):
'''Version of this :class:`Application`'''
return self.cfg.version
@property
def root_dir(self):
'''Root directory of this :class:`Configurator`.
Evaluated from the :attr:`script` attribute.
'''
if self.cfg.script:
return os.path.dirname(self.cfg.script)
def __repr__(self):
return self.name
def __str__(self):
return self.__repr__()
def python_path(self, script):
'''Called during initialisation to obtain the ``script`` name and
to add the :attr:`script` directory to the python path if not in the
path already.
If ``script`` does not evalueate to ``True`` it is evaluated from
the ``__main__`` import. Returns the real path of the python
script which runs the application.
'''
if not script:
try:
import __main__
script = getfile(__main__)
except Exception: # pragma nocover
return
script = os.path.realpath(script)
path = os.path.dirname(script)
if path not in sys.path:
sys.path.insert(0, path)
return script
def on_config(self, arbiter):
'''Callback when configuration is loaded.
This is a chance to do applications specific checks before the
concurrent machinery is put into place.
If it returns ``False`` the application will abort.
'''
pass
def load_config(self):
'''Load the application configuration from a file and/or
from the command line.
Called during application initialisation. The parameters
overriding order is the following:
* default parameters.
* the key-valued params passed in the initialisation.
* the parameters in the optional configuration file
* the parameters passed in the command line.
'''
# get the actor if available and override default cfg values with those
# from the actor
actor = get_actor()
if actor and actor.is_running():
# actor available and running.
# Unless argv is set, skip parsing
if self.argv is None:
self.console_parsed = False
# copy global settings
self.cfg.copy_globals(actor.cfg)
#
for name in list(self.cfg.params):
if name in self.cfg.settings:
value = self.cfg.params.pop(name)
if value is not None:
self.cfg.set(name, value)
# parse console args
if self.console_parsed:
self.cfg.parse_command_line(self.argv)
else:
self.cfg.params.update(self.cfg.import_from_module())
def start(self):
'''Invoked the application callable method and start
the ``arbiter`` if it wasn't already started.
It returns a :class:`~asyncio.Future` called back once the
application/applications are running. It returns ``None`` if
called more than once.
'''
on_start = self()
arbiter = pulsar.arbiter()
if arbiter and on_start:
arbiter.start()
return on_start
@classmethod
def create_config(cls, params, prefix=None, name=None):
'''Create a new :class:`.Config` container.
Invoked during initialisation, it overrides defaults
with ``params`` and apply the ``prefix`` to non global
settings.
'''
if isinstance(cls.cfg, Config):
cfg = cls.cfg.copy(name=name, prefix=prefix)
else:
cfg = cls.cfg.copy()
if name:
cfg[name] = name
if prefix:
cfg[prefix] = prefix
cfg = Config(**cfg)
cfg.update_settings()
cfg.update(params, True)
return cfg
class Application(Configurator):
"""An application interface.
Applications can be of any sorts or forms and the library is shipped with
several battery included examples in the :mod:`pulsar.apps` module.
These are the most important facts about a pulsar :class:`Application`:
* It derives from :class:`Configurator` so that it has all the
functionalities to parse command line arguments and setup the
:attr:`~Configurator.cfg` placeholder of :class:`.Setting`.
* Instances of an :class:`Application` are callable objects accepting
the calling actor as only argument. The callable method should
not be overwritten, instead one should overwrites the application
hooks available.
* When an :class:`Application` is called for the first time,
a new ``monitor`` is added to the ``arbiter``,
ready to perform its duties.
:parameter callable: Initialise the :attr:`callable` attribute.
:parameter load_config: If ``False`` the :meth:`~Configurator.load_config`
method is not invoked.
Default ``True``.
:parameter params: Passed to the :class:`Configurator` initialiser.
.. attribute:: callable
Optional callable serving or configuring your application.
If provided, the callable must be picklable, therefore it is either
a function or a picklable object.
Default ``None``
"""
def __init__(self, callable=None, load_config=True, **params):
super().__init__(load_config=load_config, **params)
self.cfg.callable = callable
self.logger = None
if load_config:
self.load_config()
@classmethod
def from_config(cls, cfg, logger=None):
c = cls.__new__(cls)
c.name = cfg.name
c.cfg = cfg
c.logger = logger or cfg.configured_logger()
return c
@property
def stream(self):
'''Actor stream handler.
'''
return get_actor().stream
def __call__(self, actor=None):
'''Register this application with the (optional) calling ``actor``.
If an ``actor`` is available (either via the function argument or via
the :func:`~pulsar.async.actor.get_actor` function) it must be
``arbiter``, otherwise this call is no-op.
If no actor is available, it means this application starts
pulsar engine by creating the ``arbiter`` with its
:ref:`global settings <setting-section-global-server-settings>`
copied to the arbiter :class:`.Config` container.
:return: the ``start`` one time event fired once this application
has fired it.
'''
if actor is None:
actor = get_actor()
monitor = None
if actor and actor.is_arbiter():
monitor = actor.get_actor(self.name)
if monitor is None and (not actor or actor.is_arbiter()):
self.cfg.on_start()
self.logger = self.cfg.configured_logger()
if not actor:
actor = pulsar.arbiter(cfg=self.cfg.clone())
else:
self.update_arbiter_params(actor)
if not self.cfg.exc_id:
self.cfg.set('exc_id', actor.cfg.exc_id)
if self.on_config(actor) is not False:
start = Future(loop=actor._loop)
actor.bind_event('start', partial(self._add_monitor, start))
return start
else:
return
raise ImproperlyConfigured('Already started or not in arbiter domain')
# WORKERS CALLBACKS
def worker_start(self, worker, exc=None):
'''Added to the ``start`` :ref:`worker hook <actor-hooks>`.'''
pass
def worker_info(self, worker, info):
'''Hook to add additional entries to the worker ``info`` dictionary.
'''
pass
def worker_stopping(self, worker, exc=None):
'''Added to the ``stopping`` :ref:`worker hook <actor-hooks>`.'''
pass
# MONITOR CALLBACKS
def actorparams(self, monitor, params=None):
'''Hook to add additional entries when the monitor spawn new actors.
'''
pass
def monitor_start(self, monitor):
'''Callback by the monitor when starting.
'''
pass
def monitor_info(self, monitor, info):
'''Hook to add additional entries to the monitor ``info`` dictionary.
'''
pass
def monitor_stopping(self, monitor):
'''Callback by the monitor before stopping.
'''
pass
def monitor_task(self, monitor):
'''Executed by the :class:`.Monitor` serving this application
at each event loop.'''
pass
def update_arbiter_params(self, arbiter):
for s in self.cfg.settings.values():
if s.is_global and s.modified:
a = arbiter.cfg.settings[s.name]
if not a.modified:
a.set(s.value)
# INTERNALS
def _add_monitor(self, start, arbiter, exc=None):
if not exc:
monitor = arbiter.add_monitor(
self.name, app=self, cfg=self.cfg,
start=monitor_start, start_event=start)
self.cfg = monitor.cfg
class MultiApp(Configurator):
'''A :class:`MultiApp` is a tool for creating several :class:`Application`
and starting them at once.
It makes sure all :ref:`settings <settings>` for the
applications created are available in the command line.
Check the :class:`~examples.taskqueue.manage.server` class in the
:ref:`taskqueue example <tutorials-taskqueue>` for an actual
implementation.
:class:`MultiApp` derives from :class:`Configurator` and therefore
supports all its configuration utilities,
:meth:`build` is the only method which must be implemented by
subclasses.
A minimal example usage::
import pulsar
class Server(pulsar.MultiApp):
def build(self):
yield self.new_app(TaskQueue)
yield self.new_app(WSGIserver, prefix="rpc", callable=..., ...)
yield self.new_app(WSGIserver, prefix="web", callable=..., ...)
'''
_apps = None
def build(self):
'''Virtual method, must be implemented by subclasses and return an
iterable over results obtained from calls to the
:meth:`new_app` method.
'''
raise NotImplementedError
def apps(self):
'''List of :class:`Application` for this :class:`MultiApp`.
The list is lazily loaded from the :meth:`build` method.
'''
if self._apps is None:
# Add modified settings values to the list of cfg params
self.cfg.params.update(((s.name, s.value) for s in
self.cfg.settings.values() if s.modified))
self.cfg.settings = {}
self._apps = OrderedDict()
self._apps.update(self._build())
if not self._apps:
return []
# Load the configuration (command line and config file)
self.load_config()
kwargs = self._get_app_params()
apps = self._apps
self._apps = []
for App, name, callable, cfg in self._iter_app(apps):
settings = self.cfg.settings
new_settings = {}
for key in cfg:
setting = settings[key].copy()
if setting.orig_name and setting.orig_name != setting.name:
setting.name = setting.orig_name
new_settings[setting.name] = setting
cfg.settings = new_settings
kwargs.update({'name': name, 'cfg': cfg, 'callable': callable})
if name == self.name:
params = kwargs.copy()
params['version'] = self.version
else:
params = kwargs
self._apps.append(App(**params))
return self._apps
def new_app(self, App, prefix=None, callable=None, **params):
'''Invoke this method in the :meth:`build` method as many times
as the number of :class:`Application` required by this
:class:`MultiApp`.
:param App: an :class:`Application` class.
:param prefix: The prefix to use for the application,
the prefix is appended to
the application :ref:`config parameters <settings>` and to the
application name. Each call to this methjod must use a different
value of for this parameter. It can be ``None``.
:param callable: optional callable (function of object) used during
initialisation of *App* (the :class:`Application.callable`).
:param params: additional key-valued parameters used when creating
an instance of *App*.
:return: a tuple used by the :meth:`apps` method.
'''
params.update(self.cfg.params.copy())
params.pop('name', None) # remove the name
prefix = prefix or ''
if not prefix and '' in self._apps:
prefix = App.name or App.__name__.lower()
if not prefix:
name = self.name
cfg = App.create_config(params, name=name)
else:
name = '%s_%s' % (prefix, self.name)
cfg = App.create_config(params, prefix=prefix, name=name)
# Add the config entry to the multi app config if not available
for k in cfg.settings:
if k not in self.cfg.settings:
self.cfg.settings[k] = cfg.settings[k]
return new_app(prefix, (App, name, callable, cfg))
def __call__(self, actor=None):
apps = [app(actor) for app in self.apps()]
return multi_async(apps, loop=get_actor()._loop)
# INTERNALS
def _build(self):
for app in self.build():
if not isinstance(app, new_app):
raise ImproperlyConfigured(
'You must use new_app when building a MultiApp')
yield app
def _iter_app(self, app_name_callables):
main = app_name_callables.pop('', None)
if not main:
raise ImproperlyConfigured('No main application in MultiApp')
yield main
for app in app_name_callables.values():
yield app
def _get_app_params(self):
params = self.cfg.params.copy()
for key, value in self.__dict__.items():
if key.startswith('_'):
continue
elif key == 'console_parsed':
params['parse_console'] = not value
else:
params[key] = value
params['load_config'] = False
return params