def next_scheduled(self, jobnames=None):
if not self.schedule_periodic:
return
if jobnames:
entries = (self.entries.get(name, None) for name in jobnames)
else:
entries = itervalues(self.entries)
next_entry = None
next_time = None
for entry in entries:
if entry is None:
continue
is_due, next_time_to_run = entry.is_due()
if is_due:
next_time = 0
next_entry = entry
break
elif next_time_to_run is not None:
if next_time is None or next_time_to_run < next_time:
next_time = next_time_to_run
next_entry = entry
if next_entry:
return (next_entry.name, max(next_time, 0))
else:
return (jobnames, None)
def _info_monitor(self, actor, info=None):
if actor.started():
info['actor'].update({'concurrency': actor.cfg.concurrency,
'workers': len(self.managed_actors)})
info['workers'] = [a.info for a in itervalues(self.managed_actors)
if a.info]
return info
def periodic_task(self, actor, **kw):
'''Override the :meth:`.Concurrency.periodic_task` to implement
the :class:`.Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
interval = 0
actor.next_periodic_task = None
#
if actor.started():
# managed actors job
self.manage_actors(actor)
for m in list(itervalues(self.monitors)):
if m.closed():
actor._remove_actor(m)
interval = MONITOR_TASK_PERIOD
if not actor.is_running() and actor.cfg.debug:
actor.logger.debug('still stopping')
#
actor.fire_event('periodic_task')
if not actor.closed():
actor.next_periodic_task = actor._loop.call_later(
interval, self.periodic_task, actor)
if actor.cfg.reload and autoreload.check_changes():
actor.stop(exit_code=autoreload.EXIT_CODE)
def stream(self, whitespace=''):
'''This function convert the :class:`css` element into a string.'''
# First we execute mixins
if self.rendered:
raise RuntimeError('%s already rendered' % self)
self.rendered = True
children = self._children
self._children = OrderedDict()
for tag, clist in iteritems(children):
for c in clist:
c._parent = None
s = c.set_parent(self)
if s: # the child (mixin) has return a string, added it.
yield (None, s)
data = []
for k, v in self._attributes:
v = as_value(v)
if v is not None:
data.append('%s %s: %s;' % (whitespace, k, v))
if data:
yield (self.tag, '\n'.join(data))
# yield Mixins and children
for child_list in itervalues(self._children):
if isinstance(child_list, list):
child = child_list[0]
for c in child_list[1:]:
child.extend(c)
for s in child.stream(whitespace):
yield s
else:
yield None, child_list
def next_scheduled(self, jobnames=None):
if not self.schedule_periodic:
return
if jobnames:
entries = (self.entries.get(name, None) for name in jobnames)
else:
entries = itervalues(self.entries)
next_entry = None
next_time = None
for entry in entries:
if entry is None:
continue
is_due, next_time_to_run = entry.is_due()
if is_due:
next_time = 0
next_entry = entry
break
elif next_time_to_run is not None:
if next_time is None or next_time_to_run < next_time:
next_time = next_time_to_run
next_entry = entry
if next_entry:
return (next_entry.name, max(next_time, 0))
else:
return (jobnames, None)
def _channel_groups(actor, channel):
for channel_re, group in itervalues(_get_pubsub_channels(actor)):
if hasattr(channel_re, 'match'):
g = channel_re.match(channel)
if g:
yield g.group(), group
elif channel_re == channel:
yield channel, group
def info(self):
info = super(Monitor, self).info()
if self.started():
info['actor'].update({'concurrency': self.cfg.concurrency,
'workers': len(self.managed_actors)})
info['workers'] = [a.info for a in itervalues(self.managed_actors)
if a.info]
return info
def _close_all(self, actor):
# Close al monitors at once
try:
for m in itervalues(self.monitors):
yield From(m.stop())
yield From(self._close_actors(actor))
except Exception:
actor.logger.exception('Exception while closing arbiter')
self._exit_arbiter(actor, True)
def info(self):
data = super(Arbiter, self).info()
monitors = {}
for m in itervalues(self.monitors):
info = m.info()
actor = info['actor']
monitors[actor['name']] = info
server = data.pop('actor')
server.update({'version': pulsar.__version__,
'name': pulsar.SERVER_NAME,
'number_of_monitors': len(self.monitors),
'number_of_actors': len(self.managed_actors)})
server.pop('is_process', None)
server.pop('ppid', None)
server.pop('actor_id', None)
server.pop('age', None)
data['server'] = server
data['workers'] = [a.info for a in itervalues(self.managed_actors)]
data['monitors'] = monitors
return data
def info(self):
info = super(Monitor, self).info()
if self.started():
info['actor'].update({
'concurrency': self.cfg.concurrency,
'workers': len(self.managed_actors)
})
info['workers'] = [
a.info for a in itervalues(self.managed_actors) if a.info
]
return info
def info(self):
data = {'actor': {'actor_class': self.actor_class.__name__,
'concurrency': self.cfg.concurrency,
'name': self.name,
'age': self.impl.age,
'workers': len(self.managed_actors)}}
if not self.started():
return data
data['workers'] = [a.info for a in itervalues(self.managed_actors)
if a.info]
return data
def stop_actors(self):
"""Maintain the number of workers by spawning or killing as required
"""
if self.cfg.workers:
num_to_kill = len(self.managed_actors) - self.cfg.workers
for i in range(num_to_kill, 0, -1):
w, kage = 0, sys.maxsize
for worker in itervalues(self.managed_actors):
age = worker.impl.age
if age < kage:
w, kage = w, age
self.manage_actor(w, True)
def stop_actors(self, monitor):
"""Maintain the number of workers by spawning or killing as required
"""
if monitor.cfg.workers:
num_to_kill = len(self.managed_actors) - monitor.cfg.workers
for i in range(num_to_kill, 0, -1):
w, kage = 0, sys.maxsize
for worker in itervalues(self.managed_actors):
age = worker.impl.age
if age < kage:
w, kage = w, age
self.manage_actor(monitor, w, True)
def _info_monitor(self, actor, info=None):
data = info
monitors = {}
for m in itervalues(self.monitors):
info = m.info()
if info:
actor = info['actor']
monitors[actor['name']] = info
server = data.pop('actor')
server.update({'version': pulsar.__version__,
'name': pulsar.SERVER_NAME,
'number_of_monitors': len(self.monitors),
'number_of_actors': len(self.managed_actors)})
server.pop('is_process', None)
server.pop('ppid', None)
server.pop('actor_id', None)
server.pop('age', None)
data['server'] = server
data['workers'] = [a.info for a in itervalues(self.managed_actors)]
data['monitors'] = monitors
return data
def tick(self, now=None):
# Run a tick, that is one iteration of the scheduler.
if not self.schedule_periodic:
return
remaining_times = []
for entry in itervalues(self.entries):
is_due, next_time_to_run = entry.is_due(now=now)
if is_due:
self.queue_task(entry.name)
if next_time_to_run:
remaining_times.append(next_time_to_run)
self.next_run = now or time.time()
if remaining_times:
self.next_run += min(remaining_times)
def tick(self, now=None):
# Run a tick, that is one iteration of the scheduler.
if not self.schedule_periodic:
return
remaining_times = []
for entry in itervalues(self.entries):
is_due, next_time_to_run = entry.is_due(now=now)
if is_due:
self.queue_task(entry.name)
if next_time_to_run:
remaining_times.append(next_time_to_run)
self.next_run = now or time.time()
if remaining_times:
self.next_run += min(remaining_times)
def periodic_task(self, actor, **kw):
'''Override the :meth:`.Concurrency.periodic_task` to implement
the :class:`.Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
interval = 0
actor.next_periodic_task = None
if actor.is_running():
# managed actors job
interval = MONITOR_TASK_PERIOD
actor.manage_actors()
for m in list(itervalues(actor.monitors)):
if m.started() and not m.is_running():
actor._remove_actor(m)
actor.next_periodic_task = actor._loop.call_later(
interval, self.periodic_task, actor)
def copy_many_times_events(self, other):
'''Copy :ref:`many times events <many-times-event>` from ``other``.
All many times events of ``other`` are copied to this handler
provided the events handlers already exist.
'''
if isinstance(other, EventHandler):
events = self._events
for event in itervalues(other._events):
if isinstance(event, ManyEvent):
ev = events.get(event.name)
# If the event is available add it
if ev:
for callback in event._handlers:
ev.bind(callback)
def get_actor(self, aid):
'''Given an actor unique id return the actor proxy.'''
a = super(Arbiter, self).get_actor(aid)
if a is None:
if aid in self.monitors: # Check in monitors aid
return self.monitors[aid]
elif aid in self.managed_actors:
return self.managed_actors[aid]
elif aid in self.registered:
return self.registered[aid]
else: # Finally check in workers in monitors
for m in itervalues(self.monitors):
if aid in m.managed_actors:
return m.managed_actors[aid]
else:
return a
def get_actor(self, aid):
'''Given an actor unique id return the actor proxy.'''
a = super(Arbiter, self).get_actor(aid)
if a is None:
if aid in self.monitors: # Check in monitors aid
return self.monitors[aid]
elif aid in self.managed_actors:
return self.managed_actors[aid]
elif aid in self.registered:
return self.registered[aid]
else: # Finally check in workers in monitors
for m in itervalues(self.monitors):
if aid in m.managed_actors:
return m.managed_actors[aid]
else:
return a
def middleware(self, app):
'''Build the API middleware.
If :setting:`API_URL` is defined, it loops through all extensions
and checks if the ``api_sections`` method is available.
'''
middleware = []
url = app.config['API_URL']
if url:
app.config['API_URL'] = url = remove_double_slash('/%s/' % url)
sections = {}
self.api = api = Api(url, sections=sections)
middleware.append(api)
docs = None
url = app.config['API_DOCS_URL']
if url:
app.config['API_DOCS_URL'] = url = remove_double_slash('/%s/' %
url)
self.docs = Router(url, sections=sections)
middleware.append(self.docs)
#
for extension in itervalues(app.extensions):
api_sections = getattr(extension, 'api_sections', None)
if api_sections:
for name, routers in api_sections(app):
# Routes must be instances of CRUD
# name is the section name
if name not in sections:
sections[name] = ApiSection()
section = sections[name]
for router in routers:
api.add_child(router)
section.append(router)
manager = router.manager
self.api_crud_routers[manager] = manager
url = app.config['ADMIN_URL']
if url:
app.config['ADMIN_URL'] = url = remove_double_slash('/%s/' % url)
sections = {}
admin = Admin(url, sections=sections)
middleware.append(admin)
#for extension in itervalues(app.extensions):
# api_sections = getattr(extension, 'api_sections', None)
# if api_sections:
# pass
return middleware
def middleware(self, app):
'''Build the API middleware.
If :setting:`API_URL` is defined, it loops through all extensions
and checks if the ``api_sections`` method is available.
'''
middleware = []
url = app.config['API_URL']
if url:
app.config['API_URL'] = url = remove_double_slash('/%s/' % url)
sections = {}
self.api = api = Api(url, sections=sections)
middleware.append(api)
docs = None
url = app.config['API_DOCS_URL']
if url:
app.config['API_DOCS_URL'] = url = remove_double_slash('/%s/'
% url)
self.docs = Router(url, sections=sections)
middleware.append(self.docs)
#
for extension in itervalues(app.extensions):
api_sections = getattr(extension, 'api_sections', None)
if api_sections:
for name, routers in api_sections(app):
# Routes must be instances of CRUD
# name is the section name
if name not in sections:
sections[name] = ApiSection()
section = sections[name]
for router in routers:
api.add_child(router)
section.append(router)
manager = router.manager
self.api_crud_routers[manager] = manager
url = app.config['ADMIN_URL']
if url:
app.config['ADMIN_URL'] = url = remove_double_slash('/%s/' % url)
sections = {}
admin = Admin(url, sections=sections)
middleware.append(admin)
#for extension in itervalues(app.extensions):
# api_sections = getattr(extension, 'api_sections', None)
# if api_sections:
# pass
return middleware
def get_actor(self, actor, aid, check_monitor=True):
'''Given an actor unique id return the actor proxy.'''
a = super(ArbiterConcurrency, self).get_actor(actor, aid)
if a is None:
if aid in self.monitors: # Check in monitors aid
return self.monitors[aid]
elif aid in self.managed_actors:
return self.managed_actors[aid]
elif aid in self.registered:
return self.registered[aid]
else: # Finally check in workers in monitors
for m in itervalues(self.monitors):
a = m.get_actor(aid, check_monitor=False)
if a is not None:
return a
else:
return a
def periodic_task(self, actor):
'''Override the :meth:`Concurrency.periodic_task` to implement
the :class:`Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
interval = 0
actor.next_periodic_task = None
if actor.is_running():
# managed actors job
interval = MONITOR_TASK_PERIOD
actor.manage_actors()
for m in list(itervalues(actor.monitors)):
if m.started():
if not m.is_running():
actor._remove_actor(m)
else:
m.start()
actor.next_periodic_task = actor.event_loop.call_later(
interval, self.periodic_task, actor)
return actor
def get_tasks(self, **filters):
tasks = []
if filters:
fs = []
for name, value in filters.items():
if not isinstance(value, (list, tuple, set, frozenset)):
value = (value,)
fs.append((name, value))
# Loop over tasks
for task in itervalues(self._tasks):
select = True
for name, values in fs:
value = getattr(task, name, None)
if value not in values:
select = False
break
if select:
tasks.append(task)
return tasks
def get_tasks(self, **filters):
tasks = []
if filters:
fs = []
for name, value in filters.items():
if not isinstance(value, (list, tuple, set, frozenset)):
value = (value, )
fs.append((name, value))
# Loop over tasks
for task in itervalues(self._tasks):
select = True
for name, values in fs:
value = getattr(task, name, None)
if value not in values:
select = False
break
if select:
tasks.append(task)
return tasks
def tick(self, now=None):
'''Run a tick, that is one iteration of the scheduler. This
method only works when :attr:`schedule_periodic` is ``True`` and
the arbiter context.
Executes all due tasks and calculate the time in seconds to wait before
running a new :meth:`tick`. For testing purposes a :class:`datetime.datetime`
value ``now`` can be passed.'''
if not self.schedule_periodic:
return
remaining_times = []
try:
for entry in itervalues(self.entries):
is_due, next_time_to_run = entry.is_due(now=now)
if is_due:
self.run_job(entry.name)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except Exception:
LOGGER.exception('Unhandled error in task backend')
self.next_run = now or datetime.now()
if remaining_times:
self.next_run += timedelta(seconds=min(remaining_times))
def handler(self):
'''The WSGI application handler for this :class:`App`.
It is lazily loaded the first time it is accessed so that
this :class:`App` can be used by pulsar in a multiprocessing setup.
'''
# do this here so that the config is already loaded before fire signal
extensions = list(itervalues(self.extensions))
self.fire('on_config')
middleware = []
rmiddleware = []
for extension in extensions:
_middleware = extension.middleware(self)
if _middleware:
middleware.extend(_middleware)
_middleware = extension.response_middleware(self)
if _middleware:
rmiddleware.extend(_middleware)
# Response middleware executed in reversed order
rmiddleware = list(reversed(rmiddleware))
handler = WsgiHandler(middleware, response_middleware=rmiddleware)
self.fire('on_loaded', handler)
return handler
def tick(self, now=None):
'''Run a tick, that is one iteration of the scheduler. This
method only works when :attr:`schedule_periodic` is ``True`` and
the arbiter context.
Executes all due tasks and calculate the time in seconds to wait before
running a new :meth:`tick`. For testing purposes a :class:`datetime.datetime`
value ``now`` can be passed.'''
if not self.schedule_periodic:
return
remaining_times = []
try:
for entry in itervalues(self.entries):
is_due, next_time_to_run = entry.is_due(now=now)
if is_due:
self.run_job(entry.name)
if next_time_to_run:
remaining_times.append(next_time_to_run)
except Exception:
LOGGER.exception('Unhandled error in task backend')
self.next_run = now or datetime.now()
if remaining_times:
self.next_run += timedelta(seconds=min(remaining_times))
def handler(self):
'''The WSGI application handler for this :class:`App`.
It is lazily loaded the first time it is accessed so that
this :class:`App` can be used by pulsar in a multiprocessing setup.
'''
# do this here so that the config is already loaded before fire signal
extensions = list(itervalues(self.extensions))
self.fire('on_config')
middleware = []
rmiddleware = []
for extension in extensions:
_middleware = extension.middleware(self)
if _middleware:
middleware.extend(_middleware)
_middleware = extension.response_middleware(self)
if _middleware:
rmiddleware.extend(_middleware)
# Response middleware executed in reversed order
rmiddleware = list(reversed(rmiddleware))
handler = WsgiHandler(middleware, response_middleware=rmiddleware)
self.fire('on_loaded', handler)
return handler
def close_monitors(self):
'''Close all :class:`.Monitor` at once.
'''
return multi_async((m.stop() for m in list(itervalues(self.monitors))))
def available_connections(self):
'''Total number of available connections.'''
return reduce(lambda x, y: x + y, (p.available_connections for p in
itervalues(self.connection_pools)),
0)
def extend(self, elem):
'''Extend by adding *elem* attributes and children.'''
self._attributes.extend(elem._attributes)
for child_list in itervalues(elem._children):
for child in child_list:
child.set_parent(self)
def close_monitors(self):
'''Close all :class:`Monitor` at once.
'''
return multi_async((m.stop() for m in list(itervalues(self.monitors))))
def concurrent_connections(self):
'''Total number of concurrent connections.'''
return reduce(lambda x, y: x + y, (p.concurrent_connections for p in
itervalues(self.connection_pools)),
0)
def available_connections(self):
'''Total number of available connections.'''
return reduce(lambda x, y: x + y, (p.available_connections for p in
itervalues(self.connection_pools)),
0)
def execute_transaction(self, transaction):
'''Execute a ``transaction``
'''
transaction.commit()
return list(itervalues(transaction.identity_map))
def concurrent_connections(self):
'''Total number of concurrent connections.'''
return reduce(lambda x, y: x + y, (p.concurrent_connections for p in
itervalues(self.connection_pools)),
0)