Exemple #1
0
 def next_scheduled(self, jobnames=None):
     if not self.schedule_periodic:
         return
     if jobnames:
         entries = (self.entries.get(name, None) for name in jobnames)
     else:
         entries = itervalues(self.entries)
     next_entry = None
     next_time = None
     for entry in entries:
         if entry is None:
             continue
         is_due, next_time_to_run = entry.is_due()
         if is_due:
             next_time = 0
             next_entry = entry
             break
         elif next_time_to_run is not None:
             if next_time is None or next_time_to_run < next_time:
                 next_time = next_time_to_run
                 next_entry = entry
     if next_entry:
         return (next_entry.name, max(next_time, 0))
     else:
         return (jobnames, None)
Exemple #2
0
 def _info_monitor(self, actor, info=None):
     if actor.started():
         info['actor'].update({'concurrency': actor.cfg.concurrency,
                               'workers': len(self.managed_actors)})
         info['workers'] = [a.info for a in itervalues(self.managed_actors)
                            if a.info]
     return info
Exemple #3
0
    def periodic_task(self, actor, **kw):
        '''Override the :meth:`.Concurrency.periodic_task` to implement
        the :class:`.Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
        interval = 0
        actor.next_periodic_task = None
        #
        if actor.started():
            # managed actors job
            self.manage_actors(actor)
            for m in list(itervalues(self.monitors)):
                if m.closed():
                    actor._remove_actor(m)

            interval = MONITOR_TASK_PERIOD
            if not actor.is_running() and actor.cfg.debug:
                actor.logger.debug('still stopping')
            #
            actor.fire_event('periodic_task')

        if not actor.closed():
            actor.next_periodic_task = actor._loop.call_later(
                interval, self.periodic_task, actor)

        if actor.cfg.reload and autoreload.check_changes():
            actor.stop(exit_code=autoreload.EXIT_CODE)
Exemple #4
0
 def stream(self, whitespace=''):
     '''This function convert the :class:`css` element into a string.'''
     # First we execute mixins
     if self.rendered:
         raise RuntimeError('%s already rendered' % self)
     self.rendered = True
     children = self._children
     self._children = OrderedDict()
     for tag, clist in iteritems(children):
         for c in clist:
             c._parent = None
             s = c.set_parent(self)
             if s:   # the child (mixin) has return a string, added it.
                 yield (None, s)
     data = []
     for k, v in self._attributes:
         v = as_value(v)
         if v is not None:
             data.append('%s    %s: %s;' % (whitespace, k, v))
     if data:
         yield (self.tag, '\n'.join(data))
     # yield Mixins and children
     for child_list in itervalues(self._children):
         if isinstance(child_list, list):
             child = child_list[0]
             for c in child_list[1:]:
                 child.extend(c)
             for s in child.stream(whitespace):
                 yield s
         else:
             yield None, child_list
Exemple #5
0
 def next_scheduled(self, jobnames=None):
     if not self.schedule_periodic:
         return
     if jobnames:
         entries = (self.entries.get(name, None) for name in jobnames)
     else:
         entries = itervalues(self.entries)
     next_entry = None
     next_time = None
     for entry in entries:
         if entry is None:
             continue
         is_due, next_time_to_run = entry.is_due()
         if is_due:
             next_time = 0
             next_entry = entry
             break
         elif next_time_to_run is not None:
             if next_time is None or next_time_to_run < next_time:
                 next_time = next_time_to_run
                 next_entry = entry
     if next_entry:
         return (next_entry.name, max(next_time, 0))
     else:
         return (jobnames, None)
Exemple #6
0
def _channel_groups(actor, channel):
    for channel_re, group in itervalues(_get_pubsub_channels(actor)):
        if hasattr(channel_re, 'match'):
            g = channel_re.match(channel)
            if g:
                yield g.group(), group
        elif channel_re == channel:
            yield channel, group
Exemple #7
0
 def info(self):
     info = super(Monitor, self).info()
     if self.started():
         info['actor'].update({'concurrency': self.cfg.concurrency,
                               'workers': len(self.managed_actors)})
         info['workers'] = [a.info for a in itervalues(self.managed_actors)
                            if a.info]
     return info
Exemple #8
0
 def _close_all(self, actor):
     # Close al monitors at once
     try:
         for m in itervalues(self.monitors):
             yield From(m.stop())
         yield From(self._close_actors(actor))
     except Exception:
         actor.logger.exception('Exception while closing arbiter')
     self._exit_arbiter(actor, True)
Exemple #9
0
 def info(self):
     data = super(Arbiter, self).info()
     monitors = {}
     for m in itervalues(self.monitors):
         info = m.info()
         actor = info['actor']
         monitors[actor['name']] = info
     server = data.pop('actor')
     server.update({'version': pulsar.__version__,
                    'name': pulsar.SERVER_NAME,
                    'number_of_monitors': len(self.monitors),
                    'number_of_actors': len(self.managed_actors)})
     server.pop('is_process', None)
     server.pop('ppid', None)
     server.pop('actor_id', None)
     server.pop('age', None)
     data['server'] = server
     data['workers'] = [a.info for a in itervalues(self.managed_actors)]
     data['monitors'] = monitors
     return data
Exemple #10
0
 def info(self):
     info = super(Monitor, self).info()
     if self.started():
         info['actor'].update({
             'concurrency': self.cfg.concurrency,
             'workers': len(self.managed_actors)
         })
         info['workers'] = [
             a.info for a in itervalues(self.managed_actors) if a.info
         ]
     return info
Exemple #11
0
 def info(self):
     data = {'actor': {'actor_class': self.actor_class.__name__,
                       'concurrency': self.cfg.concurrency,
                       'name': self.name,
                       'age': self.impl.age,
                       'workers': len(self.managed_actors)}}
     if not self.started():
         return data
     data['workers'] = [a.info for a in itervalues(self.managed_actors)
                        if a.info]
     return data
Exemple #12
0
 def stop_actors(self):
     """Maintain the number of workers by spawning or killing as required
     """
     if self.cfg.workers:
         num_to_kill = len(self.managed_actors) - self.cfg.workers
         for i in range(num_to_kill, 0, -1):
             w, kage = 0, sys.maxsize
             for worker in itervalues(self.managed_actors):
                 age = worker.impl.age
                 if age < kage:
                     w, kage = w, age
             self.manage_actor(w, True)
Exemple #13
0
 def stop_actors(self, monitor):
     """Maintain the number of workers by spawning or killing as required
     """
     if monitor.cfg.workers:
         num_to_kill = len(self.managed_actors) - monitor.cfg.workers
         for i in range(num_to_kill, 0, -1):
             w, kage = 0, sys.maxsize
             for worker in itervalues(self.managed_actors):
                 age = worker.impl.age
                 if age < kage:
                     w, kage = w, age
             self.manage_actor(monitor, w, True)
Exemple #14
0
 def _info_monitor(self, actor, info=None):
     data = info
     monitors = {}
     for m in itervalues(self.monitors):
         info = m.info()
         if info:
             actor = info['actor']
             monitors[actor['name']] = info
     server = data.pop('actor')
     server.update({'version': pulsar.__version__,
                    'name': pulsar.SERVER_NAME,
                    'number_of_monitors': len(self.monitors),
                    'number_of_actors': len(self.managed_actors)})
     server.pop('is_process', None)
     server.pop('ppid', None)
     server.pop('actor_id', None)
     server.pop('age', None)
     data['server'] = server
     data['workers'] = [a.info for a in itervalues(self.managed_actors)]
     data['monitors'] = monitors
     return data
Exemple #15
0
 def tick(self, now=None):
     # Run a tick, that is one iteration of the scheduler.
     if not self.schedule_periodic:
         return
     remaining_times = []
     for entry in itervalues(self.entries):
         is_due, next_time_to_run = entry.is_due(now=now)
         if is_due:
             self.queue_task(entry.name)
         if next_time_to_run:
             remaining_times.append(next_time_to_run)
     self.next_run = now or time.time()
     if remaining_times:
         self.next_run += min(remaining_times)
Exemple #16
0
 def tick(self, now=None):
     # Run a tick, that is one iteration of the scheduler.
     if not self.schedule_periodic:
         return
     remaining_times = []
     for entry in itervalues(self.entries):
         is_due, next_time_to_run = entry.is_due(now=now)
         if is_due:
             self.queue_task(entry.name)
         if next_time_to_run:
             remaining_times.append(next_time_to_run)
     self.next_run = now or time.time()
     if remaining_times:
         self.next_run += min(remaining_times)
Exemple #17
0
 def periodic_task(self, actor, **kw):
     '''Override the :meth:`.Concurrency.periodic_task` to implement
     the :class:`.Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
     interval = 0
     actor.next_periodic_task = None
     if actor.is_running():
         # managed actors job
         interval = MONITOR_TASK_PERIOD
         actor.manage_actors()
         for m in list(itervalues(actor.monitors)):
             if m.started() and not m.is_running():
                 actor._remove_actor(m)
     actor.next_periodic_task = actor._loop.call_later(
         interval, self.periodic_task, actor)
Exemple #18
0
    def copy_many_times_events(self, other):
        '''Copy :ref:`many times events <many-times-event>` from  ``other``.

        All many times events of ``other`` are copied to this handler
        provided the events handlers already exist.
        '''
        if isinstance(other, EventHandler):
            events = self._events
            for event in itervalues(other._events):
                if isinstance(event, ManyEvent):
                    ev = events.get(event.name)
                    # If the event is available add it
                    if ev:
                        for callback in event._handlers:
                            ev.bind(callback)
Exemple #19
0
 def get_actor(self, aid):
     '''Given an actor unique id return the actor proxy.'''
     a = super(Arbiter, self).get_actor(aid)
     if a is None:
         if aid in self.monitors:  # Check in monitors aid
             return self.monitors[aid]
         elif aid in self.managed_actors:
             return self.managed_actors[aid]
         elif aid in self.registered:
             return self.registered[aid]
         else:  # Finally check in workers in monitors
             for m in itervalues(self.monitors):
                 if aid in m.managed_actors:
                     return m.managed_actors[aid]
     else:
         return a
Exemple #20
0
 def get_actor(self, aid):
     '''Given an actor unique id return the actor proxy.'''
     a = super(Arbiter, self).get_actor(aid)
     if a is None:
         if aid in self.monitors:  # Check in monitors aid
             return self.monitors[aid]
         elif aid in self.managed_actors:
             return self.managed_actors[aid]
         elif aid in self.registered:
             return self.registered[aid]
         else:  # Finally check in workers in monitors
             for m in itervalues(self.monitors):
                 if aid in m.managed_actors:
                     return m.managed_actors[aid]
     else:
         return a
Exemple #21
0
    def middleware(self, app):
        '''Build the API middleware.

        If :setting:`API_URL` is defined, it loops through all extensions
        and checks if the ``api_sections`` method is available.
        '''
        middleware = []
        url = app.config['API_URL']
        if url:
            app.config['API_URL'] = url = remove_double_slash('/%s/' % url)
            sections = {}
            self.api = api = Api(url, sections=sections)
            middleware.append(api)
            docs = None
            url = app.config['API_DOCS_URL']
            if url:
                app.config['API_DOCS_URL'] = url = remove_double_slash('/%s/' %
                                                                       url)
                self.docs = Router(url, sections=sections)
                middleware.append(self.docs)
            #
            for extension in itervalues(app.extensions):
                api_sections = getattr(extension, 'api_sections', None)
                if api_sections:
                    for name, routers in api_sections(app):
                        # Routes must be instances of CRUD
                        # name is the section name
                        if name not in sections:
                            sections[name] = ApiSection()
                        section = sections[name]
                        for router in routers:
                            api.add_child(router)
                            section.append(router)
                            manager = router.manager
                            self.api_crud_routers[manager] = manager
        url = app.config['ADMIN_URL']
        if url:
            app.config['ADMIN_URL'] = url = remove_double_slash('/%s/' % url)
            sections = {}
            admin = Admin(url, sections=sections)
            middleware.append(admin)
            #for extension in itervalues(app.extensions):
            #    api_sections = getattr(extension, 'api_sections', None)
            #    if api_sections:
            #        pass
        return middleware
Exemple #22
0
    def middleware(self, app):
        '''Build the API middleware.

        If :setting:`API_URL` is defined, it loops through all extensions
        and checks if the ``api_sections`` method is available.
        '''
        middleware = []
        url = app.config['API_URL']
        if url:
            app.config['API_URL'] = url = remove_double_slash('/%s/' % url)
            sections = {}
            self.api = api = Api(url, sections=sections)
            middleware.append(api)
            docs = None
            url = app.config['API_DOCS_URL']
            if url:
                app.config['API_DOCS_URL'] = url = remove_double_slash('/%s/'
                                                                       % url)
                self.docs = Router(url, sections=sections)
                middleware.append(self.docs)
            #
            for extension in itervalues(app.extensions):
                api_sections = getattr(extension, 'api_sections', None)
                if api_sections:
                    for name, routers in api_sections(app):
                        # Routes must be instances of CRUD
                        # name is the section name
                        if name not in sections:
                            sections[name] = ApiSection()
                        section = sections[name]
                        for router in routers:
                            api.add_child(router)
                            section.append(router)
                            manager = router.manager
                            self.api_crud_routers[manager] = manager
        url = app.config['ADMIN_URL']
        if url:
            app.config['ADMIN_URL'] = url = remove_double_slash('/%s/' % url)
            sections = {}
            admin = Admin(url, sections=sections)
            middleware.append(admin)
            #for extension in itervalues(app.extensions):
            #    api_sections = getattr(extension, 'api_sections', None)
            #    if api_sections:
            #        pass
        return middleware
Exemple #23
0
 def get_actor(self, actor, aid, check_monitor=True):
     '''Given an actor unique id return the actor proxy.'''
     a = super(ArbiterConcurrency, self).get_actor(actor, aid)
     if a is None:
         if aid in self.monitors:  # Check in monitors aid
             return self.monitors[aid]
         elif aid in self.managed_actors:
             return self.managed_actors[aid]
         elif aid in self.registered:
             return self.registered[aid]
         else:  # Finally check in workers in monitors
             for m in itervalues(self.monitors):
                 a = m.get_actor(aid, check_monitor=False)
                 if a is not None:
                     return a
     else:
         return a
Exemple #24
0
 def periodic_task(self, actor):
     '''Override the :meth:`Concurrency.periodic_task` to implement
     the :class:`Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
     interval = 0
     actor.next_periodic_task = None
     if actor.is_running():
         # managed actors job
         interval = MONITOR_TASK_PERIOD
         actor.manage_actors()
         for m in list(itervalues(actor.monitors)):
             if m.started():
                 if not m.is_running():
                     actor._remove_actor(m)
             else:
                 m.start()
     actor.next_periodic_task = actor.event_loop.call_later(
         interval, self.periodic_task, actor)
     return actor
Exemple #25
0
 def get_tasks(self, **filters):
     tasks = []
     if filters:
         fs = []
         for name, value in filters.items():
             if not isinstance(value, (list, tuple, set, frozenset)):
                 value = (value,)
             fs.append((name, value))
         # Loop over tasks
         for task in itervalues(self._tasks):
             select = True
             for name, values in fs:
                 value = getattr(task, name, None)
                 if value not in values:
                     select = False
                     break
             if select:
                 tasks.append(task)
     return tasks
Exemple #26
0
 def get_tasks(self, **filters):
     tasks = []
     if filters:
         fs = []
         for name, value in filters.items():
             if not isinstance(value, (list, tuple, set, frozenset)):
                 value = (value, )
             fs.append((name, value))
         # Loop over tasks
         for task in itervalues(self._tasks):
             select = True
             for name, values in fs:
                 value = getattr(task, name, None)
                 if value not in values:
                     select = False
                     break
             if select:
                 tasks.append(task)
     return tasks
Exemple #27
0
    def tick(self, now=None):
        '''Run a tick, that is one iteration of the scheduler. This
method only works when :attr:`schedule_periodic` is ``True`` and
the arbiter context.

Executes all due tasks and calculate the time in seconds to wait before
running a new :meth:`tick`. For testing purposes a :class:`datetime.datetime`
value ``now`` can be passed.'''
        if not self.schedule_periodic:
            return
        remaining_times = []
        try:
            for entry in itervalues(self.entries):
                is_due, next_time_to_run = entry.is_due(now=now)
                if is_due:
                    self.run_job(entry.name)
                if next_time_to_run:
                    remaining_times.append(next_time_to_run)
        except Exception:
            LOGGER.exception('Unhandled error in task backend')
        self.next_run = now or datetime.now()
        if remaining_times:
            self.next_run += timedelta(seconds=min(remaining_times))
Exemple #28
0
    def handler(self):
        '''The WSGI application handler for this :class:`App`.

        It is lazily loaded the first time it is accessed so that
        this :class:`App` can be used by pulsar in a multiprocessing setup.
        '''
        # do this here so that the config is already loaded before fire signal
        extensions = list(itervalues(self.extensions))
        self.fire('on_config')
        middleware = []
        rmiddleware = []
        for extension in extensions:
            _middleware = extension.middleware(self)
            if _middleware:
                middleware.extend(_middleware)
            _middleware = extension.response_middleware(self)
            if _middleware:
                rmiddleware.extend(_middleware)
        # Response middleware executed in reversed order
        rmiddleware = list(reversed(rmiddleware))
        handler = WsgiHandler(middleware, response_middleware=rmiddleware)
        self.fire('on_loaded', handler)
        return handler
Exemple #29
0
    def tick(self, now=None):
        '''Run a tick, that is one iteration of the scheduler. This
method only works when :attr:`schedule_periodic` is ``True`` and
the arbiter context.

Executes all due tasks and calculate the time in seconds to wait before
running a new :meth:`tick`. For testing purposes a :class:`datetime.datetime`
value ``now`` can be passed.'''
        if not self.schedule_periodic:
            return
        remaining_times = []
        try:
            for entry in itervalues(self.entries):
                is_due, next_time_to_run = entry.is_due(now=now)
                if is_due:
                    self.run_job(entry.name)
                if next_time_to_run:
                    remaining_times.append(next_time_to_run)
        except Exception:
            LOGGER.exception('Unhandled error in task backend')
        self.next_run = now or datetime.now()
        if remaining_times:
            self.next_run += timedelta(seconds=min(remaining_times))
Exemple #30
0
    def handler(self):
        '''The WSGI application handler for this :class:`App`.

        It is lazily loaded the first time it is accessed so that
        this :class:`App` can be used by pulsar in a multiprocessing setup.
        '''
        # do this here so that the config is already loaded before fire signal
        extensions = list(itervalues(self.extensions))
        self.fire('on_config')
        middleware = []
        rmiddleware = []
        for extension in extensions:
            _middleware = extension.middleware(self)
            if _middleware:
                middleware.extend(_middleware)
            _middleware = extension.response_middleware(self)
            if _middleware:
                rmiddleware.extend(_middleware)
        # Response middleware executed in reversed order
        rmiddleware = list(reversed(rmiddleware))
        handler = WsgiHandler(middleware, response_middleware=rmiddleware)
        self.fire('on_loaded', handler)
        return handler
Exemple #31
0
 def close_monitors(self):
     '''Close all :class:`.Monitor` at once.
     '''
     return multi_async((m.stop() for m in list(itervalues(self.monitors))))
Exemple #32
0
 def available_connections(self):
     '''Total number of available connections.'''
     return reduce(lambda x, y: x + y, (p.available_connections for p in
                                        itervalues(self.connection_pools)),
                   0)
Exemple #33
0
 def extend(self, elem):
     '''Extend by adding *elem* attributes and children.'''
     self._attributes.extend(elem._attributes)
     for child_list in itervalues(elem._children):
         for child in child_list:
             child.set_parent(self)
Exemple #34
0
 def close_monitors(self):
     '''Close all :class:`Monitor` at once.
     '''
     return multi_async((m.stop() for m in list(itervalues(self.monitors))))
Exemple #35
0
 def concurrent_connections(self):
     '''Total number of concurrent connections.'''
     return reduce(lambda x, y: x + y, (p.concurrent_connections for p in
                                        itervalues(self.connection_pools)),
                   0)
Exemple #36
0
 def available_connections(self):
     '''Total number of available connections.'''
     return reduce(lambda x, y: x + y, (p.available_connections for p in
                                        itervalues(self.connection_pools)),
                   0)
Exemple #37
0
 def execute_transaction(self, transaction):
     '''Execute a ``transaction``
     '''
     transaction.commit()
     return list(itervalues(transaction.identity_map))
Exemple #38
0
 def concurrent_connections(self):
     '''Total number of concurrent connections.'''
     return reduce(lambda x, y: x + y, (p.concurrent_connections for p in
                                        itervalues(self.connection_pools)),
                   0)