def _run_until_dead(self, executor, max_dispatches=None): total_dispatched = 0 if max_dispatches is None: # NOTE(TheSriram): if max_dispatches is not set, # then the conductor will run indefinitely, and not # stop after 'n' number of dispatches max_dispatches = -1 dispatch_gen = iter_utils.iter_forever(max_dispatches) is_stopped = self._wait_timeout.is_stopped try: # Don't even do any work in the first place... if max_dispatches == 0: raise StopIteration fresh_period = timeutils.StopWatch( duration=self.REFRESH_PERIODICITY) fresh_period.start() while not is_stopped(): any_dispatched = False if fresh_period.expired(): ensure_fresh = True fresh_period.restart() else: ensure_fresh = False job_it = itertools.takewhile( self._can_claim_more_jobs, self._jobboard.iterjobs(ensure_fresh=ensure_fresh)) for job in job_it: self._log.debug("Trying to claim job: %s", job) try: self._jobboard.claim(job, self._name) except (excp.UnclaimableJob, excp.NotFound): self._log.debug( "Job already claimed or" " consumed: %s", job) else: try: fut = executor.submit(self._dispatch_job, job) except RuntimeError: with excutils.save_and_reraise_exception(): self._log.warn( "Job dispatch submitting" " failed: %s", job) self._try_finish_job(job, False) else: fut.job = job self._dispatched.add(fut) any_dispatched = True fut.add_done_callback( functools.partial(self._on_job_done, job)) total_dispatched = next(dispatch_gen) if not any_dispatched and not is_stopped(): self._wait_timeout.wait() except StopIteration: # This will be raised from 'dispatch_gen' if it reaches its # max dispatch number (which implies we should do no more work). with excutils.save_and_reraise_exception(): if max_dispatches >= 0 and total_dispatched >= max_dispatches: self._log.info("Maximum dispatch limit of %s reached", max_dispatches)
def _run_until_dead(self, executor, max_dispatches=None): total_dispatched = 0 if max_dispatches is None: # NOTE(TheSriram): if max_dispatches is not set, # then the conductor will run indefinitely, and not # stop after 'n' number of dispatches max_dispatches = -1 dispatch_gen = iter_utils.iter_forever(max_dispatches) is_stopped = self._wait_timeout.is_stopped try: # Don't even do any work in the first place... if max_dispatches == 0: raise StopIteration fresh_period = timeutils.StopWatch( duration=self.REFRESH_PERIODICITY) fresh_period.start() while not is_stopped(): any_dispatched = False if fresh_period.expired(): ensure_fresh = True fresh_period.restart() else: ensure_fresh = False job_it = itertools.takewhile( self._can_claim_more_jobs, self._jobboard.iterjobs(ensure_fresh=ensure_fresh)) for job in job_it: self._log.debug("Trying to claim job: %s", job) try: self._jobboard.claim(job, self._name) except (excp.UnclaimableJob, excp.NotFound): self._log.debug("Job already claimed or" " consumed: %s", job) else: try: fut = executor.submit(self._dispatch_job, job) except RuntimeError: with excutils.save_and_reraise_exception(): self._log.warn("Job dispatch submitting" " failed: %s", job) self._try_finish_job(job, False) else: fut.job = job self._dispatched.add(fut) any_dispatched = True fut.add_done_callback( functools.partial(self._on_job_done, job)) total_dispatched = next(dispatch_gen) if not any_dispatched and not is_stopped(): self._wait_timeout.wait() except StopIteration: # This will be raised from 'dispatch_gen' if it reaches its # max dispatch number (which implies we should do no more work). with excutils.save_and_reraise_exception(): if max_dispatches >= 0 and total_dispatched >= max_dispatches: self._log.info("Maximum dispatch limit of %s reached", max_dispatches)
def __init__(self, auth_key, dispatch_func, msg_limit=-1): if not six.callable(dispatch_func): raise ValueError("Expected provided dispatch function" " to be callable") self.auth_key = auth_key self.dispatch_func = dispatch_func msg_limiter = iter_utils.iter_forever(msg_limit) self.msg_count = six.next(msg_limiter) self._msg_limiter = msg_limiter self._buffer = misc.BytesIO() self._state = None # Local machine variables and such are stored in here. self._memory = {} self._transitions = collections.deque(self._TRANSITIONS) # This is the per state callback handler set. The first entry reads # the data and the second entry is called after reading is completed, # typically to save that data into object memory, or to validate # it. self._handlers = { 'magic_header_left': (self._read_field_data, self._save_and_validate_magic), 'mac_header_left': (self._read_field_data, functools.partial(self._save_pos_integer, 'mac_left')), 'mac_left': (functools.partial(self._read_data, 'mac'), functools.partial(self._save_data, 'mac')), 'identity_header_left': (self._read_field_data, functools.partial(self._save_pos_integer, 'identity_left')), 'identity_left': (functools.partial(self._read_data, 'identity'), functools.partial(self._save_data, 'identity')), 'msg_header_left': (self._read_field_data, functools.partial(self._save_pos_integer, 'msg_left')), 'msg_left': (functools.partial(self._read_data, 'msg'), self._dispatch_and_reset), } # Force transition into first state... self._transition()
def run(self, max_dispatches=None): self._dead.clear() # Register a conductor type entity self._jobboard.register_entity(self._get_conductor_info()) total_dispatched = 0 try: if max_dispatches is None: # NOTE(TheSriram): if max_dispatches is not set, # then the conductor will run indefinitely, and not # stop after 'n' number of dispatches max_dispatches = -1 dispatch_gen = iter_utils.iter_forever(max_dispatches) while True: if self._wait_timeout.is_stopped(): break local_dispatched = 0 for job in self._jobboard.iterjobs(): if self._wait_timeout.is_stopped(): break LOG.debug("Trying to claim job: %s", job) try: self._jobboard.claim(job, self._name) except (excp.UnclaimableJob, excp.NotFound): LOG.debug("Job already claimed or consumed: %s", job) continue consume = False try: f = self._dispatch_job(job) except KeyboardInterrupt: with excutils.save_and_reraise_exception(): LOG.warn("Job dispatching interrupted: %s", job) except Exception: LOG.warn("Job dispatching failed: %s", job, exc_info=True) else: local_dispatched += 1 consume = f.result() try: if consume: self._jobboard.consume(job, self._name) else: self._jobboard.abandon(job, self._name) except (excp.JobFailure, excp.NotFound): if consume: LOG.warn("Failed job consumption: %s", job, exc_info=True) else: LOG.warn("Failed job abandonment: %s", job, exc_info=True) total_dispatched = next(dispatch_gen) if local_dispatched == 0 and \ not self._wait_timeout.is_stopped(): self._wait_timeout.wait() except StopIteration: if max_dispatches >= 0 and total_dispatched >= max_dispatches: LOG.info("Maximum dispatch limit of %s reached", max_dispatches) finally: self._dead.set()