def test_wait_jitter_default(self, mock_sleep, mock_random): """ verify default jitter is as expected """ mock_random.return_value = 1.0 w = Worker() x = w._sleeper() six.next(x) mock_random.assert_called_with(0, 5.0) mock_sleep.assert_called_with(2.0) mock_random.return_value = 3.3 six.next(x) mock_random.assert_called_with(0, 5.0) mock_sleep.assert_called_with(4.3)
def test_wait_jitter(self, mock_sleep, mock_random): """ verify configured jitter amount """ mock_random.return_value = 1.0 w = Worker() x = w._sleeper() six.next(x) mock_random.assert_called_with(0, 10.0) mock_sleep.assert_called_with(2.0) mock_random.return_value = 2.0 six.next(x) mock_random.assert_called_with(0, 10.0) mock_sleep.assert_called_with(3.0)
def run(self): """ Returns True if all scheduled tasks were executed successfully. """ logger.info('Running Worker with %d processes', self.worker_processes) sleeper = self._sleeper() self.run_succeeded = True self._add_worker() while True: while len(self._running_tasks) >= self.worker_processes > 0: logger.debug( '%d running tasks, waiting for next task to finish', len(self._running_tasks)) self._handle_next_task() get_work_response = self._get_work() if get_work_response.worker_state == WORKER_STATE_DISABLED: self._start_phasing_out() if get_work_response.task_id is None: if not self._stop_requesting_work: self._log_remote_tasks(get_work_response) if len(self._running_tasks) == 0: self._idle_since = self._idle_since or datetime.datetime.now( ) if self._keep_alive(get_work_response): six.next(sleeper) continue else: break else: self._handle_next_task() continue # task_id is not None: logger.debug("Pending tasks: %s", get_work_response.n_pending_tasks) self._run_task(get_work_response.task_id) while len(self._running_tasks): logger.debug('Shut down Worker, %d more tasks to go', len(self._running_tasks)) self._handle_next_task() return self.run_succeeded
def _run_get_new_deps(self): # set task callbacks before running for reporter_attr, task_attr in six.iteritems(self.forward_reporter_callbacks): setattr(self.task, task_attr, getattr(self.status_reporter, reporter_attr)) task_gen = self.task.run() # reset task callbacks for reporter_attr, task_attr in six.iteritems(self.forward_reporter_callbacks): setattr(self.task, task_attr, None) if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) if all(t.complete() for t in new_req): next_send = getpaths(requires) else: new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] return new_deps
def _run_get_new_deps(self): try: t0 = time.time() task_gen = self.task.run() finally: self.task.trigger_event( Event.PROCESSING_TIME, self.task, time.time() - t0) if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if all(t.complete() for t in new_req): next_send = getpaths(requires) else: return new_deps
def _run_get_new_deps(self): run_again = False try: task_gen = self.task.run(tracking_url_callback=self.tracking_url_callback) except TypeError as ex: if 'unexpected keyword argument' not in getattr(ex, 'message', ex.args[0]): raise run_again = True if run_again: task_gen = self.task.run() if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if all(t.complete() for t in new_req): next_send = getpaths(requires) else: return new_deps
def _run_get_new_deps(self): self.task.set_tracking_url = self.status_reporter.update_tracking_url self.task.set_status_message = self.status_reporter.update_status self.task.set_progress_percentage = self.status_reporter.update_progress_percentage task_gen = self.task.run() self.task.set_tracking_url = None self.task.set_status_message = None self.task.set_progress_percentage = None if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) if all(t.complete() for t in new_req): next_send = getpaths(requires) else: new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] return new_deps
def _run_get_new_deps(self): self.task.set_tracking_url = self.tracking_url_callback self.task.set_status_message = self.status_message_callback task_gen = self.task.run() self.task.set_tracking_url = None self.task.set_status_message = None if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) if all(t.complete() for t in new_req): next_send = getpaths(requires) else: new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] return new_deps
def _run_get_new_deps(self): try: task_gen = self.task.run( tracking_url_callback=self.tracking_url_callback) except TypeError as ex: if 'unexpected keyword argument' not in getattr( ex, 'message', ex.args[0]): raise task_gen = self.task.run() if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if all(t.complete() for t in new_req): next_send = getpaths(requires) else: return new_deps
def run(self): """ Returns True if all scheduled tasks were executed successfully. """ logger.info('Running Worker with %d processes', self.worker_processes) sleeper = self._sleeper() self.run_succeeded = True self._add_worker() while self._check_version(): while len(self._running_tasks) >= self.worker_processes > 0: logger.debug('%d running tasks, waiting for next task to finish', len(self._running_tasks)) self._handle_next_task() if not self._check_version(): break get_work_response = self._get_work() if get_work_response.worker_state == WORKER_STATE_DISABLED: self._start_phasing_out() if get_work_response.task_id is None: if not self._stop_requesting_work: self._log_remote_tasks(get_work_response) if len(self._running_tasks) == 0: if self._keep_alive(get_work_response): six.next(sleeper) continue else: break else: self._handle_next_task() continue # task_id is not None: logger.debug("Pending tasks: %s", get_work_response.n_pending_tasks) self._run_task(get_work_response.task_id) while len(self._running_tasks): logger.debug('Shut down Worker, %d more tasks to go', len(self._running_tasks)) self._handle_next_task() self._remove_worker() return self.run_succeeded
def run(self): """ Returns True if all scheduled tasks were executed successfully. """ logger.info('Running Worker with %d processes', self.worker_processes) sleeper = self._sleeper() self.run_succeeded = True self._add_worker() while True: while len(self._running_tasks) >= self.worker_processes: logger.debug( '%d running tasks, waiting for next task to finish', len(self._running_tasks)) self._handle_next_task() task_id, running_tasks, n_pending_tasks, n_unique_pending = self._get_work( ) if task_id is None: if not self._stop_requesting_work: self._log_remote_tasks(running_tasks, n_pending_tasks, n_unique_pending) if len(self._running_tasks) == 0: if self._keep_alive(n_pending_tasks, n_unique_pending): six.next(sleeper) continue else: break else: self._handle_next_task() continue # task_id is not None: logger.debug("Pending tasks: %s", n_pending_tasks) self._run_task(task_id) while len(self._running_tasks): logger.debug('Shut down Worker, %d more tasks to go', len(self._running_tasks)) self._handle_next_task() return self.run_succeeded
def __iter__(self): self.generator = self.readlines('\n') has_next = True while has_next: try: chunk = six.next(self.generator) yield chunk except StopIteration: has_next = False self.close()
def run(self): """ Returns True if all scheduled tasks were executed successfully. """ logger.info('Running Worker with %d processes', self.worker_processes) sleeper = self._sleeper() self.run_succeeded = True self._add_worker() while True: while len(self._running_tasks) >= self.worker_processes: logger.debug('%d running tasks, waiting for next task to finish', len(self._running_tasks)) self._handle_next_task() task_id, running_tasks, n_pending_tasks, n_unique_pending = self._get_work() if task_id is None: if not self._stop_requesting_work: self._log_remote_tasks(running_tasks, n_pending_tasks, n_unique_pending) if len(self._running_tasks) == 0: if self._keep_alive(n_pending_tasks, n_unique_pending): six.next(sleeper) continue else: break else: self._handle_next_task() continue # task_id is not None: logger.debug("Pending tasks: %s", n_pending_tasks) self._run_task(task_id) while len(self._running_tasks): logger.debug('Shut down Worker, %d more tasks to go', len(self._running_tasks)) self._handle_next_task() return self.run_succeeded
def _run_get_new_deps(self): self.task.set_tracking_url = self.tracking_url_callback self.task.set_status_message = self.status_message_callback def deprecated_tracking_url_callback(*args, **kwargs): warnings.warn( "tracking_url_callback in run() args is deprecated, use " "set_tracking_url instead.", DeprecationWarning) self.tracking_url_callback(*args, **kwargs) run_again = False try: task_gen = self.task.run( tracking_url_callback=deprecated_tracking_url_callback) except TypeError as ex: if 'unexpected keyword argument' not in str(ex): raise run_again = True if run_again: task_gen = self.task.run() self.task.set_tracking_url = None self.task.set_status_message = None if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if all(t.complete() for t in new_req): next_send = getpaths(requires) else: return new_deps
def _run_get_new_deps(self): self.task.set_tracking_url = self.tracking_url_callback self.task.set_status_message = self.status_message_callback def deprecated_tracking_url_callback(*args, **kwargs): warnings.warn("tracking_url_callback in run() args is deprecated, use " "set_tracking_url instead.", DeprecationWarning) self.tracking_url_callback(*args, **kwargs) run_again = False try: task_gen = self.task.run(tracking_url_callback=deprecated_tracking_url_callback) except TypeError as ex: if 'unexpected keyword argument' not in str(ex): raise run_again = True if run_again: task_gen = self.task.run() self.task.set_tracking_url = None self.task.set_status_message = None if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if all(t.complete() for t in new_req): next_send = getpaths(requires) else: return new_deps
def _run_get_new_deps(self): task_gen = self.task.run() if not isinstance(task_gen, types.GeneratorType): return None next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: return None new_req = flatten(requires) if all(t.complete() for t in new_req): next_send = getpaths(requires) else: new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] return new_deps
def run(self): logger.info('[pid %s] Worker %s running %s', os.getpid(), self.worker_id, self.task.task_id) if self.random_seed: # Need to have different random seeds if running in separate processes random.seed((os.getpid(), time.time())) status = FAILED error_message = '' missing = [] new_deps = [] try: # Verify that all the tasks are fulfilled! missing = [ dep.task_id for dep in self.task.deps() if not dep.complete() ] if missing: deps = 'dependency' if len(missing) == 1 else 'dependencies' raise RuntimeError('Unfulfilled %s at run time: %s' % (deps, ', '.join(missing))) self.task.trigger_event(Event.START, self.task) t0 = time.time() status = None try: task_gen = self.task.run() if isinstance(task_gen, types.GeneratorType): # new deps next_send = None while True: try: if next_send is None: requires = six.next(task_gen) else: requires = task_gen.send(next_send) except StopIteration: break new_req = flatten(requires) status = (RUNNING if all( t.complete() for t in new_req) else SUSPENDED) new_deps = [(t.task_module, t.task_family, t.to_str_params()) for t in new_req] if status == RUNNING: self.result_queue.put((self.task.task_id, status, '', missing, new_deps)) next_send = getpaths(requires) new_deps = [] else: logger.info( '[pid %s] Worker %s new requirements %s', os.getpid(), self.worker_id, self.task.task_id) return finally: if status != SUSPENDED: self.task.trigger_event(Event.PROCESSING_TIME, self.task, time.time() - t0) error_message = json.dumps(self.task.on_success()) logger.info('[pid %s] Worker %s done %s', os.getpid(), self.worker_id, self.task.task_id) self.task.trigger_event(Event.SUCCESS, self.task) status = DONE except KeyboardInterrupt: raise except BaseException as ex: status = FAILED logger.exception("[pid %s] Worker %s failed %s", os.getpid(), self.worker_id, self.task) error_message = notifications.wrap_traceback( self.task.on_failure(ex)) self.task.trigger_event(Event.FAILURE, self.task, ex) subject = "Luigi: %s FAILED" % self.task notifications.send_error_email(subject, error_message) finally: self.result_queue.put( (self.task.task_id, status, error_message, missing, new_deps))