def iteratorToProducer(iterator, consumer=None, wrapper=None): """ Makes an iterator into an L{IterationProducer}. Converts a possibly slow-running iterator into a Twisted-friendly producer, returning a deferred that fires with the producer when it's ready. If the the supplied object is not a suitable iterator (perhaps empty), the result will be C{None}. If a consumer is not supplied, whatever consumer gets this must register with the producer by calling its non-interface method L{IterationProducer.registerConsumer} and then its L{IterationProducer.run} method to start the iteration/production. If you supply a consumer, those two steps will be done automatically, and this method will fire with a C{Deferred} that fires when the iteration/production is done. """ result = None if Deferator.isIterator(iterator): pf = Prefetcherator() ok = yield pf.setup(iterator) if ok: if wrapper: if callable(wrapper): args = (wrapper, pf.getNext) else: result = Failure(TypeError( "Wrapper '{}' is not a callable".format( repr(wrapper)))) else: args = (pf.getNext,) dr = Deferator(repr(iterator), *args) result = IterationProducer(dr, consumer) if consumer: yield result.run() defer.returnValue(result)
def taskDone(self, statusResult, task, **kw): """ Processes the status/result tuple from a worker running a task. You don't need to call this directly. - B{e}: An B{e}xception was raised; the result is a pretty-printed traceback string. If the keyword 'returnFailure' was set for my constructor or this task, I will make it into a failure so the task's errback is triggered. - B{r}: The task B{r}an fine, the result is the return value of the call. - B{i}: Ran fine, but the result was an B{i}terable other than a standard Python one. So my result is a Deferator that yields deferreds to the worker's iterations, or, if you specified a consumer, an IterationProducer registered with the consumer that needs to get running to write iterations to it. If the iterator was empty, the result is just an empty list. - B{c}: Ran fine (on an AMP server), but the result is being B{c}hunked because it was too big for a single return value. So the result is a deferred that will eventually fire with the result after all the chunks of the return value have arrived and been magically pieced together and unpickled. - B{t}: The task B{t}imed out. I'll try to re-run it, once. - B{n}: The task returned [n]othing, as will I. - B{d}: The task B{d}idn't run, probably because there was a disconnection. I'll re-run it. """ @contextmanager def taskInfo(ID): if hasattr(self, 'logger'): if ID: taskInfo = self.info.aboutCall(ID) self.info.forgetID(ID) yield taskInfo else: # Why do logging without an info object? yield "TASK" else: yield None if self.spew: taskInfo += " -> {}".format(result) self.logger.info(taskInfo) def retryTask(): self.tasksBeingRetried.append(task) task.rush() self.q.put(task) return task.reset().addCallback(self.taskDone, task, **kw) status, result = statusResult # Deal with any info for this task call with taskInfo(kw.get('ID', None)) as prefix: if status == 'e': # There was an error... if prefix: # ...log it self.logger.error("{}: {}".format(prefix, result)) if kw.get('rf', False): # ...just return the Failure result = Failure(errors.WorkerError(result)) elif not self.warn: # ...stop the reactor import sys for msg in ("ERROR: {}".format(result), "Shutting down in one second!\n"): sys.stderr.write("\n{}".format(msg)) self._dc = reactor.callLater(1.0, reactor.stop) return result if status in "rc": # A plain result, or a deferred to a chunked one return result if status == 'i': # An iteration, possibly an IterationConsumer that we need # to run now if kw.get('consumer', None): if hasattr(result, 'run'): return result.run() # Nothing to produce from an empty iterator, consider # the iterations "done" right away. return defer.succeed(None) return result if status == 't': # Timed out. Try again, once. if task in self.tasksBeingRetried: self.tasksBeingRetried.remove(task) return Failure( errors.TimeoutError("Timed out after two tries, gave up")) return retryTask() if status == 'n': # None object return if status == 'd': # Didn't run. Try again, hopefully with a different worker. return retryTask() return Failure(errors.WorkerError( "Unknown status '{}'".format(status)))