def iteratorToProducer(iterator, consumer=None, wrapper=None):
"""
Makes an iterator into an L{IterationProducer}.
Converts a possibly slow-running iterator into a Twisted-friendly
producer, returning a deferred that fires with the producer when
it's ready. If the the supplied object is not a suitable iterator
(perhaps empty), the result will be C{None}.
If a consumer is not supplied, whatever consumer gets this must
register with the producer by calling its non-interface method
L{IterationProducer.registerConsumer} and then its
L{IterationProducer.run} method to start the iteration/production.
If you supply a consumer, those two steps will be done
automatically, and this method will fire with a C{Deferred} that
fires when the iteration/production is done.
"""
result = None
if Deferator.isIterator(iterator):
pf = Prefetcherator()
ok = yield pf.setup(iterator)
if ok:
if wrapper:
if callable(wrapper):
args = (wrapper, pf.getNext)
else:
result = Failure(TypeError(
"Wrapper '{}' is not a callable".format(
repr(wrapper))))
else:
args = (pf.getNext,)
dr = Deferator(repr(iterator), *args)
result = IterationProducer(dr, consumer)
if consumer:
yield result.run()
defer.returnValue(result)
def taskDone(self, statusResult, task, **kw):
"""
Processes the status/result tuple from a worker running a
task. You don't need to call this directly.
- B{e}: An B{e}xception was raised; the result is a
pretty-printed traceback string. If the keyword
'returnFailure' was set for my constructor or this task, I
will make it into a failure so the task's errback is
triggered.
- B{r}: The task B{r}an fine, the result is the return value
of the call.
- B{i}: Ran fine, but the result was an B{i}terable other
than a standard Python one. So my result is a Deferator
that yields deferreds to the worker's iterations, or, if
you specified a consumer, an IterationProducer registered
with the consumer that needs to get running to write
iterations to it. If the iterator was empty, the result is
just an empty list.
- B{c}: Ran fine (on an AMP server), but the result is being
B{c}hunked because it was too big for a single return
value. So the result is a deferred that will eventually
fire with the result after all the chunks of the return
value have arrived and been magically pieced together and
unpickled.
- B{t}: The task B{t}imed out. I'll try to re-run it, once.
- B{n}: The task returned [n]othing, as will I.
- B{d}: The task B{d}idn't run, probably because there was a
disconnection. I'll re-run it.
"""
@contextmanager
def taskInfo(ID):
if hasattr(self, 'logger'):
if ID:
taskInfo = self.info.aboutCall(ID)
self.info.forgetID(ID)
yield taskInfo
else:
# Why do logging without an info object?
yield "TASK"
else:
yield None
if self.spew:
taskInfo += " -> {}".format(result)
self.logger.info(taskInfo)
def retryTask():
self.tasksBeingRetried.append(task)
task.rush()
self.q.put(task)
return task.reset().addCallback(self.taskDone, task, **kw)
status, result = statusResult
# Deal with any info for this task call
with taskInfo(kw.get('ID', None)) as prefix:
if status == 'e':
# There was an error...
if prefix:
# ...log it
self.logger.error("{}: {}".format(prefix, result))
if kw.get('rf', False):
# ...just return the Failure
result = Failure(errors.WorkerError(result))
elif not self.warn:
# ...stop the reactor
import sys
for msg in ("ERROR: {}".format(result),
"Shutting down in one second!\n"):
sys.stderr.write("\n{}".format(msg))
self._dc = reactor.callLater(1.0, reactor.stop)
return result
if status in "rc":
# A plain result, or a deferred to a chunked one
return result
if status == 'i':
# An iteration, possibly an IterationConsumer that we need
# to run now
if kw.get('consumer', None):
if hasattr(result, 'run'):
return result.run()
# Nothing to produce from an empty iterator, consider
# the iterations "done" right away.
return defer.succeed(None)
return result
if status == 't':
# Timed out. Try again, once.
if task in self.tasksBeingRetried:
self.tasksBeingRetried.remove(task)
return Failure(
errors.TimeoutError("Timed out after two tries, gave up"))
return retryTask()
if status == 'n':
# None object
return
if status == 'd':
# Didn't run. Try again, hopefully with a different worker.
return retryTask()
return Failure(errors.WorkerError(
"Unknown status '{}'".format(status)))
