def getResult(self, targets, i=None):
log.msg("getResult on %s" % targets)
engines = self.engineList(targets)
l = []
for e in engines:
l.append(e.getResult(i))
return gatherBoth(l)
def run(self, task):
"""Run task in worker's namespace.
:Parameters:
task : a `Task` object
:Returns: `Deferred` to the result of the task in the form of a `dict`,
{resultName:result}. If no `resultNames`, then returns {'result':None} on
success.
"""
dl = []
index = 1
if task.clearBefore:
dl.append(self.queuedEngine.reset())
index += 1
if task.setupNS:
dl.append(self.queuedEngine.push(**task.setupNS))
index += 1
dl.append(self.queuedEngine.execute(task.expression))
if task.resultNames:
d = self.queuedEngine.pull(*task.resultNames)
else:
d = defer.succeed(None)
dl.append(d)
if task.clearAfter:
dl.append(self.queuedEngine.reset())
names = task.resultNames or ['result']
d = gatherBoth(dl, consumeErrors=True)
return d.addBoth(self._zipResults, names, index)
def pullSerialized(self, *keys):
msg = """engine %r
method: pullSerialized(*keys)
keys = %r""" % (self.id, keys)
if len(keys) > 1:
pulledDeferreds = []
for key in keys:
d = self.executeAndRaise(msg, self.shell.pull,key)
pulledDeferreds.append(d)
# This will fire on the first failure and log the rest.
dList = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=0,
consumeErrors=1)
@dList.addCallback
def packThemUp(values):
serials = []
for v in values:
try:
serials = newserialized.serialize(v)
except:
return defer.fail(failure.Failure())
return dict(zip(keys, values))
return packThemUp
else:
key = keys[0]
d = self.executeAndRaise(msg, self.shell.pull, key)
d.addCallback(newserialized.serialize)
return d
def status(self, targets):
log.msg("retrieving status of %s" % targets)
engines = self.engineList(targets)
l = []
for e in engines:
l.append(e.status().addCallback(lambda s: (e.id, s)))
return gatherBoth(l)
def pull(self, *keys):
if len(keys) > 1:
pulledDeferreds = []
for key in keys:
pulledDeferreds.append(defer.execute(self.shell.get, key))
d = gatherBoth(pulledDeferreds)
return d
else:
return defer.execute(self.shell.get, keys[0])
def pushSerializedNew(self, **namespace):
dList = []
for k, v in namespace.iteritems():
d = self.pushPagingSerialized(k, v)
dList.append(d)
return gatherBoth(dList,
fireOnOneErrback=1,
logErrors=0,
consumeErrors=1)
def getAllPendingDeferreds(self, clientID):
dList = []
keys = self.pdManagers[clientID].pendingDeferreds.keys()
for k in keys:
dList.append(self.pdManagers[clientID].getPendingDeferred(
k, block=True))
if len(dList) > 0:
return gatherBoth(dList, consumeErrors=1)
else:
return defer.succeed([None])
def _performOnEnginesAndGatherBoth(self, methodName, targets, *args, **kwargs):
"""Called _performOnEngines and wraps result/exception into deferred."""
try:
dList = self._performOnEngines(methodName, targets, *args, **kwargs)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def execute(self, targets, lines):
if len(lines) > 64:
linestr = lines[:61] + '...'
else:
linestr = lines
log.msg("executing %s on %s" % (linestr, targets))
engines = self.engineList(targets)
l = []
for e in engines:
d = e.execute(lines).addCallback(self.notify)
l.append(d)
return gatherBoth(l)
def pullNew(self, *keys):
if len(keys) == 1:
return self.pullPaging(keys[0])
else:
pulledDeferreds = []
for k in keys:
pulledDeferreds.append(self.pullPaging(k))
d = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=0,
consumeErrors=1)
return d
def pullSerialized(self, targets, *keys):
try:
dList = self._performOnEngines('pullSerialized', targets, *keys)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
for d in dList:
d.addCallback(self._logSizes)
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def pushSerialized(self, targets, **namespace):
log.msg("pushing Serialized to %s" % targets)
engines = self.engineList(targets)
l = []
# Call unpack on values that aren't registered as allowed Serialized types
for k, v in namespace.iteritems():
if not isinstance(v, self.serialTypes) and isinstance(
v, Serialized):
log.msg("unpacking serial, ", k)
namespace[k] = v.unpack()
for e in engines:
l.append(e.pushSerialized(**namespace))
return gatherBoth(l)
def kill(self, targets, controller=False):
log.msg("killing engines %s" % targets)
if controller:
targets = 'all' # kill all engines if killing controller
engines = self.engineList(targets)
l = []
for e in engines:
l.append(e.kill())
d = gatherBoth(l)
if controller:
log.msg("Killing controller")
reactor.callLater(2.0, reactor.stop)
return d
def gather(self, targets, key, style='basic'):
"""gather a distributed object, and reassemble it"""
log.msg("gathering %s from %s" % (key, targets))
engines = self.engineList(targets)
nEngines = len(engines)
l = []
for e in engines:
l.append(e.pull(key))
mapClass = Map.styles[style]
mapObject = mapClass()
return gatherBoth(l).addCallback(mapObject.joinPartitions)
def pull(self, *keys):
if len(keys) > 1:
pulledDeferreds = []
for key in keys:
pulledDeferreds.append(defer.execute(self.shell.get, key))
# This will fire on the first failure and log the rest.
d = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=1,
consumeErrors=1)
return d
else:
return defer.execute(self.shell.get, keys[0])
def queueStatus(self, targets):
log.msg("Getting queue status on %s" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = []
for e in engines:
dList.append(e.queueStatus().addCallback(lambda s:(e.id, s)))
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def scatter(self, targets, key, seq, style='basic', flatten=False):
log.msg("scattering %s to %s" % (key, targets))
engines = self.engineList(targets)
nEngines = len(engines)
mapClass = Map.styles[style]
mapObject = mapClass()
l = []
for index, engine in enumerate(engines):
partition = mapObject.getPartition(seq, index, nEngines)
if flatten and len(partition) == 1:
l.append(engine.push(**{key: partition[0]}))
else:
l.append(engine.push(**{key: partition}))
return gatherBoth(l)
def pullSerialized(self, *keys):
if len(keys) > 1:
pulledDeferreds = []
for key in keys:
d = defer.execute(self.shell.get, key)
d.addCallback(serialized.serialize, key)
d.addErrback(self.handlePullProblems)
pulledDeferreds.append(d)
dList = gatherBoth(pulledDeferreds)
return dList
else:
key = keys[0]
d = defer.execute(self.shell.get, key)
d.addCallback(serialized.serialize, key)
d.addErrback(self.handlePullProblems)
return d
def pull(self, *keys):
msg = """engine %r
method: pull(*keys)
keys = %r""" % (self.id, keys)
if len(keys) > 1:
pulledDeferreds = []
for key in keys:
d = self.executeAndRaise(msg, self.shell.pull, key)
pulledDeferreds.append(d)
# This will fire on the first failure and log the rest.
dTotal = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=1,
consumeErrors=1)
return dTotal
else:
return self.executeAndRaise(msg, self.shell.pull, keys[0])
def gather(self, targets, key, style='basic'):
"""gather a distributed object, and reassemble it"""
log.msg("Gathering %s from %s" % (key, targets))
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
nEngines = len(engines)
dList = []
for e in engines:
dList.append(e.pull(key))
mapClass = Map.styles[style]
mapObject = mapClass()
d = gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
return d.addCallback(mapObject.joinPartitions)
def scatter(self, targets, key, seq, style='basic', flatten=False):
log.msg("Scattering %s to %s" % (key, targets))
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
nEngines = len(engines)
mapClass = Map.styles[style]
mapObject = mapClass()
dList = []
for index, engine in enumerate(engines):
partition = mapObject.getPartition(seq, index, nEngines)
if flatten and len(partition) == 1:
dList.append(engine.push(**{key: partition[0]}))
else:
dList.append(engine.push(**{key: partition}))
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def blockOn(deferrable,
fireOnOneCallback=0,
fireOnOneErrback=0,
consumeErrors=0):
"""Make a Deferred look synchronous.
Given a Deferrable object, this will run the Twisted event look until
the Deferred's callback and errback chains have run. It will then
return the actual result or raise an exception if an error occured.
>>> blockOn(functionReturningDeferred())
10
You can also pass a list of Deferreds to this function and you will
get a list of results.
>>> blockOn([d0, d1, d2])
['this', 'is', 'heresy']
"""
if not isinstance(deferrable, list):
deferrable = [deferrable]
# Add a check to simply pass through plain objects.
for i in range(len(deferrable)):
if hasattr(deferrable[i], '__defer__'):
deferrable[i] = deferrable[i].__defer__()
d = gatherBoth(deferrable,
fireOnOneCallback,
fireOnOneErrback,
consumeErrors,
logErrors=0)
if not fireOnOneCallback:
d.addCallback(_parseResults)
bd = BlockingDeferred(d)
return bd.blockOn()