def pullSerialized(self, *keys):
msg = """engine %r
method: pullSerialized(*keys)
keys = %r""" % (self.id, keys)
if len(keys) == 1:
key = keys[0]
d = self.executeAndRaise(msg, self.shell.pull, key)
d.addCallback(newserialized.serialize)
return d
elif len(keys) > 1:
pulledDeferreds = []
for key in keys:
d = self.executeAndRaise(msg, self.shell.pull, key)
pulledDeferreds.append(d)
# This will fire on the first failure and log the rest.
dList = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=0,
consumeErrors=1)
@dList.addCallback
def packThemUp(values):
serials = []
for v in values:
try:
serials.append(newserialized.serialize(v))
except:
return defer.fail(failure.Failure())
return dict(zip(keys, serials))
return packThemUp
def pushSerializedNew(self, **namespace):
dList = []
for k, v in namespace.iteritems():
d = self.pushPagingSerialized(k, v)
dList.append(d)
return gatherBoth(dList,
fireOnOneErrback=1,
logErrors=0,
consumeErrors=1)
def process_did_list(did_list):
"""Turn a list of deferred_ids into a final result or failure."""
new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
final_d = gatherBoth(new_d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
final_d.addCallback(error.collect_exceptions, 'scatter')
final_d.addCallback(lambda lop: [i[0] for i in lop])
return final_d
def getAllPendingDeferreds(self, clientID):
dList = []
keys = self.pdManagers[clientID].pendingDeferreds.keys()
for k in keys:
dList.append(self.pdManagers[clientID].getPendingDeferred(
k, block=True))
if len(dList) > 0:
return gatherBoth(dList, consumeErrors=1)
else:
return defer.succeed([None])
def do_scatter(engines):
nEngines = len(engines)
mapClass = Map.styles[style]
mapObject = mapClass()
d_list = []
# Loop through and push to each engine in non-blocking mode.
# This returns a set of deferreds to deferred_ids
for index, engineid in enumerate(engines):
partition = mapObject.getPartition(seq, index, nEngines)
if flatten and len(partition) == 1:
d = self.push({key: partition[0]}, targets=engineid, block=False)
else:
d = self.push({key: partition}, targets=engineid, block=False)
d_list.append(d)
# Collect the deferred to deferred_ids
d = gatherBoth(d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
# Now d has a list of deferred_ids or Failures coming
d.addCallback(error.collect_exceptions, 'scatter')
def process_did_list(did_list):
"""Turn a list of deferred_ids into a final result or failure."""
new_d_list = [self.get_pending_deferred(did, True) for did in did_list]
final_d = gatherBoth(new_d_list,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
final_d.addCallback(error.collect_exceptions, 'scatter')
final_d.addCallback(lambda lop: [i[0] for i in lop])
return final_d
# Now, depending on block, we need to handle the list deferred_ids
# coming down the pipe diferently.
if block:
# If we are blocking register a callback that will transform the
# list of deferred_ids into the final result.
d.addCallback(process_did_list)
return d
else:
# Here we are going to use a _local_ PendingDeferredManager.
deferred_id = self.pdm.get_deferred_id()
# This is the deferred we will return to the user that will fire
# with the local deferred_id AFTER we have received the list of
# primary deferred_ids
d_to_return = defer.Deferred()
def do_it(did_list):
"""Produce a deferred to the final result, but first fire the
deferred we will return to the user that has the local
deferred id."""
d_to_return.callback(deferred_id)
return process_did_list(did_list)
d.addCallback(do_it)
# Now save the deferred to the final result
self.pdm.save_pending_deferred(d, deferred_id)
return d_to_return
def pullNew(self, *keys):
if len(keys) == 1:
return self.pullPaging(keys[0])
else:
pulledDeferreds = []
for k in keys:
pulledDeferreds.append(self.pullPaging(k))
d = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=0,
consumeErrors=1)
return d
def clearProperties(self, targets):
log.msg("Clearing properties on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError,
error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = [e.clearProperties() for e in engines]
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def pullSerialized(self, targets, *keys):
try:
dList = self._performOnEngines('pullSerialized', targets, *keys)
except (error.InvalidEngineID, AttributeError,
error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
for d in dList:
d.addCallback(self._logSizes)
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def clear_properties(self, targets='all'):
log.msg("Clearing properties on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = [e.clear_properties() for e in engines]
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'clear_properties')
return d
def pull_serialized(self, keys, targets='all'):
try:
dList = self._performOnEngines('pull_serialized', keys, targets=targets)
except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
for d in dList:
d.addCallback(self._logSizes)
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, 'pull_serialized')
return d
def _performOnEnginesAndGatherBoth(self, methodName, targets, *args,
**kwargs):
"""Called _performOnEngines and wraps result/exception into deferred."""
try:
dList = self._performOnEngines(methodName, targets, *args,
**kwargs)
except (error.InvalidEngineID, AttributeError,
error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def queueStatus(self, targets):
log.msg("Getting queue status on %r" % targets)
try:
engines = self.engineList(targets)
except (error.InvalidEngineID, AttributeError,
error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
dList = []
for e in engines:
dList.append(e.queueStatus().addCallback(lambda s: (e.id, s)))
return gatherBoth(dList,
fireOnOneErrback=1,
consumeErrors=1,
logErrors=0)
def pullFunction(self, *keys):
msg = """engine %r
method: pullFunction(*keys)
keys = %r""" % (self.id, keys)
if len(keys) == 1:
return self.executeAndRaise(msg, self.shell.pullFunction, keys[0])
elif len(keys) > 1:
pulledDeferreds = []
for key in keys:
d = self.executeAndRaise(msg, self.shell.pullFunction, key)
pulledDeferreds.append(d)
# This will fire on the first failure and log the rest.
dTotal = gatherBoth(pulledDeferreds,
fireOnOneErrback=1,
logErrors=1,
consumeErrors=1)
return dTotal
else:
return self.executeAndRaise(msg, self.shell.pullFunction, None)
def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs):
"""Called _performOnEngines and wraps result/exception into deferred."""
try:
dList = self._performOnEngines(methodName, *args, **kwargs)
except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered):
return defer.fail(failure.Failure())
else:
# Having fireOnOneErrback is causing problems with the determinacy
# of the system. Basically, once a single engine has errbacked, this
# method returns. In some cases, this will cause client to submit
# another command. Because the previous command is still running
# on some engines, this command will be queued. When those commands
# then errback, the second command will raise QueueCleared. Ahhh!
d = gatherBoth(dList,
fireOnOneErrback=0,
consumeErrors=1,
logErrors=0)
d.addCallback(error.collect_exceptions, methodName)
return d
