def pullSerialized(self, *keys): msg = """engine %r method: pullSerialized(*keys) keys = %r""" % (self.id, keys) if len(keys) == 1: key = keys[0] d = self.executeAndRaise(msg, self.shell.pull, key) d.addCallback(newserialized.serialize) return d elif len(keys) > 1: pulledDeferreds = [] for key in keys: d = self.executeAndRaise(msg, self.shell.pull, key) pulledDeferreds.append(d) # This will fire on the first failure and log the rest. dList = gatherBoth(pulledDeferreds, fireOnOneErrback=1, logErrors=0, consumeErrors=1) @dList.addCallback def packThemUp(values): serials = [] for v in values: try: serials.append(newserialized.serialize(v)) except: return defer.fail(failure.Failure()) return dict(zip(keys, serials)) return packThemUp
def pushSerializedNew(self, **namespace): dList = [] for k, v in namespace.iteritems(): d = self.pushPagingSerialized(k, v) dList.append(d) return gatherBoth(dList, fireOnOneErrback=1, logErrors=0, consumeErrors=1)
def process_did_list(did_list): """Turn a list of deferred_ids into a final result or failure.""" new_d_list = [self.get_pending_deferred(did, True) for did in did_list] final_d = gatherBoth(new_d_list, fireOnOneErrback=0, consumeErrors=1, logErrors=0) final_d.addCallback(error.collect_exceptions, 'scatter') final_d.addCallback(lambda lop: [i[0] for i in lop]) return final_d
def getAllPendingDeferreds(self, clientID): dList = [] keys = self.pdManagers[clientID].pendingDeferreds.keys() for k in keys: dList.append(self.pdManagers[clientID].getPendingDeferred( k, block=True)) if len(dList) > 0: return gatherBoth(dList, consumeErrors=1) else: return defer.succeed([None])
def do_scatter(engines): nEngines = len(engines) mapClass = Map.styles[style] mapObject = mapClass() d_list = [] # Loop through and push to each engine in non-blocking mode. # This returns a set of deferreds to deferred_ids for index, engineid in enumerate(engines): partition = mapObject.getPartition(seq, index, nEngines) if flatten and len(partition) == 1: d = self.push({key: partition[0]}, targets=engineid, block=False) else: d = self.push({key: partition}, targets=engineid, block=False) d_list.append(d) # Collect the deferred to deferred_ids d = gatherBoth(d_list, fireOnOneErrback=0, consumeErrors=1, logErrors=0) # Now d has a list of deferred_ids or Failures coming d.addCallback(error.collect_exceptions, 'scatter') def process_did_list(did_list): """Turn a list of deferred_ids into a final result or failure.""" new_d_list = [self.get_pending_deferred(did, True) for did in did_list] final_d = gatherBoth(new_d_list, fireOnOneErrback=0, consumeErrors=1, logErrors=0) final_d.addCallback(error.collect_exceptions, 'scatter') final_d.addCallback(lambda lop: [i[0] for i in lop]) return final_d # Now, depending on block, we need to handle the list deferred_ids # coming down the pipe diferently. if block: # If we are blocking register a callback that will transform the # list of deferred_ids into the final result. d.addCallback(process_did_list) return d else: # Here we are going to use a _local_ PendingDeferredManager. deferred_id = self.pdm.get_deferred_id() # This is the deferred we will return to the user that will fire # with the local deferred_id AFTER we have received the list of # primary deferred_ids d_to_return = defer.Deferred() def do_it(did_list): """Produce a deferred to the final result, but first fire the deferred we will return to the user that has the local deferred id.""" d_to_return.callback(deferred_id) return process_did_list(did_list) d.addCallback(do_it) # Now save the deferred to the final result self.pdm.save_pending_deferred(d, deferred_id) return d_to_return
def pullNew(self, *keys): if len(keys) == 1: return self.pullPaging(keys[0]) else: pulledDeferreds = [] for k in keys: pulledDeferreds.append(self.pullPaging(k)) d = gatherBoth(pulledDeferreds, fireOnOneErrback=1, logErrors=0, consumeErrors=1) return d
def clearProperties(self, targets): log.msg("Clearing properties on %r" % targets) try: engines = self.engineList(targets) except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: dList = [e.clearProperties() for e in engines] return gatherBoth(dList, fireOnOneErrback=1, consumeErrors=1, logErrors=0)
def pullSerialized(self, targets, *keys): try: dList = self._performOnEngines('pullSerialized', targets, *keys) except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: for d in dList: d.addCallback(self._logSizes) return gatherBoth(dList, fireOnOneErrback=1, consumeErrors=1, logErrors=0)
def clear_properties(self, targets='all'): log.msg("Clearing properties on %r" % targets) try: engines = self.engineList(targets) except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: dList = [e.clear_properties() for e in engines] d = gatherBoth(dList, fireOnOneErrback=0, consumeErrors=1, logErrors=0) d.addCallback(error.collect_exceptions, 'clear_properties') return d
def pull_serialized(self, keys, targets='all'): try: dList = self._performOnEngines('pull_serialized', keys, targets=targets) except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: for d in dList: d.addCallback(self._logSizes) d = gatherBoth(dList, fireOnOneErrback=0, consumeErrors=1, logErrors=0) d.addCallback(error.collect_exceptions, 'pull_serialized') return d
def _performOnEnginesAndGatherBoth(self, methodName, targets, *args, **kwargs): """Called _performOnEngines and wraps result/exception into deferred.""" try: dList = self._performOnEngines(methodName, targets, *args, **kwargs) except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: return gatherBoth(dList, fireOnOneErrback=1, consumeErrors=1, logErrors=0)
def queueStatus(self, targets): log.msg("Getting queue status on %r" % targets) try: engines = self.engineList(targets) except (error.InvalidEngineID, AttributeError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: dList = [] for e in engines: dList.append(e.queueStatus().addCallback(lambda s: (e.id, s))) return gatherBoth(dList, fireOnOneErrback=1, consumeErrors=1, logErrors=0)
def pullFunction(self, *keys): msg = """engine %r method: pullFunction(*keys) keys = %r""" % (self.id, keys) if len(keys) == 1: return self.executeAndRaise(msg, self.shell.pullFunction, keys[0]) elif len(keys) > 1: pulledDeferreds = [] for key in keys: d = self.executeAndRaise(msg, self.shell.pullFunction, key) pulledDeferreds.append(d) # This will fire on the first failure and log the rest. dTotal = gatherBoth(pulledDeferreds, fireOnOneErrback=1, logErrors=1, consumeErrors=1) return dTotal else: return self.executeAndRaise(msg, self.shell.pullFunction, None)
def _performOnEnginesAndGatherBoth(self, methodName, *args, **kwargs): """Called _performOnEngines and wraps result/exception into deferred.""" try: dList = self._performOnEngines(methodName, *args, **kwargs) except (error.InvalidEngineID, AttributeError, KeyError, error.NoEnginesRegistered): return defer.fail(failure.Failure()) else: # Having fireOnOneErrback is causing problems with the determinacy # of the system. Basically, once a single engine has errbacked, this # method returns. In some cases, this will cause client to submit # another command. Because the previous command is still running # on some engines, this command will be queued. When those commands # then errback, the second command will raise QueueCleared. Ahhh! d = gatherBoth(dList, fireOnOneErrback=0, consumeErrors=1, logErrors=0) d.addCallback(error.collect_exceptions, methodName) return d