def disconnectAll(self): reactor.runUntilCurrent() def _disconnectAll(): for subscriber in self.connections_ready: yield subscriber.transport.loseConnection() task.cooperate(_disconnectAll())
def main(): print "Starting up..." state = SharedState() bot = BotFactory(config, state) reactor.connectSSL(config.get('server'), config.get('port'), bot, ssl.ClientContextFactory()) cooperate(SlackBot(state, config).listen_to_slack()) reactor.run()
def setup(self): sb = self.blackboard.bot_standing_on_block(self.blackboard.bot_object) while sb is None: yield utils.reactor_break() sb = self.blackboard.bot_standing_on_block(self.blackboard.bot_object) else: if self.travel_multiple_goals is not None: d = cooperate(AStarMultiCoords(dimension=self.blackboard.dimension, start_coords=sb.coords, goal_coords=self.travel_coords, multiple_goals=self.travel_multiple_goals)).whenDone() elif self.travel_coords is not None: d = cooperate(AStarCoords(dimension=self.blackboard.dimension, start_coords=sb.coords, goal_coords=self.travel_coords)).whenDone() else: d = cooperate(AStarBBCol(dimension=self.blackboard.dimension, start_coords=sb.coords, bb=self.travel_bb)).whenDone() d.addErrback(logbot.exit_on_error) astar = yield d if astar.path is not None: current_start = self.blackboard.bot_standing_on_block(self.blackboard.bot_object) if sb == current_start: self.path = astar.path if self.shorten_path_by > 0: self.path = self.path[self.shorten_path_by:] self.start_coords = current_start.coords
def update_chunks(self): x, y, z = self.location.pos.to_block() x, chaff, z, chaff = split_coords(x, z) new = set((i + x, j + z) for i, j in circle) old = set(self.chunks.iterkeys()) added = new - old discarded = old - new # Perhaps some explanation is in order. # The cooperate() function iterates over the iterable it is fed, # without tying up the reactor, by yielding after each iteration. The # inner part of the generator expression generates all of the chunks # around the currently needed chunk, and it sorts them by distance to # the current chunk. The end result is that we load chunks one-by-one, # nearest to furthest, without stalling other clients. if self.chunk_tasks: for task in self.chunk_tasks: try: task.stop() except (TaskDone, TaskFailed): pass self.chunk_tasks = [ cooperate( self.enable_chunk(i, j) for i, j in sorted(added, key=lambda t: (t[0] - x)**2 + (t[1] - z)**2) ), cooperate(self.disable_chunk(i, j) for i, j in discarded) ]
def pushInfoToSubscribers(self): # Generate the json doc vars = {} for subscriber in self.connections_ready: subscriber_vars = subscriber.vars.copy() subscriber_vars['port'] = subscriber.id + 12000 subscriber_vars['host'] = subscriber.transport.getPeer().host vars[subscriber.id] = subscriber_vars json_vars = json.dumps(vars) del vars msg("Pushing a %d bytes long json doc." % len(json_vars)) # Send the json doc to the subscribers task.cooperate(self._sendLineToAllGenerator(json_vars))
def _disconnect(self, receipt, failure, timeout): if failure: self._disconnectReason = failure self.log.info('Disconnecting ...%s' % ('' if (not failure) else ('[reason=%s]' % failure))) protocol = self._protocol try: # notify that we are ready to disconnect after outstanding messages are ack'ed if self._messages: self.log.info('Waiting for outstanding message handlers to finish ... [timeout=%s]' % timeout) try: yield task.cooperate(iter([wait(handler, timeout, StompCancelledError('Going down to disconnect now')) for handler in self._messages.values()])).whenDone() except StompCancelledError as e: self._disconnectReason = StompCancelledError('Handlers did not finish in time.') else: self.log.info('All handlers complete. Resuming disconnect ...') if self.session.state == self.session.CONNECTED: frame = self.session.disconnect(receipt) try: self.sendFrame(frame) except Exception as e: self._disconnectReason = StompConnectionError('Could not send %s. [%s]' % (frame.info(), e)) try: yield self._waitForReceipt(receipt) except StompCancelledError: self._disconnectReason = StompCancelledError('Receipt for disconnect command did not arrive on time.') protocol.loseConnection() except Exception as e: self._disconnectReason = e
def connectionMade(self): print("Connected protocol") self.queue = Queue.Queue() #self.transport.registerProducer(QueueProducer(self.transport), False) self.sendLine("3,0,0") gen = DoodleQueueWorkload(self.transport.reactor, self) self.task = cooperate(gen)
def main(reactor, *argv): parameters = ConnectionParameters.fromCommandLine(reactor, argv) endpoint = parameters.endpointForCommand(b"/bin/cat") done = [] factory = Factory() factory.protocol = Protocol d = endpoint.connect(factory) def gotConnection(proto): conn = proto.transport.conn for i in range(50): factory = Factory() factory.protocol = PrinterProtocol factory.done = Deferred() done.append(factory.done) e = SSHCommandClientEndpoint.existingConnection( conn, b"/bin/echo %d" % (i,)) yield e.connect(factory) d.addCallback(gotConnection) d.addCallback(lambda work: cooperate(work).whenDone()) d.addCallback(lambda ignored: gatherResults(done)) return d
def startStreaming(self): """ This should be called by the consumer when the producer is registered. Start streaming data to the consumer. """ self._coopTask = cooperate(self._pull())
def startProducing(self, consumer): """ Must NOT call registerProducer on the consumer """ self._consumer = consumer self._iterable = json.JSONEncoder().iterencode(self.body) self._task = cooperate(self._produce()) d = self._task.whenDone() return d
def searchIndices(indices, query, period, lastId=None, reverse=False, fields=None, limit=100): """ Search the specified indices for events matching the specified query. :param indices: A list of indices to search. :type indices: A list of objects implementing :class:`terane.bier.index.IIndex` :param query: The programmatic query to use for searching the indices. :type query: An object implementing :class:`terane.bier.searching.IQuery` :param period: The period within which the search is constrained. :type period: :class:`terane.bier.searching.Period` :param lastId: The real key to start iterating from. :type lastId: :class:`terane.bier.evid.EVID` :param reverse: If True, then reverse the order of events. :type reverse: bool :param fields: If not None, then only return the specified fields of each event. :type fields: list or None :param limit: Only returned the specified number of events. :type limit: int :returns: A CooperativeTask which contains a Deferred and manages the search task. :rtype: :class:`twisted.internet.task.CooperativeTask` """ start = time.time() # determine the evids to use as start and end keys if reverse == False: startId, endId = period.getRange() else: endId, startId = period.getRange() if lastId != None: if not lastId in period: raise SearcherError("lastId %s is not within period" % lastId) startId = lastId # search each index separately, then merge the results try: searchers = [] postingLists = [] for index in indices: if not IIndex.providedBy(index): raise TypeError("index does not implement IIndex") # we create a copy of the original query, which can possibly be optimized # with index-specific knowledge. _query = copy.deepcopy(query) try: _query = _query.optimizeMatcher(index) except NotImplementedError, e: raise SearcherError(str(e)) logger.debug("optimized query for index '%s': %s" % (index.name, str(_query))) # if the query optimized out entirely, then skip to the next index if _query == None: continue # get the posting list to iterate through searcher = index.newSearcher() if not ISearcher.providedBy(searcher): raise TypeError("searcher does not implement ISearcher") postingList = _query.iterMatches(searcher, startId, endId) if not IPostingList.providedBy(postingList): raise TypeError("posting list does not implement IPostingList") searchers.append(searcher) postingLists.append(postingList) # return a cooperative task return cooperate(ResultSet(searchers, postingLists, start, reverse, fields, limit))
def startProducing(self, consumer): self._consumer = consumer self._iterable = SmapEncoder().iterencode(self._value) self._task = cooperate(self._produce()) d = self._task.whenDone() d.addBoth(self._unregister) return d
def send_initial_chunk_and_location(self): bigx, smallx, bigz, smallz = split_coords(self.location.x, self.location.z) # Spawn the 25 chunks in a square around the spawn, *before* spawning # the player. Otherwise, there's a funky Beta 1.2 bug which causes the # player to not be able to move. d = cooperate( self.enable_chunk(i, j) for i, j in product( xrange(bigx - 3, bigx + 3), xrange(bigz - 3, bigz + 3) ) ).whenDone() # Don't dare send more chunks beyond the initial one until we've # spawned. d.addCallback(lambda none: self.update_location()) d.addCallback(lambda none: self.position_changed()) # Send the MOTD. if self.motd: packet = make_packet("chat", message=self.motd.replace("<tagline>", get_motd())) d.addCallback(lambda none: self.transport.write(packet)) # Finally, start the secondary chunk loop. d.addCallback(lambda none: self.update_chunks())
def _driveWorker(self, worker, result, testCases, cooperate): """ Drive a L{LocalWorkerAMP} instance, iterating the tests and calling C{run} for every one of them. @param worker: The L{LocalWorkerAMP} to drive. @param result: The global L{DistReporter} instance. @param testCases: The global list of tests to iterate. @param cooperate: The cooperate function to use, to be customized in tests. @type cooperate: C{function} @return: A C{Deferred} firing when all the tests are finished. """ def resultErrback(error, case): result.original.addFailure(case, error) return error def task(case): d = worker.run(case, result) d.addErrback(resultErrback, case) return d return cooperate(task(case) for case in testCases).whenDone()
def test_onDisconnect_waitForOutstandingMessagesToFinish(self): config = self.getConfig(StompSpec.VERSION_1_0) client = async.Stomp(config) client.add(ReceiptListener(1.0)) # connect yield client.connect(host=VIRTUALHOST) yield task.cooperate( iter([client.send(self.queue, self.frame, receipt="message-%d" % j) for j in range(self.numMsgs)]) ).whenDone() client.subscribe( self.queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}, listener=SubscriptionListener(self._frameHandler), ) # wait for disconnect yield client.disconnected # reconnect and subscribe again to make sure that all messages in the queue were ack'ed yield client.connect(host=VIRTUALHOST) self.timeExpired = False self.timeoutDelayedCall = reactor.callLater(1, self._timesUp, client) # @UndefinedVariable client.subscribe( self.queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}, listener=SubscriptionListener(self._eatOneFrameAndDisconnect), ) # wait for disconnect yield client.disconnected # time should have expired if there were no messages left in the queue self.assertTrue(self.timeExpired)
def wrapper(n): def stop_task(unused_deferred): try: t.stop() except task.TaskFailed: pass d = defer.Deferred(canceller=stop_task) t = task.cooperate(cooperator(func(n), n+1, yield_interval, d.callback)) return d
def compute(self, node, recheck=False): if self.incomplete_nodes: self.incomplete_nodes[node] = recheck else: self.incomplete_nodes[node] = recheck cootask = cooperate(self.do_incomplete_nodes()) d = cootask.whenDone() d.addErrback(logbot.exit_on_error)
def main(res): """ Go main Go! """ log.msg("main() fired!") jobs = DeferredQueue() ingest = MyProductIngestor() ingest.jobs = jobs ldmbridge.LDMProductFactory(ingest) for _ in range(6): cooperate(worker(jobs)) lc = LoopingCall(save_current) lc.start(373, now=False) lc2 = LoopingCall(service_guard, jobs) lc2.start(61, now=False)
def begin(self, consumer): self._consumer=consumer self._iterable=JSONEncoder().iterencode(self._value) self._consumer.registerProducer(self, True) self._task=cooperate(self._produce()) defer=self._task.whenDone() defer.addBoth(self._unregister) return defer
def _receivedEvent(self, event): # run the fields through the filter chain, then reschedule the signal self._input.getContract().validateEventAfter(event, self.parent._fieldstore) task = cooperate(EventProcessor(event, self._filters, self.parent._fieldstore)) d = task.whenDone() d.addCallbacks(self._processedEvent, lambda failure: failure) d.addErrback(self._errorProcessingEvent) self._scheduleReceivedEvent()
def main(): logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s") logging.info('Starting up') #sensor_pollers, buffer_unloaders, sqlite_unloaders are worker lists sensor_pollers, buffer_unloaders, sqlite_unloaders = [], [], [] upload_tasks = defer.DeferredQueue() #Setup monitoring of fake sensors #Create sensor instance in sensor list fake_sensors = [fake_sensor()] #Iterate over sensors for source in fake_sensors: name = 'temp%d' % (1,) #Create a datapointBuffer which will call source_method (with optional source_method_args) to retrieve a datapoint) dp_buffer = sensorup.datapointBuffer(source_method=source.get_temp_channel, source_method_args=0, datastream_id=name) #Add this new datasource (datapointBuffer instance) to worker lists #sensorPollAndBuffer: poll for new data and store in an in-memory list sensor_pollers.append(task.LoopingCall(sensorup.sensorPollAndBuffer, dp_buffer)) #bufferUnloadAndUpload: Flushes in-memory list of datapoints, and attempts uploading to Cosm. Failures go into sqlite. buffer_unloaders.append(task.LoopingCall(sensorup.bufferUnloadAndUpload, dp_buffer, upload_tasks)) #Collect datapoint every 1s for poller in sensor_pollers: poller.start(1.0) #Upload every 30s for poller in buffer_unloaders: poller.start(30.0, False) #interval for sqlite_unloaders should be sufficiently large as to prevent double uploading # sqlite_unloader interval > buffer_unloaders interval # sqlite_unloader interval > client.request_timeout * sensor count #Flush old datapoints from the database every 86400s sqlite_purger = task.LoopingCall(sensorup.sqlite_purge) sqlite_purger.start(86400) #Only one concurrent upload task task.cooperate(sensorup.worker(upload_tasks)) reactor.run()
def beginProducing(self, consumer): """ See: twisted.internet.interfaces.IConsumer """ self._consumer = consumer self._iterable = json.JSONEncoder().iterencode(self.value) self._consumer.registerProducer(self, True) self._task = cooperate(self._produce()) d = self._task.whenDone() d.addBoth(self._unregister) return d
def update_chunks(self): # Don't send chunks unless we're located. if self.state != STATE_LOCATED: return x, y, z = self.location.pos.to_block() x, chaff, z, chaff = split_coords(x, z) # These numbers come from a couple spots, including minecraftwiki, but # I verified them experimentally using torches and pillars to mark # distances on each setting. ~ C. distances = { "tiny": 2, "short": 4, "far": 16, } radius = distances.get(self.settings.distance, 8) new = set(circling(x, z, radius)) old = set(self.chunks.iterkeys()) added = new - old discarded = old - new # Perhaps some explanation is in order. # The cooperate() function iterates over the iterable it is fed, # without tying up the reactor, by yielding after each iteration. The # inner part of the generator expression generates all of the chunks # around the currently needed chunk, and it sorts them by distance to # the current chunk. The end result is that we load chunks one-by-one, # nearest to furthest, without stalling other clients. if self.chunk_tasks: for task in self.chunk_tasks: try: task.stop() except (TaskDone, TaskFailed): pass to_enable = sorted_by_distance(added, x, z) self.chunk_tasks = [ cooperate(self.enable_chunk(i, j) for i, j in to_enable), cooperate(self.disable_chunk(i, j) for i, j in discarded), ]
def _method_wrapper(self, request, serializer, handler, method, logger): self.request = request self.serializer = serializer self.handler = handler self.method = method self.logger = logger self.request.registerProducer(self, True) self._task = cooperate(self._call_method()) d = self._task.whenDone() d.addBoth(self._unregister) return d
def uploadEvents(numEvents, agent, uri, cal): def worker(): for i in range(numEvents): event = makeEvent(i, 1, 0) yield agent.request( 'PUT', '%s%s%d.ics' % (uri, cal, i), Headers({"content-type": ["text/calendar"]}), StringProducer(event)) worker = worker() return gatherResults([ cooperate(worker).whenDone() for _ignore_i in range(3)])
def run(service, config): log.debug("Starting run") servers = yield service.listSharesToUpdate() work = ( service.updateOne(*r).addErrback(log_err, log, "failed to update") for r in servers ) numworkers = int(config.get("workers", 10)) or 1 workers = [task.cooperate(work).whenDone() for i in xrange(numworkers)] dl = yield defer.DeferredList(workers) log.debug("Done")
def perform_task(self): log.debug("Starting run") service = self.parent.updater servers = yield service.listSharesToUpdate() work = ( service.updateOne(*r).addErrback(log_err, log, "failed to update") for r in servers ) numworkers = self.workers workers = [task.cooperate(work).whenDone() for i in xrange(numworkers)] dl = yield defer.DeferredList(workers) log.debug("Done")
def test_cooperate(self): """ L{twisted.internet.task.cooperate} ought to run the generator that it is """ d = defer.Deferred() def doit(): yield 1 yield 2 yield 3 d.callback("yay") it = doit() theTask = task.cooperate(it) self.assertIn(theTask, task._theCooperator._tasks) return d
def created(filesystem): # Save it for later. self.filesystem = filesystem # Take a couple snapshots now that there is a filesystem. return cooperate( zfs_command( reactor, [ b"snapshot", u"{}@{}".format(filesystem.name, name).encode("ascii"), ] ) for name in expected_names ).whenDone()
def coopReduce(func, iterable, initializer=None): it = iter(iterable) x = initializer or next(it) def cooperator(func, it, x): for y in it: x = func(x, y) yield returnValue(x) task = cooperate(cooperator(func, it, x)) result = yield task.whenDone() returnValue(result)
def collect_samples(ignored): collecting = Deferred() task = cooperate( sample(operation, metric, i).addCallback(samples.append) for i in range(num_samples)) # If the scenario collapses, stop sampling def stop_sampling_on_scenario_collapse(failure): task.stop() collecting.errback(failure) scenario.maintained().addErrback(stop_sampling_on_scenario_collapse) # Leaving the errback unhandled makes tests fail task.whenDone().addCallbacks( lambda ignored: collecting.callback(samples), lambda ignored: None) return collecting
def send_initial_chunk_and_location(self): bigx, smallx, bigz, smallz = split_coords(self.location.x, self.location.z) # Spawn the 25 chunks in a square around the spawn, *before* spawning # the player. Otherwise, there's a funky Beta 1.2 bug which causes the # player to not be able to move. d = cooperate( self.enable_chunk(i, j) for i, j in product(xrange(bigx - 3, bigx + 3), xrange(bigz - 3, bigz + 3))).whenDone() # Don't dare send more chunks beyond the initial one until we've # spawned. d.addCallback(lambda none: self.update_location()) d.addCallback(lambda none: self.position_changed()) d.addCallback(lambda none: self.update_chunks())
def main(reactor): """ Create and verify one S4 subscription. """ root_url = argv[1] emails = argv[2:] tahoe_env = mkdtemp(prefix=u"tahoe-lafs-env") task = cooperate( subscribe( reactor, root_url, email, ).addCallback( check_subscription, tahoe_env, ) for email in emails) return task.whenDone()
def critical(): self._datastore.flush(key) snap = tables.open_file(snapfile, "w") raw = snap.create_vlarray( snap.root, "raw", atom=tables.UInt8Atom(shape=()), filters=tables.Filters(complevel=0)) snap.create_group(snap.root, "tables") for segment in segments: yield cooperate(generate(raw, key, segment)).whenDone() yield details.progress({ "capture_key": key, "status": True }) raw.flush() snap.flush() snap.close()
def run(self): """ Perform the migration. """ if self._running: return self._running = True def _done(ign): self._running = False migrations = chain( self.store.query(PendingMigration, PendingMigration.parent == self), iter(self._nextObject, None)) it = (m.attemptMigration() for m in migrations) tasks = [cooperate(it) for _ in xrange(self.concurrency)] d = gatherResults([task.whenDone() for task in tasks]) d.addCallback(_done) return d
def test_onDisconnect_waitForOutstandingMessagesToFinish(self): config = self.getConfig(StompSpec.VERSION_1_0) client = async.Stomp(config) client.add(ReceiptListener(1.0)) # connect client = yield client.connect(host=VIRTUALHOST) yield task.cooperate(iter([client.send(self.queue, self.frame, receipt='message-%d' % j) for j in xrange(self.numMsgs)])).whenDone() client.subscribe(self.queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}, listener=SubscriptionListener(self._frameHandler)) # wait for disconnect yield client.disconnected # reconnect and subscribe again to make sure that all messages in the queue were ack'ed client = yield client.connect(host=VIRTUALHOST) self.timeExpired = False self.timeoutDelayedCall = reactor.callLater(1, self._timesUp, client) # @UndefinedVariable client.subscribe(self.queue, {StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}, listener=SubscriptionListener(self._eatOneFrameAndDisconnect)) # wait for disconnect yield client.disconnected # time should have expired if there were no messages left in the queue self.assertTrue(self.timeExpired)
def start(self): iterator = self._gen_data() self.task = task.cooperate(iterator) self.request.registerProducer(self, streaming=True)
def iterate(): return cooperate(generate([key, ref])).whenDone()
def create_cooperate(iterObj): return cooperate(TxnProcessor.iterator(iterObj))
def cb_start_task(self, writer): self._task = task.cooperate(self._writeFlusher(writer))
def attachTarget(self, target): self.target = target task.cooperate(self._gen_data())
def startProducing(self, consumer): self._consumer = consumer self._task = cooperate(self._produce()) d = self._task.whenDone() d.addBoth(self._unregister) return d
def __init__(self, iterator): self.task = cooperate(iterator)
def go(): task = cooperate(work(job) for job in f) d = task.whenDone() d.addErrback(log.err, "Problem coordinating benchmarks") d.addCallback(lambda ignored: reactor.stop())
def _waitForMessages(self, timeout): return task.cooperate( handler.wait( timeout, StompCancelledError( 'Handlers did not finish in time.')) for handler in self._messages.values()).whenDone()