def publish_astral(self, now=None): """I am called every minute.""" def _logFailure(failure): self.log.debug("reported {message}", message=failure.getErrorMessage()) return failure def _logAll(*args): self.log.debug("all publishing complete args={args!r}", args=args) deferreds = [] now = self._now(now) moon_msg = self.moon_message(now) if moon_msg: deferreds.append(self._create_msg('moon', moon_msg)) solar_msg = self.sun_message(now) if solar_msg: deferreds.append(self._create_msg('solar', solar_msg)) if deferreds: dl = DeferredList(deferreds) dl.addCallbacks(_logAll, _logFailure) return dl else: return None
def exchange((xi, rhoi1, rhoi2)): # Send share to all receivers. ds = runtime.broadcast(receivers, receivers, str((str(xi), str(rhoi1), str(rhoi2)))) dls = DeferredList(ds) dls.addCallbacks(check, runtime.error_handler) return dls
def enqueue(self): now = int(time.time()) # Compare the heap min timestamp with now(). # If it's time for the item to be queued, pop it, update the # timestamp and add it back to the heap for the next go round. queue_items = [] if self.amqp_queue_size < 100000: queue_items_a = queue_items.append LOGGER.debug("%s:%s" % (self.heap[0][0], now)) while self.heap[0][0] < now and len(queue_items) < 1000: job = heappop(self.heap) uuid = UUID(bytes=job[1][0]) if not uuid.hex in self.unscheduled_items: queue_items_a(job[1][0]) new_job = (now + job[1][1], job[1]) heappush(self.heap, new_job) else: self.unscheduled_items.remove(uuid.hex) else: LOGGER.critical('AMQP queue is at or beyond max limit (%d/100000)' % self.amqp_queue_size) # add items to amqp if queue_items: LOGGER.info('Found %d new uuids, adding them to the queue' % len(queue_items)) msgs = [Content(uuid) for uuid in queue_items] deferreds = [self.chan.basic_publish( exchange=self.amqp_exchange, content=msg) for msg in msgs] d = DeferredList(deferreds, consumeErrors=True) d.addCallbacks(self._addToQueueComplete, self._addToQueueErr) else: self.enqueueCallLater = reactor.callLater(1, self.enqueue)
def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks 返回带有所有成功调用给定回调的输出的Deferred """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d
def _newConnections(self, conn_str): """ Make a pool of new connections. """ # lock with the semaphore before calling this logging.debug("IndxConnectionPool _newConnections") return_d = Deferred() self.connections[conn_str] = DBConnectionPool(conn_str) try: d_list = [] for i in range(MIN_CONNS): connection_d = self._newConnection(conn_str) d_list.append(connection_d) dl = DeferredList(d_list) dl.addCallbacks(return_d.callback, return_d.errback) except Exception as e: logging.error("IndxConnectionPool error in _newConnections: {0}".format(e)) return_d.errback(Failure(e)) return return_d
def _newConnections(self, conn_str): """ Make a pool of new connections. """ # lock with the semaphore before calling this logging.debug("IndxConnectionPool _newConnections") return_d = Deferred() self.connections[conn_str] = DBConnectionPool(conn_str) try: d_list = [] for i in range(MIN_CONNS): connection_d = self._newConnection(conn_str) d_list.append(connection_d) dl = DeferredList(d_list) dl.addCallbacks(return_d.callback, return_d.errback) except Exception as e: logging.error( "IndxConnectionPool error in _newConnections: {0}".format(e)) return_d.errback(Failure(e)) return return_d
def _process_incoming_email(self, otherself, filepath, mask): """ Callback that processes incoming email @param otherself: Watch object for the current callback from inotify @type otherself: twisted.internet.inotify._Watch @param filepath: Path of the file that changed @type filepath: twisted.python.filepath.FilePath @param mask: identifier for the type of change that triggered this callback @type mask: int """ if os.path.split(filepath.dirname())[-1] == "new": log.msg("Processing new mail at %s" % (filepath.path,)) with filepath.open("r") as f: mail_data = f.read() mail = message_from_string(mail_data) owner = mail["To"] if owner is None: # default to Delivered-To owner = mail["Delivered-To"] if owner is None: log.err("Malformed mail, neither To: nor " "Delivered-To: field") log.msg("Mail owner: %s" % (owner,)) log.msg("%s received a new mail" % (owner,)) dpubk = self._users_cdb.getPubKey(owner) duuid = self._users_cdb.queryByAddress(owner) d = DeferredList([dpubk, duuid]) d.addCallbacks(self._gather_uuid_pubkey, log.err) d.addCallbacks(self._encrypt_message, log.err, (owner, mail_data)) d.addCallbacks(self._export_message, log.err) d.addCallbacks(self._conditional_remove, log.err, (filepath,)) d.addErrback(log.err)
def test_minimal(self): embedded_components, client_components = [], [ Case2_Backend, Case2_Frontend ] ## create a WAMP router factory ## router_factory = RouterFactory() ## create a WAMP router session factory ## session_factory = RouterSessionFactory(router_factory) ## .. and create and add an WAMP application session to ## run next to the router ## config = types.ComponentConfig(realm=self.realm, extra={'caselog': 'case1.log'}) try: log = io.open('caselog.log', 'w') except Exception as e: print(e) return # log = io.open(config.extra['caselog'], 'w') config.log = log config.dlog = [] config.components = [] config.all_done = [] for C in embedded_components: one_done = Deferred() config.all_done.append(one_done) c = C(config, one_done) config.components.append(c) session_factory.add(c) if self.transport == "websocket": ## create a WAMP-over-WebSocket transport server factory ## transport_factory = WampWebSocketServerFactory( session_factory, debug_wamp=self.debug) transport_factory.setProtocolOptions(failByDrop=False, openHandshakeTimeout=0, closeHandshakeTimeout=0) elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']: ## create a WAMP-over-RawSocket transport server factory ## if self.transport == 'rawsocket-msgpack': serializer = MsgPackSerializer() elif self.transport == 'rawsocket-json': serializer = JsonSerializer() else: raise Exception("should not arrive here") transport_factory = WampRawSocketServerFactory( session_factory, serializer, debug=self.debug) else: raise Exception("should not arrive here") ## start the server from an endpoint ## from twisted.internet import reactor server = serverFromString(reactor, self.server) d = server.listen(transport_factory) def onlisten(port): config.port = port d.addCallback(onlisten) clients = [] clients_d = [] for C in client_components: ## create a WAMP application session factory ## session_factory = ApplicationSessionFactory(config) one_done = Deferred() config.all_done.append(one_done) def make_make(Klass, done): def make(config): c = Klass(config, done) config.components.append(c) return c return make ## .. and set the session class on the factory ## session_factory.session = make_make(C, one_done) if self.transport == "websocket": serializers = [JsonSerializer()] ## create a WAMP-over-WebSocket transport client factory ## transport_factory = WampWebSocketClientFactory( session_factory, serializers=serializers, url=self.url, debug_wamp=self.debug) if True: def maker(Klass): class TestClientProtocol( WampWebSocketClientProtocol): def onOpen(self): self.txcnt = 0 self.rxcnt = 0 WampWebSocketClientProtocol.onOpen(self) def sendMessage(self, bytes, isBinary): self.txcnt += 1 print("> : {:>3} : {:<20} : {}".format( self.txcnt, Klass.__name__, bytes)) WampWebSocketClientProtocol.sendMessage( self, bytes, isBinary) def onMessage(self, bytes, isBinary): self.rxcnt += 1 print("< : {:>3} : {:<20} : {}".format( self.rxcnt, Klass.__name__, bytes)) WampWebSocketClientProtocol.onMessage( self, bytes, isBinary) return TestClientProtocol transport_factory.protocol = maker(C) else: transport_factory.protocol = WampWebSocketClientProtocol transport_factory.setProtocolOptions( failByDrop=False, openHandshakeTimeout=0, closeHandshakeTimeout=0) elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']: ## create a WAMP-over-RawSocket transport client factory ## if self.transport == 'rawsocket-msgpack': serializer = MsgPackSerializer() elif self.transport == 'rawsocket-json': serializer = JsonSerializer() else: raise Exception("should not arrive here") transport_factory = WampRawSocketClientFactory( session_factory, serializer, debug=self.debug) ## start the client from an endpoint ## cl = clientFromString(reactor, self.client) clients_d.append(cl.connect(transport_factory)) clients.append(cl) config.connected_clients = None def client_connected(res): config.connected_clients = [ proto for success, proto in res if success ] DeferredList(clients_d).addCallback(client_connected) d = DeferredList(config.all_done, consumeErrors=True) #d = config.components[1]._done def done(res): log.flush() log.close() if config.port: config.port.stopListening() if config.connected_clients: for proto in config.connected_clients: proto.transport.abortConnection() print("Log length: {}".format(len(config.dlog))) print(config.dlog) #from twisted.internet import reactor #reactor.callLater(1, reactor.stop) def error(err): print(err) d.addCallbacks(done, error) # d2 = Deferred() return d
def test_minimal(self): embedded_components, client_components = [], [Case2_Backend, Case2_Frontend] ## create a WAMP router factory ## router_factory = RouterFactory() ## create a WAMP router session factory ## session_factory = RouterSessionFactory(router_factory) ## .. and create and add an WAMP application session to ## run next to the router ## config = types.ComponentConfig(realm = self.realm, extra = { 'caselog': 'case1.log' } ) try: log = io.open('caselog.log', 'w') except Exception as e: print(e) return # log = io.open(config.extra['caselog'], 'w') config.log = log config.dlog = [] config.components = [] config.all_done = [] for C in embedded_components: one_done = Deferred() config.all_done.append(one_done) c = C(config, one_done) config.components.append(c) session_factory.add(c) if self.transport == "websocket": ## create a WAMP-over-WebSocket transport server factory ## transport_factory = WampWebSocketServerFactory(session_factory, debug_wamp = self.debug) transport_factory.setProtocolOptions(failByDrop = False, openHandshakeTimeout = 0, closeHandshakeTimeout = 0) elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']: ## create a WAMP-over-RawSocket transport server factory ## if self.transport == 'rawsocket-msgpack': serializer = MsgPackSerializer() elif self.transport == 'rawsocket-json': serializer = JsonSerializer() else: raise Exception("should not arrive here") transport_factory = WampRawSocketServerFactory(session_factory, serializer, debug = self.debug) else: raise Exception("should not arrive here") ## start the server from an endpoint ## from twisted.internet import reactor server = serverFromString(reactor, self.server) d = server.listen(transport_factory) def onlisten(port): config.port = port d.addCallback(onlisten) clients = [] clients_d = [] for C in client_components: ## create a WAMP application session factory ## session_factory = ApplicationSessionFactory(config) one_done = Deferred() config.all_done.append(one_done) def make_make(Klass, done): def make(config): c = Klass(config, done) config.components.append(c) return c return make ## .. and set the session class on the factory ## session_factory.session = make_make(C, one_done) if self.transport == "websocket": serializers = [JsonSerializer()] ## create a WAMP-over-WebSocket transport client factory ## transport_factory = WampWebSocketClientFactory(session_factory, serializers = serializers, url = self.url, debug_wamp = self.debug) if True: def maker(Klass): class TestClientProtocol(WampWebSocketClientProtocol): def onOpen(self): self.txcnt = 0 self.rxcnt = 0 WampWebSocketClientProtocol.onOpen(self) def sendMessage(self, payload, isBinary): self.txcnt += 1 print("> : {0:>3} : {1:<20} : {3}".format(self.txcnt, Klass.__name__, payload)) WampWebSocketClientProtocol.sendMessage(self, payload, isBinary) def onMessage(self, payload, isBinary): self.rxcnt += 1 print("< : {0:>3} : {1:<20} : {2}".format(self.rxcnt, Klass.__name__, payload)) WampWebSocketClientProtocol.onMessage(self, payload, isBinary) return TestClientProtocol transport_factory.protocol = maker(C) else: transport_factory.protocol = WampWebSocketClientProtocol transport_factory.setProtocolOptions(failByDrop = False, openHandshakeTimeout = 0, closeHandshakeTimeout = 0) elif self.transport in ['rawsocket-json', 'rawsocket-msgpack']: ## create a WAMP-over-RawSocket transport client factory ## if self.transport == 'rawsocket-msgpack': serializer = MsgPackSerializer() elif self.transport == 'rawsocket-json': serializer = JsonSerializer() else: raise Exception("should not arrive here") transport_factory = WampRawSocketClientFactory(session_factory, serializer, debug = self.debug) ## start the client from an endpoint ## cl = clientFromString(reactor, self.client) clients_d.append(cl.connect(transport_factory)) clients.append(cl) config.connected_clients = None def client_connected(res): config.connected_clients = [proto for success, proto in res if success] DeferredList(clients_d).addCallback(client_connected) d = DeferredList(config.all_done, consumeErrors = True) #d = config.components[1]._done def done(_): log.flush() log.close() if config.port: config.port.stopListening() if config.connected_clients: for proto in config.connected_clients: proto.transport.abortConnection() print("Log length: {0}".format(len(config.dlog))) print(config.dlog) #from twisted.internet import reactor #reactor.callLater(1, reactor.stop) def error(err): print(err) d.addCallbacks(done, error) # d2 = Deferred() return d
df = [] for i in range(options.workers): args = [options.wsuri, str(options.threads), str(options.conns), str(options.lowmark), str(options.highmark), options.resultfile % i] ## run wsperf executable d = getProcessOutput(options.wsperf, args, os.environ) ## accumulate any output df.append(d) d = DeferredList(df, consumeErrors = True) def onok(res): if not options.skipanalyze: analyze.printResults(resultfiles) reactor.stop() def onerr(err): print err reactor.stop() d.addCallbacks(onok, onerr) reactor.run()
""" entry = make_entry(logid, ts, app, level, msg, src_names, dst_names) l = [] for f in self.filters: try: r = f(entry) if isinstance(r, Deferred): l.append(r) except Exception, e: self.report([(False, e)]) dl = DeferredList(l, fireOnOneCallback=False, fireOnOneErrback=False, consumeErrors=True) dl.addCallback(self.report) # DeferredList never calls an errback dl.addCallbacks(defer.callback, defer.errback) #process_ = lambda logid, ts, app, level, msg, src_names, dst_names:\ # process(logid, ts, app, level, msg, src_names, dst_names, d) # #self.uel.get_log_entry(id, process_) #return d def report(self, results): for result in results: if not result[0]: # Unfortunatively, we can't print out the exact # Failure (and included stack trace) due to a bug # in Twisted mail library. It'll result in an # exception within the library: #
for i in range(options.workers): args = [ options.wsuri, str(options.threads), str(options.conns), str(options.lowmark), str(options.highmark), options.resultfile % i ] ## run wsperf executable d = getProcessOutput(options.wsperf, args, os.environ) ## accumulate any output df.append(d) d = DeferredList(df, consumeErrors=True) def onok(res): if not options.skipanalyze: analyze.printResults(resultfiles) reactor.stop() def onerr(err): print err reactor.stop() d.addCallbacks(onok, onerr) reactor.run()
def exchange((xi, rhoi1, rhoi2)): # Send share to all receivers. ds = runtime.broadcast(receivers, receivers, str((str(xi), str(rhoi1), str(rhoi2)))) dls = DeferredList(ds) dls.addCallbacks(check, runtime.error_handler) return dls
class asyncScanner: def __init__(self, ips, ports, max_threads): self.ips = ips self.ports = ports self.max_threads = max_threads self.scan_complete = False # this is a semaphore to limit the number of concurrent requests. #self.semaphore = DeferredSemaphore(self.max_threads) self.targets_list = Targets(ips, ports) self.ok = [] self.error = [] self.count = 0 for ip in self.ips: # create target for port in ports: self.count += 1 self.scan() def CheckPort(self, host, port, max_threads, timeout=10): d=Deferred() factory = ScanClientFactory(d, host, port) reactor.connectTCP(host, port, factory, timeout) return d def scan(self): self.scan_list = list() for ip in self.ips: for port in self.ports: # make a deferred job to check the host d = self.CheckPort(ip, port, self.max_threads) d.addCallbacks(self.scan_success, self.scan_failure) d.addBoth(self.scan_completed) self.scan_list.append(d) self.dl = DeferredList(self.scan_list) return self.dl.addCallbacks(self.jobSuccess, self.jobError) def get_results(self): if self.scan_complete: return self.ok else: return None ### Count number of success def scan_success(self, result): self.ok.append(result) ### Count number of errors def scan_failure(self, reason): self.error.append(reason) # When a scan completes, check if it's the last one... to stop the reactor def scan_completed(self, data): if len(self.ok) + len(self.error) == self.count: reactor.stop() def jobSuccess(self, x): self.scan_complete = True return True def jobError(self, reason): return False #print 'Scan Error'
entry = make_entry(logid, ts, app, level, msg, src_names, dst_names) l = [] for f in self.filters: try: r = f(entry) if isinstance(r, Deferred): l.append(r) except Exception, e: self.report([(False, e)]) dl = DeferredList(l, fireOnOneCallback=False, fireOnOneErrback=False, consumeErrors=True) dl.addCallback(self.report) # DeferredList never calls an errback dl.addCallbacks(defer.callback, defer.errback) #process_ = lambda logid, ts, app, level, msg, src_names, dst_names:\ # process(logid, ts, app, level, msg, src_names, dst_names, d) # #self.uel.get_log_entry(id, process_) #return d def report(self, results): for result in results: if not result[0]: # Unfortunatively, we can't print out the exact # Failure (and included stack trace) due to a bug # in Twisted mail library. It'll result in an # exception within the library: #