class PostgresListenerServiceSpy(PostgresListenerService): """Save received notifies `captured_notifies` before processing them..""" HANDLE_NOTIFY_DELAY = CHANNEL_REGISTRAR_DELAY = 0 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Captured notifications from the database will go here. self._captured_notifies = DeferredQueue() # Change notifications to a frozenset. This makes sure that the system # message does not go into the queue. Instead it should call the # handler directly in `doRead`. self.notifications = frozenset() def _process_notifies(self): for notify in self.connection.connection.notifies: self._captured_notifies.put(notify) super()._process_notifies() @inlineCallbacks def wait_notification(self, channel): """Wait for a notification to be received.""" while True: notice = yield self._captured_notifies.get() if notice.channel == channel: returnValue(notice)
class OmciRxProxy(pb.Root): def __init__(self): self.pb_server_ip = '192.168.24.20' # registry('main').get_args().external_host_address self.pb_server_port = 24497 self.pb_server_factory = pb.PBServerFactory(self) # start PB server self.listen_port = reactor.listenTCP(self.pb_server_port, self.pb_server_factory) self.omci_rx_queue = DeferredQueue() log.info('PB-server-started-on-port', port=self.pb_server_port) def get_ip(self): return self.pb_server_ip def get_port(self): return self.pb_server_port def get_host(self): return self.listen_port.getHost() def remote_echo(self, pkt_type, pon, onu, port, crc_ok, msg_size, msg_data): log.info('received-omci-msg', pkt_type=pkt_type, pon_id=pon, onu_id=onu, port_id=port, crc_ok=crc_ok, msg_size=msg_size, msg_data=hexify(msg_data)) self.omci_rx_queue.put((onu, msg_data)) def receive(self): return self.omci_rx_queue.get()
class SSMIServerProtocol(Protocol): delimiter = TruteqTransportProtocol.delimiter def __init__(self): self.receive_queue = DeferredQueue() self._buf = b"" def dataReceived(self, data): self._buf += data self.parse_commands() def parse_commands(self): while self.delimiter in self._buf: line, _, self._buf = self._buf.partition(self.delimiter) if line: self.receive_queue.put(SSMIRequest.parse(line)) def send(self, command): self.transport.write(str(command)) self.transport.write(self.delimiter) return wait0() def receive(self): return self.receive_queue.get() def disconnect(self): self.transport.loseConnection()
class MockHttpServer(object): """ NOTE: This is deprecated. Please use :class:`vumi.tests.http_helpers.MockHttpHelper` instead. """ def __init__(self, handler=None): self.queue = DeferredQueue() self._handler = handler or self.handle_request self._webserver = None self.addr = None self.url = None def handle_request(self, request): self.queue.put(request) @inlineCallbacks def start(self): root = MockResource(self._handler) site_factory = LogFilterSite(root) self._webserver = yield reactor.listenTCP( 0, site_factory, interface='127.0.0.1') self.addr = self._webserver.getHost() self.url = "http://%s:%s/" % (self.addr.host, self.addr.port) @inlineCallbacks def stop(self): yield self._webserver.stopListening() yield self._webserver.loseConnection()
class TestHTTPClientBase(TestCase): # TODO: Run client tests synchronously with treq.testing tools (#38) run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=0.1) def setUp(self): super(TestHTTPClientBase, self).setUp() self.requests = DeferredQueue() self.fake_server = FakeHttpServer(self.handle_request) fake_client = treq_HTTPClient(self.fake_server.get_agent()) self.client = self.get_client(fake_client) # Spin the reactor once at the end of each test to clean up any # cancelled deferreds self.addCleanup(wait0) def handle_request(self, request): self.requests.put(request) return NOT_DONE_YET def get_client(self, client): """To be implemented by subclass""" raise NotImplementedError() def uri(self, path): return '%s%s' % (self.client.url, path,) def cleanup_d(self, d): self.addCleanup(lambda: d) return d
class OmciProxy(pb.Root): def __init__(self): reactor.listenTCP(24497, pb.PBServerFactory(self)) self.remote = None self.response_queue = DeferredQueue() @inlineCallbacks def connect(self): factory = pb.PBClientFactory() reactor.connectTCP("10.111.101.206", 24498, factory) self.remote = yield factory.getRootObject() print 'connected' yield self.remote.callRemote("setRemote", port=24496) def remote_echo(self, pkt_type, pon, onu, port, crc, size, data): print "Packet Type:", pkt_type print "PON:", pon print "ONU ID:", onu print "Port:", port print "CRC OK:", crc print "Packet Size:", size print "received:", hexify(data) self.response_queue.put(data) @inlineCallbacks def send_omci(self, msg): if isinstance(msg, Packet): msg = str(msg) try: print ' sending:', msg yield self.remote.callRemote("send_omci", 0, 0, 1, msg) print 'msg sent' except Exception, e: print >> sys.stderr, 'Blew up:', str(e)
class WeChatTestCase(VumiTestCase): def setUp(self): self.tx_helper = self.add_helper(TransportHelper(WeChatTransport)) self.request_queue = DeferredQueue() self.mock_server = MockHttpServer(self.handle_api_request) self.add_cleanup(self.mock_server.stop) return self.mock_server.start() def handle_api_request(self, request): self.request_queue.put(request) return NOT_DONE_YET def get_transport(self, **config): defaults = { 'api_url': self.mock_server.url, 'auth_token': 'token', 'twisted_endpoint': 'tcp:0', 'wechat_appid': 'appid', 'wechat_secret': 'secret', 'embed_user_profile': False, } defaults.update(config) return self.tx_helper.get_transport(defaults) @inlineCallbacks def get_transport_with_access_token(self, access_token, **config): transport = yield self.get_transport(**config) yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY, access_token) returnValue(transport)
class MockHttpServer(object): """ NOTE: This is deprecated. Please use :class:`vumi.tests.http_helpers.MockHttpHelper` instead. """ def __init__(self, handler=None): self.queue = DeferredQueue() self._handler = handler or self.handle_request self._webserver = None self.addr = None self.url = None def handle_request(self, request): self.queue.put(request) @inlineCallbacks def start(self): root = MockResource(self._handler) site_factory = LogFilterSite(root) self._webserver = yield reactor.listenTCP(0, site_factory, interface='127.0.0.1') self.addr = self._webserver.getHost() self.url = "http://%s:%s/" % (self.addr.host, self.addr.port) @inlineCallbacks def stop(self): yield self._webserver.stopListening() yield self._webserver.loseConnection()
class ProcessPool(object): def __init__(self, count=10): self.limiter = DeferredSemaphore(count) self.processes = [spawnProcess() for _ in xrange(count)] self.workQueue = DeferredQueue() for process in self.processes: process.onconnect.addCallback(self._prepareForWork) @inlineCallbacks def _prepareForWork(self, proto): deferred, func, args = yield self.workQueue.get() proto.queueWork(deferred, func, *args) def requeue(result): self._prepareForWork(proto) return result deferred.addCallback(requeue) returnValue(proto) def queueWork(self, function, *args): resultDeferred = Deferred() innerDeferred = Deferred() self.workQueue.put((innerDeferred, function, args)) def callResult(obj): resultDeferred.callback(obj) return obj innerDeferred.addCallback(callResult) return resultDeferred def stop(self): for process in self.processes: process.protocol.kill() process.transport.loseConnection()
class RateLimitAgent(object): def __init__(self): self.pool = HTTPConnectionPool(reactor, persistent=True) self.pool.maxPersistentPerHost = 4 self.pool._factory = QuietHTTP11ClientFactory self.contextFactory = WhitelistContextFactory() self.agent = Agent(reactor, pool=self.pool, contextFactory=self.contextFactory) self.queue = DeferredQueue() self.getRequest() def request(self, method, url, headers, body): finished = Deferred() self.queue.put((finished, method, url, headers, body)) return finished def getRequest(self): d = self.queue.get() d.addCallback(self.gotRequest) def gotRequest(self, request): finished, method, url, headers, body = request d = self.agent.request(method, bytes(url), headers, body) d.addCallback(self.cbRequest, finished) d.addErrback(self.ebRequest, finished) def cbRequest(self, response, finished): finished.callback(response) reactor.callLater(1.3, self.getRequest) def ebRequest(self, failure, finished): finished.errback(failure) reactor.callLater(1.3, self.getRequest)
class WeChatTestCase(VumiTestCase): def setUp(self): self.tx_helper = self.add_helper(TransportHelper(WeChatTransport)) self.request_queue = DeferredQueue() self.fake_http = FakeHttpServer(self.handle_api_request) self.api_url = 'https://api.wechat.com/cgi-bin/' def handle_api_request(self, request): self.assertEqual(request.path[:len(self.api_url)], self.api_url) self.request_queue.put(request) return NOT_DONE_YET @inlineCallbacks def get_transport(self, **config): defaults = { 'auth_token': 'token', 'twisted_endpoint': 'tcp:0', 'wechat_appid': 'appid', 'wechat_secret': 'secret', 'embed_user_profile': False, 'publish_status': True, } defaults.update(config) transport = yield self.tx_helper.get_transport(defaults) transport.agent_factory = self.fake_http.get_agent returnValue(transport) @inlineCallbacks def get_transport_with_access_token(self, access_token, **config): transport = yield self.get_transport(**config) yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY, access_token) returnValue(transport)
class WeChatTestCase(VumiTestCase): def setUp(self): self.tx_helper = self.add_helper(TransportHelper(WeChatTransport)) self.request_queue = DeferredQueue() self.fake_http = FakeHttpServer(self.handle_api_request) self.api_url = 'https://api.wechat.com/cgi-bin/' def handle_api_request(self, request): self.assertEqual(request.path[:len(self.api_url)], self.api_url) self.request_queue.put(request) return NOT_DONE_YET @inlineCallbacks def get_transport(self, **config): defaults = { 'auth_token': 'token', 'twisted_endpoint': 'tcp:0', 'wechat_appid': 'appid', 'wechat_secret': 'secret', 'embed_user_profile': False, } defaults.update(config) transport = yield self.tx_helper.get_transport(defaults) transport.agent_factory = self.fake_http.get_agent returnValue(transport) @inlineCallbacks def get_transport_with_access_token(self, access_token, **config): transport = yield self.get_transport(**config) yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY, access_token) returnValue(transport)
class _MemoryWormhole(object): """ Represent one side of a wormhole as conceived by ``MemoryWormholeServer``. """ _code = None def __init__(self, view): self._view = view self._payload = DeferredQueue() def allocate_code(self): self._code = self._view.allocate_code(self, None) def set_code(self, code): if self._code is None: self._code = code self._view.allocate_code(self, code) else: raise ValueError( "set_code used with a wormhole which already has a code") def when_code(self): if self._code is None: raise NotImplementedError( "This implementation requires allocate_code before when_code.") return succeed(self._code) get_code = when_code def get_welcome(self): return succeed('welcome') def send_message(self, payload): self._payload.put(payload) def when_received(self): if self._code is None: raise ValueError( "This implementation requires set_code or allocate_code " "before when_received.") d = self._view.wormhole_by_code(self._code, exclude=self) def got_wormhole(wormhole): return wormhole._payload.get() d.addCallback(got_wormhole) return d get_message = when_received def close(self): pass # 0.9.2 compatibility def get_code(self): self.allocate_code() return self.when_code() get = when_received
class Game(Protocol): def __init__(self, users): self.users = users self.queue = DeferredQueue() def connectionMade(self): global state connections['GAME'] = self state = "GO" self.flag = True def dataReceived(self, data): self.queue.put(data) self.queue.get().addCallback(self.ForwardData) def ForwardData(self, data): positions = pickle.loads(data) if is_client == "host": gs.player2.rect = positions["p2_rect"] gs.player2.image = pygame.transform.rotate(gs.player2.orig_image, positions["p2_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P2_Laser(gs.player2.rect.centerx, gs.player2.rect.centery, mx, my, gs) self.laser.rect.x = gs.player2.rect.centerx self.laser.rect.y = gs.player2.rect.centery # Add the laser to the lists gs.laser_list2.add(self.laser) gs.all_sprites_list.add(self.laser) else: gs.player1.rect = positions["p1_rect"] gs.player1.image = pygame.transform.rotate(gs.player1.orig_image, positions["p1_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P1_Laser(gs.player1.rect.centerx, gs.player1.rect.centery, mx, my, gs) self.laser.rect.x = gs.player1.rect.centerx self.laser.rect.y = gs.player1.rect.centery # Add the laser to the lists gs.laser_list1.add(self.laser) gs.all_sprites_list.add(self.laser) if positions["change"] == True: gs.player2.rect.x = positions["newx"] gs.player2.rect.y = positions["newy"] if self.queue.waiting > 0: self.queue.get().addCallback(self.ForwardData) def connectionLost(self, reason): print "Connection lost - goodbye!" reactor.stop() def clientConnectionLost(self, reason): print "Connection lost - goodbye!" reactor.stop()
class PatchedMessengerTransport(MessengerTransport): def __init__(self, *args, **kwargs): super(PatchedMessengerTransport, self).__init__(*args, **kwargs) self.request_queue = DeferredQueue() def request(self, method, url, data, **kwargs): d = Deferred() self.request_queue.put((d, (method, url, data), kwargs)) return d
class DataConnection(Protocol): def __init__(self): self.queue = DeferredQueue() def dataReceived(self, data): self.queue.put(data) # put data from client into queue def connectionMade(self): reactor.connectTCP(SSH_HOST, SSH_PORT, ServiceConnFactory(self)) def forwardData(self, data): self.transport.write(data) # send data to Home through data connection
class ClientProtocol(LineReceiver): def __init__(self): self.queue = DeferredQueue() def lineReceived(self, line): self.queue.put(line) def connectionLost(self, reason): self.queue.put("DONE")
class ClientConnection(Protocol): def __init__(self, conns): self.queue = DeferredQueue() self.startForwarding() self.conns = conns # client has connected to server def connectionMade(self): # limit the number of client connections to 2 if len(self.conns) >= 2: self.transport.write('two players are already playing') self.transport.loseConnection() return if not self.conns: # first client connection self.num = 0 else: # second client connection self.num = 1 print('player {} connected'.format(self.num)) self.conns.append(self) # both players are connected if len(self.conns) == 2: for c in self.conns: c.transport.write('start the game,{}'.format(c.num)) def connectionLost(self, reason): if self in self.conns: # remove this connection self.conns.remove(self) print('connection lost') print('number of connections: {}'.format(len(self.conns))) # notify the other connection if there is one # that their opponent left if self.conns: self.conns[0].transport.write('connection lost') def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): self.writeOther(data) self.queue.get().addCallback(self.forwardData) def startForwarding(self): self.queue.get().addCallback(self.forwardData) def writeOther(self, data): # 0 writes to 1 and 1 writes to 0 if len(self.conns) == 2: other = 0 if self.num else 1 self.conns[other].transport.write(data)
class DataConn(Protocol): def __init__(self): self.serv_stream_live = False self.dq = DeferredQueue() def dataReceived(self, data): if self.serv_stream_live: conndict['servConn'].transport.write(data) else: self.dq.put(data)
class ClientConnection(Protocol): def __init__(self, gs): self.queue = DeferredQueue() self.gs = gs self.gs.connection = self self.startForwarding() # player has joined the game def connectionMade(self): print('client connection made') self.transport.setTcpNoDelay(True) # send update over connection def update(self, data): self.transport.write('{}'.format(str(data))) # put the data on the queue def dataReceived(self, data): self.queue.put(data) # handle the data that was received def forwardData(self, data): if 'start the game' in data: # set which connection is which player and start the game _, num = data.split(',', 1) self.gs.set_players(num) self.gs.started = True elif self.gs.started: print('data: {}'.format(data)) # this player won if 'win' in data: self.gs.game_over_screen('win') # this player lost elif 'lose' in data: self.gs.game_over_screen('lose') # other player disconnected elif 'connection lost' in data: self.gs.text_screen('Other player has left. Click to quit.') # a move has been sent, update oppenent's position elif 'draw' in data or 'up' in data or 'down' in data or 'left' in data or 'right' in data: self.gs.opponent.receive_move_location(data, gs.grid.data) # re-add itself as callback self.queue.get().addCallback(self.forwardData) # add the callback to the deferred queue def startForwarding(self): self.queue.get().addCallback(self.forwardData) # stop running when the connection is lost def connectionLost(self, reason): if reactor.running: reactor.stop()
class FakeCellulant(object): def __init__(self): self.cellulant_sms_calls = DeferredQueue() self.fake_http = FakeHttpServer(self.handle_request) self.response = '' self.get_agent = self.fake_http.get_agent self.get = self.cellulant_sms_calls.get def handle_request(self, request): self.cellulant_sms_calls.put(request) return self.response
class Game(Protocol): def __init__(self, users): self.users = users self.queue = DeferredQueue() def connectionMade(self): global state connections['GAME'] = self state = "GO" self.flag = True def dataReceived(self, data): self.queue.put(data) self.queue.get().addCallback(self.ForwardData) def ForwardData(self, data): positions = pickle.loads(data) if is_client == "host": gs.player2.rect = positions["p2_rect"] gs.player2.image = pygame.transform.rotate(gs.player2.orig_image, positions["p2_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P2_Laser(gs.player2.rect.centerx, gs.player2.rect.centery,mx,my,gs) self.laser.rect.x = gs.player2.rect.centerx self.laser.rect.y = gs.player2.rect.centery # Add the laser to the lists gs.laser_list2.add(self.laser) gs.all_sprites_list.add(self.laser) else: gs.player1.rect = positions["p1_rect"] gs.player1.image = pygame.transform.rotate(gs.player1.orig_image, positions["p1_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P1_Laser(gs.player1.rect.centerx, gs.player1.rect.centery,mx,my,gs) self.laser.rect.x = gs.player1.rect.centerx self.laser.rect.y = gs.player1.rect.centery # Add the laser to the lists gs.laser_list1.add(self.laser) gs.all_sprites_list.add(self.laser) if positions["change"] == True: gs.player2.rect.x = positions["newx"] gs.player2.rect.y = positions["newy"] if self.queue.waiting > 0: self.queue.get().addCallback(self.ForwardData) def connectionLost(self,reason): print "Connection lost - goodbye!" reactor.stop() def clientConnectionLost(self, reason): print "Connection lost - goodbye!" reactor.stop()
class EslTransport(StringTransport): def __init__(self): StringTransport.__init__(self) self.cmds = DeferredQueue() self.esl_parser = EslParser() def write(self, data): StringTransport.write(self, data) for cmd in self.esl_parser.parse(data): self.cmds.put(cmd)
class BenchTransport(Transport): WORKER_QUEUE = DeferredQueue() @inlineCallbacks def startWorker(self): yield Transport.startWorker(self) self.message_queue = DeferredQueue() self.WORKER_QUEUE.put(self) def handle_outbound_message(self, msg): self.message_queue.put(msg)
class TestStreamingClient(VumiTestCase): def setUp(self): self.fake_http = FakeHttpServer(self.handle_request) self.request_queue = DeferredQueue() self.client = StreamingClient(self.fake_http.get_agent) self.messages_received = DeferredQueue() self.errors_received = DeferredQueue() self.disconnects_received = DeferredQueue() def reason_trapper(reason): if reason.trap(ResponseDone): self.disconnects_received.put(reason.getErrorMessage()) self.receiver = self.client.stream( Message, self.messages_received.put, self.errors_received.put, "http://vumi-go-api.example.com/", on_disconnect=reason_trapper) def handle_request(self, request): self.request_queue.put(request) return NOT_DONE_YET def test_default_agent_factory(self): """ If `None` is passed as the `agent_factory`, `Agent` is used instead. """ self.assertNotIsInstance(self.client.agent, Agent) self.assertIsInstance(StreamingClient(None).agent, Agent) self.assertIsInstance(StreamingClient().agent, Agent) @inlineCallbacks def test_callback_on_disconnect(self): req = yield self.request_queue.get() req.write( '%s\n' % (Message(foo='bar').to_json().encode('utf-8'),)) req.finish() message = yield self.messages_received.get() self.assertEqual(message['foo'], 'bar') reason = yield self.disconnects_received.get() # this is the error message we get when a ResponseDone is raised # which happens when the remote server closes the connection. self.assertEqual(reason, 'Response body fully received') @inlineCallbacks def test_invalid_json(self): req = yield self.request_queue.get() req.write("Hello\n") req.finish() err = yield self.assertFailure( self.errors_received.get(), VumiBridgeInvalidJsonError) self.assertEqual(err.args, ("Hello",))
class TestTransport(TestCase): @inlineCallbacks def setUp(self): DelayedCall.debug = True self.ok_transport_calls = DeferredQueue() self.mock_service = MockHttpServer(self.handle_request) yield self.mock_service.start() config = { 'transport_name': 'test_ok_transport', 'transport_type': 'ok', 'ussd_string_prefix': '', 'web_path': "foo", 'web_port': 0, 'url': self.mock_service.url, 'username': '******', 'password': '******', } self.worker = get_stubbed_worker(OkTransport, config) self.broker = self.worker._amqp_client.broker yield self.worker.startWorker() self.worker_url = self.worker.get_transport_url() @inlineCallbacks def tearDown(self): yield self.worker.stopWorker() yield self.mock_service.stop() def handle_request(self, request): self.ok_transport_calls.put(request) return '' @inlineCallbacks def test_health(self): result = yield http_request(self.worker_url + "health", "", method='GET') self.assertEqual(json.loads(result), { 'pending_requests': 0 }) @inlineCallbacks def test_inbound(self): d = http_request(self.worker_url + "foo", '', method='GET') msg, = yield self.broker.wait_messages("vumi", "test_ok_transport.inbound", 1) payload = msg.payload tum = TransportUserMessage(**payload) rep = tum.reply("OK") self.broker.publish_message("vumi", "test_ok_transport.outbound", rep) response = yield d self.assertEqual(response, 'OK')
class ClientConnection(Protocol): def __init__(self, server): self.server = server self.queue = DeferredQueue() def dataReceived(self, data): self.queue.put(data) # put data from client into queue def connectionMade(self): self.server.newConnect() if self.server.check == 1: # only listen to a port once reactor.listenTCP(DATA_PORT, DataConnFactory(self)) def forwardData(self, data): self.transport.write(data) # forward data to client def connectionLost(self, reason): self.server.check = 0
class QueueWrapper(object): """ Wrap a queue to have notifications when get is called on this particular queue. """ def __init__(self, queue): self._real_queue_get = queue.get self.event_queue = DeferredQueue() queue.get = self.get def get(self, timeout=None): self.event_queue.put(None) return self._real_queue_get(timeout)
class ClientConn(LineReceiver): def __init__(self): self.data_stream_live = False self.dq = DeferredQueue() def connectionMade(self): reactor.listenTCP(32002, DataConnFactory()) conndict['cmdConn'].sendLine("new client") def dataReceived(self, data): if self.data_stream_live: conndict['dataConn'].transport.write(data) else: self.dq.put(data)
class TestStreamingClient(VumiTestCase): @inlineCallbacks def setUp(self): self.mock_server = MockHttpServer(self.handle_request) self.add_cleanup(self.mock_server.stop) yield self.mock_server.start() self.url = self.mock_server.url self.client = StreamingClient() self.messages_received = DeferredQueue() self.errors_received = DeferredQueue() self.disconnects_received = DeferredQueue() def reason_trapper(reason): if reason.trap(ResponseDone): self.disconnects_received.put(reason.getErrorMessage()) self.receiver = self.client.stream( Message, self.messages_received.put, self.errors_received.put, self.url, on_disconnect=reason_trapper) def handle_request(self, request): self.mock_server.queue.put(request) return NOT_DONE_YET @inlineCallbacks def test_callback_on_disconnect(self): req = yield self.mock_server.queue.get() req.write( '%s\n' % (Message(foo='bar').to_json().encode('utf-8'),)) req.finish() message = yield self.messages_received.get() self.assertEqual(message['foo'], 'bar') reason = yield self.disconnects_received.get() # this is the error message we get when a ResponseDone is raised # which happens when the remote server closes the connection. self.assertEqual(reason, 'Response body fully received') @inlineCallbacks def test_invalid_json(self): req = yield self.mock_server.queue.get() req.write("Hello\n") req.finish() try: yield self.errors_received.get() except VumiBridgeInvalidJsonError, e: self.assertEqual(e.args, ("Hello",)) else:
class TcpProxyProtocol(Protocol, object): """A simple TCP proxy""" def __init__(self): """Create a new TCP proxy. `self.server_queue` contains messages from end server to client. `self.client_queue` contains messages from client to end server. """ self.server_queue = DeferredQueue() self.client_queue = DeferredQueue() self.server_queue.get().addCallback(self.serverQueueCallback) def connectServer(self, hostname, port): """Tell the proxy what the end server is and start the connection. The messages in `self.client_queue` will automatically be consumed. This method should only be called once. :param str hostname: :param int port: """ endpoint = TCP4ClientEndpoint(reactor, hostname, port) protocol = ServerProtocol( self.server_queue, self.client_queue) connectProtocol(endpoint, protocol) def serverQueueCallback(self, data): """A callback for `self.server_queue` :param str data: data from server queue """ if data is False: self.transport.loseConnection() return self.transport.write(data) self.server_queue.get().addCallback(self.serverQueueCallback) def dataReceived(self, data): """Received data from client, put into client queue """ self.client_queue.put(data) def connectionLost(self, why): """Client closed connection, or some other issue. close connection to server """ self.client_queue.put(False)
class home_DataConn(protocol.Protocol): def __init__(self): print 'home listening for data connection from work' self.queue = DeferredQueue() def connectionMade(self): home_connections['data_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): home_connections['client_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class work_ServiceConn(protocol.Protocol): def __init__(self): print 'service connection established' self.queue = DeferredQueue() def connectionMade(self): work_connections['service_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): work_connections['data_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class ClientProtocol(LineReceiver): def __init__(self): self.queue = DeferredQueue() self.connect_d = Deferred() self.disconnect_d = Deferred() def connectionMade(self): self.connect_d.callback(None) def lineReceived(self, line): self.queue.put(line) def connectionLost(self, reason): self.queue.put("DONE") self.disconnect_d.callback(None)
class TestStreamingClient(VumiTestCase): @inlineCallbacks def setUp(self): self.mock_server = MockHttpServer(self.handle_request) self.add_cleanup(self.mock_server.stop) yield self.mock_server.start() self.url = self.mock_server.url self.client = StreamingClient() self.messages_received = DeferredQueue() self.errors_received = DeferredQueue() self.disconnects_received = DeferredQueue() def reason_trapper(reason): if reason.trap(ResponseDone): self.disconnects_received.put(reason.getErrorMessage()) self.receiver = self.client.stream(Message, self.messages_received.put, self.errors_received.put, self.url, on_disconnect=reason_trapper) def handle_request(self, request): self.mock_server.queue.put(request) return NOT_DONE_YET @inlineCallbacks def test_callback_on_disconnect(self): req = yield self.mock_server.queue.get() req.write('%s\n' % (Message(foo='bar').to_json().encode('utf-8'), )) req.finish() message = yield self.messages_received.get() self.assertEqual(message['foo'], 'bar') reason = yield self.disconnects_received.get() # this is the error message we get when a ResponseDone is raised # which happens when the remote server closes the connection. self.assertEqual(reason, 'Response body fully received') @inlineCallbacks def test_invalid_json(self): req = yield self.mock_server.queue.get() req.write("Hello\n") req.finish() try: yield self.errors_received.get() except VumiBridgeInvalidJsonError, e: self.assertEqual(e.args, ("Hello", )) else:
def do_work2(db): queue = DeferredQueue() defs = [] nworkers = len(entries)//5+1 if nworkers > self.nworkers: nworkers = self.nworkers for i in range(nworkers): d = self.fetch(db,queue) d.addErrback(log.err,"fetch()") defs.append(d) workers = set() for id,inum,typ in entries: if not self.running: break trace('copyrun',"%d: %s",inum,typ) self.last_entry = id if typ == 'd': def dt(inum): path = build_path(self.fs.store,inum, create=False) try: os.unlink(path) except EnvironmentError as e: if e.errno != errno.ENOENT: raise yield deferToThread(dt,inum) else: inode = SqlInode(self.fs,inum) yield inode._load(db) if typ == 'f': if inum in workers: trace('copyrun',"%d: in workers",inum,typ) continue workers.add(inum) queue.put((id,inode)) elif typ == 't': if inode.cache: yield inode.cache.trim(inode.size) else: raise RuntimeError("Typ '%s' not found (inode %d)" % (typ,inum)) continue for i in range(nworkers): queue.put(None) yield DeferredList(defs)
class QueuePoller(object): implements(IPoller) def __init__(self, config): self.config = config self.update_projects() self.dq = DeferredQueue(size=1) @inlineCallbacks def poll(self): if self.dq.pending: return for p, q in self.queues.iteritems(): c = yield maybeDeferred(q.count) if c: msg = yield maybeDeferred(q.pop) returnValue(self.dq.put(self._message(msg, p))) def next(self): return self.dq.get() def update_projects(self): self.queues = get_spider_queues(self.config) def _message(self, queue_msg, project): d = queue_msg.copy() d['_project'] = project d['_spider'] = d.pop('name') return d
class QueuePoller(object): implements(IPoller) def __init__(self, settings): self.q = RedisSpiderQueue(settings) self.dq = DeferredQueue(size=1) @inlineCallbacks def poll(self): if self.dq.pending: return c = yield maybeDeferred(self.q.count) if c: msg = yield maybeDeferred(self.q.pop) returnValue(self.dq.put(self._message(msg))) def next(self): return self.dq.get() def _message(self, queue_msg): d = queue_msg.copy() d['_project'] = SCRAPY_PROJECT d['_spider'] = d.pop('name') d['_job'] = d.pop('jobid', uuid.uuid1().hex) return d
class MockXmlOverTcpServer(MockServer): def __init__(self): self.responses = {} self.received_queue = DeferredQueue() def wait_for_data(self): return self.received_queue.get() def send_data(self, data): self.transport.write(data) def dataReceived(self, data): response = self.responses.get(data) if response is not None: self.transport.write(response) self.received_queue.put(data)
class QueuePoller(object): def __init__(self, config): self.config = config self.update_projects() self.dq = DeferredQueue() @inlineCallbacks def poll(self): if not self.dq.waiting: return for p, q in iteritems(self.queues): c = yield maybeDeferred(q.count) if c: msg = yield maybeDeferred(q.pop) if msg is not None: # In case of a concurrently accessed queue returnValue(self.dq.put(self._message(msg, p))) def next(self): return self.dq.get() def update_projects(self): self.queues = get_spider_queues(self.config) def _message(self, queue_msg, project): d = queue_msg.copy() d['_project'] = project d['_spider'] = d.pop('name') return d
class TcpProxyProtocol(Protocol, object): """A simple TCP proxy""" def __init__(self): """Create a new TCP proxy. `self.server_queue` contains messages from end server to client. `self.client_queue` contains messages from client to end server. """ self.server_queue = DeferredQueue() self.client_queue = DeferredQueue() self.server_queue.get().addCallback(self.serverQueueCallback) def connectServer(self, hostname, port): """Tell the proxy what the end server is and start the connection. The messages in `self.client_queue` will automatically be consumed. This method should only be called once. :param str hostname: :param int port: """ endpoint = TCP4ClientEndpoint(reactor, hostname, port) protocol = ServerProtocol(self.server_queue, self.client_queue) connectProtocol(endpoint, protocol) def serverQueueCallback(self, data): """A callback for `self.server_queue` :param str data: data from server queue """ if data is False: self.transport.loseConnection() return self.transport.write(data) self.server_queue.get().addCallback(self.serverQueueCallback) def dataReceived(self, data): """Received data from client, put into client queue """ self.client_queue.put(data) def connectionLost(self, why): """Client closed connection, or some other issue. close connection to server """ self.client_queue.put(False)
class ConsumerQueue(object): def __init__(self, stop_on_error=False, empty=None): self.stop_on_error = stop_on_error self.empty = empty self.queue = DeferredQueue() self.size = 0 self.running = True self._deferred = Deferred() def _consume_next(self, *args): if not self.running: return self._deferred = self.queue.get() self._deferred.addCallbacks(self._consumer, self._error) def _consumer(self, item): self.size -= 1 r = self.consume(item) if self.size == 0 and self.empty is not None: self.empty() if isinstance(r, Deferred): r.addCallbacks(self._consume_next, self._consume_next) else: self._consume_next() def _error(self, fail): self.error(fail) if not self.stop_on_error: self._consume_next() def add(self, item): self.size += 1 self.queue.put(item) def consume(self, item): raise NotImplementedError def error(self, fail): raise NotImplementedError def start(self): self.running = True self._consume_next() def stop(self): self.running = False self._deferred.cancel()
class ToyXmlOverTcpClient(XmlOverTcpClient): _PACKET_RECEIVED_HANDLERS = {'DummyPacket': 'dummy_packet_received'} def __init__(self): XmlOverTcpClient.__init__(self, 'root', 'toor', '1029384756') self.PACKET_RECEIVED_HANDLERS.update(self._PACKET_RECEIVED_HANDLERS) self.received_dummy_packets = [] self.received_data_request_packets = [] self.disconnected = False self.session_id_counter = count() self.generated_session_ids = [] self.request_id_counter = count() self.generated_request_ids = [] self.received_queue = DeferredQueue() def wait_for_data(self): return self.received_queue.get() def connectionMade(self): pass def dataReceived(self, data): XmlOverTcpClient.dataReceived(self, data) self.received_queue.put(data) def dummy_packet_received(self, session_id, params): self.received_dummy_packets.append((session_id, params)) def data_request_received(self, session_id, params): self.received_data_request_packets.append((session_id, params)) def disconnect(self): self.disconnected = True @classmethod def session_id_from_nr(cls, nr): return cls.serialize_header_field(nr, cls.SESSION_ID_HEADER_SIZE) def gen_session_id(self): return self.session_id_from_nr(next(self.session_id_counter)) def gen_request_id(self): return str(next(self.request_id_counter))
class DataConnection(Protocol): def __init__(self): self.q = DeferredQueue() def connectionMade(self): reactor.connectTCP('student02.cse.nd.edu', 22, GenericClientFactory(ServiceConnection, self)) def dataReceived(self, data): self.q.put(data) def start_forwarding_data(self, service): self.service = service self.q.get().addCallback(self.forward_data) def forward_data(self, data): self.service.transport.write(data) self.q.get().addCallback(self.forward_data) # Mayhaps this one
class ClientConnection(Protocol): def __init__(self, cmd): self.cmd = cmd self.q = DeferredQueue() def connectionMade(self): reactor.listenTCP(DATA_PORT, GenericFactory(DataConnection, self)) self.cmd.transport.write('start data connection') def dataReceived(self, data): self.q.put(data) def start_forwarding_client_data(self, data): self.data = data self.q.get().addCallback(self.forward_data) def forward_data(self, data): self.data.transport.write(data) self.q.get().addCallback(self.forward_data)
class ProtocolP2(Protocol): def __init__(self, friend): self.friend = friend self.data_queue = DeferredQueue() def connectionMade(self): logging.debug('P2 connection made') def dataReceived(self, data): logging.debug('P2 got data "%s"', data) self.data_queue.put(data) def start_forwarding(self): self.data_queue.get().addCallback(self.send_to_friend) def send_to_friend(self, data): logging.debug('P2 sent "%s" to friend', data) self.friend.transport.write(data) self.data_queue.get().addCallback(self.send_to_friend)