class PostgresListenerServiceSpy(PostgresListenerService):
"""Save received notifies `captured_notifies` before processing them.."""
HANDLE_NOTIFY_DELAY = CHANNEL_REGISTRAR_DELAY = 0
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Captured notifications from the database will go here.
self._captured_notifies = DeferredQueue()
# Change notifications to a frozenset. This makes sure that the system
# message does not go into the queue. Instead it should call the
# handler directly in `doRead`.
self.notifications = frozenset()
def _process_notifies(self):
for notify in self.connection.connection.notifies:
self._captured_notifies.put(notify)
super()._process_notifies()
@inlineCallbacks
def wait_notification(self, channel):
"""Wait for a notification to be received."""
while True:
notice = yield self._captured_notifies.get()
if notice.channel == channel:
returnValue(notice)
class OmciRxProxy(pb.Root):
def __init__(self):
self.pb_server_ip = '192.168.24.20' # registry('main').get_args().external_host_address
self.pb_server_port = 24497
self.pb_server_factory = pb.PBServerFactory(self)
# start PB server
self.listen_port = reactor.listenTCP(self.pb_server_port,
self.pb_server_factory)
self.omci_rx_queue = DeferredQueue()
log.info('PB-server-started-on-port', port=self.pb_server_port)
def get_ip(self):
return self.pb_server_ip
def get_port(self):
return self.pb_server_port
def get_host(self):
return self.listen_port.getHost()
def remote_echo(self, pkt_type, pon, onu, port, crc_ok, msg_size,
msg_data):
log.info('received-omci-msg',
pkt_type=pkt_type,
pon_id=pon,
onu_id=onu,
port_id=port,
crc_ok=crc_ok,
msg_size=msg_size,
msg_data=hexify(msg_data))
self.omci_rx_queue.put((onu, msg_data))
def receive(self):
return self.omci_rx_queue.get()
class SSMIServerProtocol(Protocol):
delimiter = TruteqTransportProtocol.delimiter
def __init__(self):
self.receive_queue = DeferredQueue()
self._buf = b""
def dataReceived(self, data):
self._buf += data
self.parse_commands()
def parse_commands(self):
while self.delimiter in self._buf:
line, _, self._buf = self._buf.partition(self.delimiter)
if line:
self.receive_queue.put(SSMIRequest.parse(line))
def send(self, command):
self.transport.write(str(command))
self.transport.write(self.delimiter)
return wait0()
def receive(self):
return self.receive_queue.get()
def disconnect(self):
self.transport.loseConnection()
class MockHttpServer(object):
"""
NOTE: This is deprecated.
Please use :class:`vumi.tests.http_helpers.MockHttpHelper` instead.
"""
def __init__(self, handler=None):
self.queue = DeferredQueue()
self._handler = handler or self.handle_request
self._webserver = None
self.addr = None
self.url = None
def handle_request(self, request):
self.queue.put(request)
@inlineCallbacks
def start(self):
root = MockResource(self._handler)
site_factory = LogFilterSite(root)
self._webserver = yield reactor.listenTCP(
0, site_factory, interface='127.0.0.1')
self.addr = self._webserver.getHost()
self.url = "http://%s:%s/" % (self.addr.host, self.addr.port)
@inlineCallbacks
def stop(self):
yield self._webserver.stopListening()
yield self._webserver.loseConnection()
class SSMIServerProtocol(Protocol):
delimiter = TruteqTransportProtocol.delimiter
def __init__(self):
self.receive_queue = DeferredQueue()
self._buf = b""
def dataReceived(self, data):
self._buf += data
self.parse_commands()
def parse_commands(self):
while self.delimiter in self._buf:
line, _, self._buf = self._buf.partition(self.delimiter)
if line:
self.receive_queue.put(SSMIRequest.parse(line))
def send(self, command):
self.transport.write(str(command))
self.transport.write(self.delimiter)
return wait0()
def receive(self):
return self.receive_queue.get()
def disconnect(self):
self.transport.loseConnection()
class TestHTTPClientBase(TestCase):
# TODO: Run client tests synchronously with treq.testing tools (#38)
run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=0.1)
def setUp(self):
super(TestHTTPClientBase, self).setUp()
self.requests = DeferredQueue()
self.fake_server = FakeHttpServer(self.handle_request)
fake_client = treq_HTTPClient(self.fake_server.get_agent())
self.client = self.get_client(fake_client)
# Spin the reactor once at the end of each test to clean up any
# cancelled deferreds
self.addCleanup(wait0)
def handle_request(self, request):
self.requests.put(request)
return NOT_DONE_YET
def get_client(self, client):
"""To be implemented by subclass"""
raise NotImplementedError()
def uri(self, path):
return '%s%s' % (self.client.url, path,)
def cleanup_d(self, d):
self.addCleanup(lambda: d)
return d
class OmciProxy(pb.Root):
def __init__(self):
reactor.listenTCP(24497, pb.PBServerFactory(self))
self.remote = None
self.response_queue = DeferredQueue()
@inlineCallbacks
def connect(self):
factory = pb.PBClientFactory()
reactor.connectTCP("10.111.101.206", 24498, factory)
self.remote = yield factory.getRootObject()
print 'connected'
yield self.remote.callRemote("setRemote", port=24496)
def remote_echo(self, pkt_type, pon, onu, port, crc, size, data):
print "Packet Type:", pkt_type
print "PON:", pon
print "ONU ID:", onu
print "Port:", port
print "CRC OK:", crc
print "Packet Size:", size
print "received:", hexify(data)
self.response_queue.put(data)
@inlineCallbacks
def send_omci(self, msg):
if isinstance(msg, Packet):
msg = str(msg)
try:
print ' sending:', msg
yield self.remote.callRemote("send_omci", 0, 0, 1, msg)
print 'msg sent'
except Exception, e:
print >> sys.stderr, 'Blew up:', str(e)
class TestHTTPClientBase(TestCase):
# TODO: Run client tests synchronously with treq.testing tools (#38)
run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=0.1)
def setUp(self):
super(TestHTTPClientBase, self).setUp()
self.requests = DeferredQueue()
self.fake_server = FakeHttpServer(self.handle_request)
fake_client = treq_HTTPClient(self.fake_server.get_agent())
self.client = self.get_client(fake_client)
# Spin the reactor once at the end of each test to clean up any
# cancelled deferreds
self.addCleanup(wait0)
def handle_request(self, request):
self.requests.put(request)
return NOT_DONE_YET
def get_client(self, client):
"""To be implemented by subclass"""
raise NotImplementedError()
def uri(self, path):
return '%s%s' % (self.client.url, path,)
def cleanup_d(self, d):
self.addCleanup(lambda: d)
return d
class WeChatTestCase(VumiTestCase):
def setUp(self):
self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
self.request_queue = DeferredQueue()
self.mock_server = MockHttpServer(self.handle_api_request)
self.add_cleanup(self.mock_server.stop)
return self.mock_server.start()
def handle_api_request(self, request):
self.request_queue.put(request)
return NOT_DONE_YET
def get_transport(self, **config):
defaults = {
'api_url': self.mock_server.url,
'auth_token': 'token',
'twisted_endpoint': 'tcp:0',
'wechat_appid': 'appid',
'wechat_secret': 'secret',
'embed_user_profile': False,
}
defaults.update(config)
return self.tx_helper.get_transport(defaults)
@inlineCallbacks
def get_transport_with_access_token(self, access_token, **config):
transport = yield self.get_transport(**config)
yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
access_token)
returnValue(transport)
class MockHttpServer(object):
"""
NOTE: This is deprecated.
Please use :class:`vumi.tests.http_helpers.MockHttpHelper` instead.
"""
def __init__(self, handler=None):
self.queue = DeferredQueue()
self._handler = handler or self.handle_request
self._webserver = None
self.addr = None
self.url = None
def handle_request(self, request):
self.queue.put(request)
@inlineCallbacks
def start(self):
root = MockResource(self._handler)
site_factory = LogFilterSite(root)
self._webserver = yield reactor.listenTCP(0,
site_factory,
interface='127.0.0.1')
self.addr = self._webserver.getHost()
self.url = "http://%s:%s/" % (self.addr.host, self.addr.port)
@inlineCallbacks
def stop(self):
yield self._webserver.stopListening()
yield self._webserver.loseConnection()
class ProcessPool(object): def __init__(self, count=10): self.limiter = DeferredSemaphore(count) self.processes = [spawnProcess() for _ in xrange(count)] self.workQueue = DeferredQueue() for process in self.processes: process.onconnect.addCallback(self._prepareForWork) @inlineCallbacks def _prepareForWork(self, proto): deferred, func, args = yield self.workQueue.get() proto.queueWork(deferred, func, *args) def requeue(result): self._prepareForWork(proto) return result deferred.addCallback(requeue) returnValue(proto) def queueWork(self, function, *args): resultDeferred = Deferred() innerDeferred = Deferred() self.workQueue.put((innerDeferred, function, args)) def callResult(obj): resultDeferred.callback(obj) return obj innerDeferred.addCallback(callResult) return resultDeferred def stop(self): for process in self.processes: process.protocol.kill() process.transport.loseConnection()
class RateLimitAgent(object):
def __init__(self):
self.pool = HTTPConnectionPool(reactor, persistent=True)
self.pool.maxPersistentPerHost = 4
self.pool._factory = QuietHTTP11ClientFactory
self.contextFactory = WhitelistContextFactory()
self.agent = Agent(reactor, pool=self.pool, contextFactory=self.contextFactory)
self.queue = DeferredQueue()
self.getRequest()
def request(self, method, url, headers, body):
finished = Deferred()
self.queue.put((finished, method, url, headers, body))
return finished
def getRequest(self):
d = self.queue.get()
d.addCallback(self.gotRequest)
def gotRequest(self, request):
finished, method, url, headers, body = request
d = self.agent.request(method, bytes(url), headers, body)
d.addCallback(self.cbRequest, finished)
d.addErrback(self.ebRequest, finished)
def cbRequest(self, response, finished):
finished.callback(response)
reactor.callLater(1.3, self.getRequest)
def ebRequest(self, failure, finished):
finished.errback(failure)
reactor.callLater(1.3, self.getRequest)
class WeChatTestCase(VumiTestCase):
def setUp(self):
self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
self.request_queue = DeferredQueue()
self.fake_http = FakeHttpServer(self.handle_api_request)
self.api_url = 'https://api.wechat.com/cgi-bin/'
def handle_api_request(self, request):
self.assertEqual(request.path[:len(self.api_url)], self.api_url)
self.request_queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def get_transport(self, **config):
defaults = {
'auth_token': 'token',
'twisted_endpoint': 'tcp:0',
'wechat_appid': 'appid',
'wechat_secret': 'secret',
'embed_user_profile': False,
'publish_status': True,
}
defaults.update(config)
transport = yield self.tx_helper.get_transport(defaults)
transport.agent_factory = self.fake_http.get_agent
returnValue(transport)
@inlineCallbacks
def get_transport_with_access_token(self, access_token, **config):
transport = yield self.get_transport(**config)
yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
access_token)
returnValue(transport)
class WeChatTestCase(VumiTestCase):
def setUp(self):
self.tx_helper = self.add_helper(TransportHelper(WeChatTransport))
self.request_queue = DeferredQueue()
self.fake_http = FakeHttpServer(self.handle_api_request)
self.api_url = 'https://api.wechat.com/cgi-bin/'
def handle_api_request(self, request):
self.assertEqual(request.path[:len(self.api_url)], self.api_url)
self.request_queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def get_transport(self, **config):
defaults = {
'auth_token': 'token',
'twisted_endpoint': 'tcp:0',
'wechat_appid': 'appid',
'wechat_secret': 'secret',
'embed_user_profile': False,
}
defaults.update(config)
transport = yield self.tx_helper.get_transport(defaults)
transport.agent_factory = self.fake_http.get_agent
returnValue(transport)
@inlineCallbacks
def get_transport_with_access_token(self, access_token, **config):
transport = yield self.get_transport(**config)
yield transport.redis.set(WeChatTransport.ACCESS_TOKEN_KEY,
access_token)
returnValue(transport)
class _MemoryWormhole(object):
"""
Represent one side of a wormhole as conceived by ``MemoryWormholeServer``.
"""
_code = None
def __init__(self, view):
self._view = view
self._payload = DeferredQueue()
def allocate_code(self):
self._code = self._view.allocate_code(self, None)
def set_code(self, code):
if self._code is None:
self._code = code
self._view.allocate_code(self, code)
else:
raise ValueError(
"set_code used with a wormhole which already has a code")
def when_code(self):
if self._code is None:
raise NotImplementedError(
"This implementation requires allocate_code before when_code.")
return succeed(self._code)
get_code = when_code
def get_welcome(self):
return succeed('welcome')
def send_message(self, payload):
self._payload.put(payload)
def when_received(self):
if self._code is None:
raise ValueError(
"This implementation requires set_code or allocate_code "
"before when_received.")
d = self._view.wormhole_by_code(self._code, exclude=self)
def got_wormhole(wormhole):
return wormhole._payload.get()
d.addCallback(got_wormhole)
return d
get_message = when_received
def close(self):
pass
# 0.9.2 compatibility
def get_code(self):
self.allocate_code()
return self.when_code()
get = when_received
class Game(Protocol):
def __init__(self, users):
self.users = users
self.queue = DeferredQueue()
def connectionMade(self):
global state
connections['GAME'] = self
state = "GO"
self.flag = True
def dataReceived(self, data):
self.queue.put(data)
self.queue.get().addCallback(self.ForwardData)
def ForwardData(self, data):
positions = pickle.loads(data)
if is_client == "host":
gs.player2.rect = positions["p2_rect"]
gs.player2.image = pygame.transform.rotate(gs.player2.orig_image,
positions["p2_angle"])
if positions["firing"] == True:
mx = positions["mouse_x"]
my = positions["mouse_y"]
self.laser = P2_Laser(gs.player2.rect.centerx,
gs.player2.rect.centery, mx, my, gs)
self.laser.rect.x = gs.player2.rect.centerx
self.laser.rect.y = gs.player2.rect.centery
# Add the laser to the lists
gs.laser_list2.add(self.laser)
gs.all_sprites_list.add(self.laser)
else:
gs.player1.rect = positions["p1_rect"]
gs.player1.image = pygame.transform.rotate(gs.player1.orig_image,
positions["p1_angle"])
if positions["firing"] == True:
mx = positions["mouse_x"]
my = positions["mouse_y"]
self.laser = P1_Laser(gs.player1.rect.centerx,
gs.player1.rect.centery, mx, my, gs)
self.laser.rect.x = gs.player1.rect.centerx
self.laser.rect.y = gs.player1.rect.centery
# Add the laser to the lists
gs.laser_list1.add(self.laser)
gs.all_sprites_list.add(self.laser)
if positions["change"] == True:
gs.player2.rect.x = positions["newx"]
gs.player2.rect.y = positions["newy"]
if self.queue.waiting > 0:
self.queue.get().addCallback(self.ForwardData)
def connectionLost(self, reason):
print "Connection lost - goodbye!"
reactor.stop()
def clientConnectionLost(self, reason):
print "Connection lost - goodbye!"
reactor.stop()
class PatchedMessengerTransport(MessengerTransport):
def __init__(self, *args, **kwargs):
super(PatchedMessengerTransport, self).__init__(*args, **kwargs)
self.request_queue = DeferredQueue()
def request(self, method, url, data, **kwargs):
d = Deferred()
self.request_queue.put((d, (method, url, data), kwargs))
return d
class DataConnection(Protocol): def __init__(self): self.queue = DeferredQueue() def dataReceived(self, data): self.queue.put(data) # put data from client into queue def connectionMade(self): reactor.connectTCP(SSH_HOST, SSH_PORT, ServiceConnFactory(self)) def forwardData(self, data): self.transport.write(data) # send data to Home through data connection
class ClientProtocol(LineReceiver):
def __init__(self):
self.queue = DeferredQueue()
def lineReceived(self, line):
self.queue.put(line)
def connectionLost(self, reason):
self.queue.put("DONE")
class ClientConnection(Protocol):
def __init__(self, conns):
self.queue = DeferredQueue()
self.startForwarding()
self.conns = conns
# client has connected to server
def connectionMade(self):
# limit the number of client connections to 2
if len(self.conns) >= 2:
self.transport.write('two players are already playing')
self.transport.loseConnection()
return
if not self.conns:
# first client connection
self.num = 0
else:
# second client connection
self.num = 1
print('player {} connected'.format(self.num))
self.conns.append(self)
# both players are connected
if len(self.conns) == 2:
for c in self.conns:
c.transport.write('start the game,{}'.format(c.num))
def connectionLost(self, reason):
if self in self.conns:
# remove this connection
self.conns.remove(self)
print('connection lost')
print('number of connections: {}'.format(len(self.conns)))
# notify the other connection if there is one
# that their opponent left
if self.conns:
self.conns[0].transport.write('connection lost')
def dataReceived(self, data):
self.queue.put(data)
def forwardData(self, data):
self.writeOther(data)
self.queue.get().addCallback(self.forwardData)
def startForwarding(self):
self.queue.get().addCallback(self.forwardData)
def writeOther(self, data):
# 0 writes to 1 and 1 writes to 0
if len(self.conns) == 2:
other = 0 if self.num else 1
self.conns[other].transport.write(data)
class ClientProtocol(LineReceiver):
def __init__(self):
self.queue = DeferredQueue()
def lineReceived(self, line):
self.queue.put(line)
def connectionLost(self, reason):
self.queue.put("DONE")
class DataConn(Protocol): def __init__(self): self.serv_stream_live = False self.dq = DeferredQueue() def dataReceived(self, data): if self.serv_stream_live: conndict['servConn'].transport.write(data) else: self.dq.put(data)
class PatchedMessengerTransport(MessengerTransport):
def __init__(self, *args, **kwargs):
super(PatchedMessengerTransport, self).__init__(*args, **kwargs)
self.request_queue = DeferredQueue()
def request(self, method, url, data, **kwargs):
d = Deferred()
self.request_queue.put((d, (method, url, data), kwargs))
return d
class ClientConnection(Protocol):
def __init__(self, gs):
self.queue = DeferredQueue()
self.gs = gs
self.gs.connection = self
self.startForwarding()
# player has joined the game
def connectionMade(self):
print('client connection made')
self.transport.setTcpNoDelay(True)
# send update over connection
def update(self, data):
self.transport.write('{}'.format(str(data)))
# put the data on the queue
def dataReceived(self, data):
self.queue.put(data)
# handle the data that was received
def forwardData(self, data):
if 'start the game' in data:
# set which connection is which player and start the game
_, num = data.split(',', 1)
self.gs.set_players(num)
self.gs.started = True
elif self.gs.started:
print('data: {}'.format(data))
# this player won
if 'win' in data:
self.gs.game_over_screen('win')
# this player lost
elif 'lose' in data:
self.gs.game_over_screen('lose')
# other player disconnected
elif 'connection lost' in data:
self.gs.text_screen('Other player has left. Click to quit.')
# a move has been sent, update oppenent's position
elif 'draw' in data or 'up' in data or 'down' in data or 'left' in data or 'right' in data:
self.gs.opponent.receive_move_location(data, gs.grid.data)
# re-add itself as callback
self.queue.get().addCallback(self.forwardData)
# add the callback to the deferred queue
def startForwarding(self):
self.queue.get().addCallback(self.forwardData)
# stop running when the connection is lost
def connectionLost(self, reason):
if reactor.running:
reactor.stop()
class FakeCellulant(object):
def __init__(self):
self.cellulant_sms_calls = DeferredQueue()
self.fake_http = FakeHttpServer(self.handle_request)
self.response = ''
self.get_agent = self.fake_http.get_agent
self.get = self.cellulant_sms_calls.get
def handle_request(self, request):
self.cellulant_sms_calls.put(request)
return self.response
class Game(Protocol): def __init__(self, users): self.users = users self.queue = DeferredQueue() def connectionMade(self): global state connections['GAME'] = self state = "GO" self.flag = True def dataReceived(self, data): self.queue.put(data) self.queue.get().addCallback(self.ForwardData) def ForwardData(self, data): positions = pickle.loads(data) if is_client == "host": gs.player2.rect = positions["p2_rect"] gs.player2.image = pygame.transform.rotate(gs.player2.orig_image, positions["p2_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P2_Laser(gs.player2.rect.centerx, gs.player2.rect.centery,mx,my,gs) self.laser.rect.x = gs.player2.rect.centerx self.laser.rect.y = gs.player2.rect.centery # Add the laser to the lists gs.laser_list2.add(self.laser) gs.all_sprites_list.add(self.laser) else: gs.player1.rect = positions["p1_rect"] gs.player1.image = pygame.transform.rotate(gs.player1.orig_image, positions["p1_angle"]) if positions["firing"] == True: mx = positions["mouse_x"] my = positions["mouse_y"] self.laser = P1_Laser(gs.player1.rect.centerx, gs.player1.rect.centery,mx,my,gs) self.laser.rect.x = gs.player1.rect.centerx self.laser.rect.y = gs.player1.rect.centery # Add the laser to the lists gs.laser_list1.add(self.laser) gs.all_sprites_list.add(self.laser) if positions["change"] == True: gs.player2.rect.x = positions["newx"] gs.player2.rect.y = positions["newy"] if self.queue.waiting > 0: self.queue.get().addCallback(self.ForwardData) def connectionLost(self,reason): print "Connection lost - goodbye!" reactor.stop() def clientConnectionLost(self, reason): print "Connection lost - goodbye!" reactor.stop()
class EslTransport(StringTransport):
def __init__(self):
StringTransport.__init__(self)
self.cmds = DeferredQueue()
self.esl_parser = EslParser()
def write(self, data):
StringTransport.write(self, data)
for cmd in self.esl_parser.parse(data):
self.cmds.put(cmd)
class FakeCellulant(object):
def __init__(self):
self.cellulant_sms_calls = DeferredQueue()
self.fake_http = FakeHttpServer(self.handle_request)
self.response = ''
self.get_agent = self.fake_http.get_agent
self.get = self.cellulant_sms_calls.get
def handle_request(self, request):
self.cellulant_sms_calls.put(request)
return self.response
class BenchTransport(Transport):
WORKER_QUEUE = DeferredQueue()
@inlineCallbacks
def startWorker(self):
yield Transport.startWorker(self)
self.message_queue = DeferredQueue()
self.WORKER_QUEUE.put(self)
def handle_outbound_message(self, msg):
self.message_queue.put(msg)
class BenchTransport(Transport):
WORKER_QUEUE = DeferredQueue()
@inlineCallbacks
def startWorker(self):
yield Transport.startWorker(self)
self.message_queue = DeferredQueue()
self.WORKER_QUEUE.put(self)
def handle_outbound_message(self, msg):
self.message_queue.put(msg)
class TestStreamingClient(VumiTestCase):
def setUp(self):
self.fake_http = FakeHttpServer(self.handle_request)
self.request_queue = DeferredQueue()
self.client = StreamingClient(self.fake_http.get_agent)
self.messages_received = DeferredQueue()
self.errors_received = DeferredQueue()
self.disconnects_received = DeferredQueue()
def reason_trapper(reason):
if reason.trap(ResponseDone):
self.disconnects_received.put(reason.getErrorMessage())
self.receiver = self.client.stream(
Message,
self.messages_received.put, self.errors_received.put,
"http://vumi-go-api.example.com/", on_disconnect=reason_trapper)
def handle_request(self, request):
self.request_queue.put(request)
return NOT_DONE_YET
def test_default_agent_factory(self):
"""
If `None` is passed as the `agent_factory`, `Agent` is used instead.
"""
self.assertNotIsInstance(self.client.agent, Agent)
self.assertIsInstance(StreamingClient(None).agent, Agent)
self.assertIsInstance(StreamingClient().agent, Agent)
@inlineCallbacks
def test_callback_on_disconnect(self):
req = yield self.request_queue.get()
req.write(
'%s\n' % (Message(foo='bar').to_json().encode('utf-8'),))
req.finish()
message = yield self.messages_received.get()
self.assertEqual(message['foo'], 'bar')
reason = yield self.disconnects_received.get()
# this is the error message we get when a ResponseDone is raised
# which happens when the remote server closes the connection.
self.assertEqual(reason, 'Response body fully received')
@inlineCallbacks
def test_invalid_json(self):
req = yield self.request_queue.get()
req.write("Hello\n")
req.finish()
err = yield self.assertFailure(
self.errors_received.get(), VumiBridgeInvalidJsonError)
self.assertEqual(err.args, ("Hello",))
class TestTransport(TestCase):
@inlineCallbacks
def setUp(self):
DelayedCall.debug = True
self.ok_transport_calls = DeferredQueue()
self.mock_service = MockHttpServer(self.handle_request)
yield self.mock_service.start()
config = {
'transport_name': 'test_ok_transport',
'transport_type': 'ok',
'ussd_string_prefix': '',
'web_path': "foo",
'web_port': 0,
'url': self.mock_service.url,
'username': '******',
'password': '******',
}
self.worker = get_stubbed_worker(OkTransport, config)
self.broker = self.worker._amqp_client.broker
yield self.worker.startWorker()
self.worker_url = self.worker.get_transport_url()
@inlineCallbacks
def tearDown(self):
yield self.worker.stopWorker()
yield self.mock_service.stop()
def handle_request(self, request):
self.ok_transport_calls.put(request)
return ''
@inlineCallbacks
def test_health(self):
result = yield http_request(self.worker_url + "health", "",
method='GET')
self.assertEqual(json.loads(result), {
'pending_requests': 0
})
@inlineCallbacks
def test_inbound(self):
d = http_request(self.worker_url + "foo", '', method='GET')
msg, = yield self.broker.wait_messages("vumi",
"test_ok_transport.inbound", 1)
payload = msg.payload
tum = TransportUserMessage(**payload)
rep = tum.reply("OK")
self.broker.publish_message("vumi", "test_ok_transport.outbound",
rep)
response = yield d
self.assertEqual(response, 'OK')
class ClientConnection(Protocol): def __init__(self, server): self.server = server self.queue = DeferredQueue() def dataReceived(self, data): self.queue.put(data) # put data from client into queue def connectionMade(self): self.server.newConnect() if self.server.check == 1: # only listen to a port once reactor.listenTCP(DATA_PORT, DataConnFactory(self)) def forwardData(self, data): self.transport.write(data) # forward data to client def connectionLost(self, reason): self.server.check = 0
class QueueWrapper(object):
"""
Wrap a queue to have notifications when get is called on this particular
queue.
"""
def __init__(self, queue):
self._real_queue_get = queue.get
self.event_queue = DeferredQueue()
queue.get = self.get
def get(self, timeout=None):
self.event_queue.put(None)
return self._real_queue_get(timeout)
class ClientConn(LineReceiver):
def __init__(self):
self.data_stream_live = False
self.dq = DeferredQueue()
def connectionMade(self):
reactor.listenTCP(32002, DataConnFactory())
conndict['cmdConn'].sendLine("new client")
def dataReceived(self, data):
if self.data_stream_live:
conndict['dataConn'].transport.write(data)
else:
self.dq.put(data)
class TestStreamingClient(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.mock_server = MockHttpServer(self.handle_request)
self.add_cleanup(self.mock_server.stop)
yield self.mock_server.start()
self.url = self.mock_server.url
self.client = StreamingClient()
self.messages_received = DeferredQueue()
self.errors_received = DeferredQueue()
self.disconnects_received = DeferredQueue()
def reason_trapper(reason):
if reason.trap(ResponseDone):
self.disconnects_received.put(reason.getErrorMessage())
self.receiver = self.client.stream(
Message,
self.messages_received.put, self.errors_received.put,
self.url, on_disconnect=reason_trapper)
def handle_request(self, request):
self.mock_server.queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def test_callback_on_disconnect(self):
req = yield self.mock_server.queue.get()
req.write(
'%s\n' % (Message(foo='bar').to_json().encode('utf-8'),))
req.finish()
message = yield self.messages_received.get()
self.assertEqual(message['foo'], 'bar')
reason = yield self.disconnects_received.get()
# this is the error message we get when a ResponseDone is raised
# which happens when the remote server closes the connection.
self.assertEqual(reason, 'Response body fully received')
@inlineCallbacks
def test_invalid_json(self):
req = yield self.mock_server.queue.get()
req.write("Hello\n")
req.finish()
try:
yield self.errors_received.get()
except VumiBridgeInvalidJsonError, e:
self.assertEqual(e.args, ("Hello",))
else:
class TcpProxyProtocol(Protocol, object):
"""A simple TCP proxy"""
def __init__(self):
"""Create a new TCP proxy.
`self.server_queue` contains messages from end server to client.
`self.client_queue` contains messages from client to end server.
"""
self.server_queue = DeferredQueue()
self.client_queue = DeferredQueue()
self.server_queue.get().addCallback(self.serverQueueCallback)
def connectServer(self, hostname, port):
"""Tell the proxy what the end server is and start the connection.
The messages in `self.client_queue` will automatically be consumed.
This method should only be called once.
:param str hostname:
:param int port:
"""
endpoint = TCP4ClientEndpoint(reactor, hostname, port)
protocol = ServerProtocol(
self.server_queue, self.client_queue)
connectProtocol(endpoint, protocol)
def serverQueueCallback(self, data):
"""A callback for `self.server_queue`
:param str data: data from server queue
"""
if data is False:
self.transport.loseConnection()
return
self.transport.write(data)
self.server_queue.get().addCallback(self.serverQueueCallback)
def dataReceived(self, data):
"""Received data from client, put into client queue
"""
self.client_queue.put(data)
def connectionLost(self, why):
"""Client closed connection, or some other issue. close connection
to server
"""
self.client_queue.put(False)
class home_DataConn(protocol.Protocol): def __init__(self): print 'home listening for data connection from work' self.queue = DeferredQueue() def connectionMade(self): home_connections['data_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): home_connections['client_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class work_ServiceConn(protocol.Protocol): def __init__(self): print 'service connection established' self.queue = DeferredQueue() def connectionMade(self): work_connections['service_conn'] = self self.queue.get().addCallback(self.forwardData) def dataReceived(self, data): self.queue.put(data) def forwardData(self, data): work_connections['data_conn'].transport.write(data) self.queue.get().addCallback(self.forwardData)
class ClientProtocol(LineReceiver):
def __init__(self):
self.queue = DeferredQueue()
self.connect_d = Deferred()
self.disconnect_d = Deferred()
def connectionMade(self):
self.connect_d.callback(None)
def lineReceived(self, line):
self.queue.put(line)
def connectionLost(self, reason):
self.queue.put("DONE")
self.disconnect_d.callback(None)
class TestStreamingClient(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.mock_server = MockHttpServer(self.handle_request)
self.add_cleanup(self.mock_server.stop)
yield self.mock_server.start()
self.url = self.mock_server.url
self.client = StreamingClient()
self.messages_received = DeferredQueue()
self.errors_received = DeferredQueue()
self.disconnects_received = DeferredQueue()
def reason_trapper(reason):
if reason.trap(ResponseDone):
self.disconnects_received.put(reason.getErrorMessage())
self.receiver = self.client.stream(Message,
self.messages_received.put,
self.errors_received.put,
self.url,
on_disconnect=reason_trapper)
def handle_request(self, request):
self.mock_server.queue.put(request)
return NOT_DONE_YET
@inlineCallbacks
def test_callback_on_disconnect(self):
req = yield self.mock_server.queue.get()
req.write('%s\n' % (Message(foo='bar').to_json().encode('utf-8'), ))
req.finish()
message = yield self.messages_received.get()
self.assertEqual(message['foo'], 'bar')
reason = yield self.disconnects_received.get()
# this is the error message we get when a ResponseDone is raised
# which happens when the remote server closes the connection.
self.assertEqual(reason, 'Response body fully received')
@inlineCallbacks
def test_invalid_json(self):
req = yield self.mock_server.queue.get()
req.write("Hello\n")
req.finish()
try:
yield self.errors_received.get()
except VumiBridgeInvalidJsonError, e:
self.assertEqual(e.args, ("Hello", ))
else:
def do_work2(db):
queue = DeferredQueue()
defs = []
nworkers = len(entries)//5+1
if nworkers > self.nworkers:
nworkers = self.nworkers
for i in range(nworkers):
d = self.fetch(db,queue)
d.addErrback(log.err,"fetch()")
defs.append(d)
workers = set()
for id,inum,typ in entries:
if not self.running:
break
trace('copyrun',"%d: %s",inum,typ)
self.last_entry = id
if typ == 'd':
def dt(inum):
path = build_path(self.fs.store,inum, create=False)
try:
os.unlink(path)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
yield deferToThread(dt,inum)
else:
inode = SqlInode(self.fs,inum)
yield inode._load(db)
if typ == 'f':
if inum in workers:
trace('copyrun',"%d: in workers",inum,typ)
continue
workers.add(inum)
queue.put((id,inode))
elif typ == 't':
if inode.cache:
yield inode.cache.trim(inode.size)
else:
raise RuntimeError("Typ '%s' not found (inode %d)" % (typ,inum))
continue
for i in range(nworkers):
queue.put(None)
yield DeferredList(defs)
class QueuePoller(object):
implements(IPoller)
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
for p, q in self.queues.iteritems():
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
class QueuePoller(object):
implements(IPoller)
def __init__(self, settings):
self.q = RedisSpiderQueue(settings)
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
c = yield maybeDeferred(self.q.count)
if c:
msg = yield maybeDeferred(self.q.pop)
returnValue(self.dq.put(self._message(msg)))
def next(self):
return self.dq.get()
def _message(self, queue_msg):
d = queue_msg.copy()
d['_project'] = SCRAPY_PROJECT
d['_spider'] = d.pop('name')
d['_job'] = d.pop('jobid', uuid.uuid1().hex)
return d
class MockXmlOverTcpServer(MockServer):
def __init__(self):
self.responses = {}
self.received_queue = DeferredQueue()
def wait_for_data(self):
return self.received_queue.get()
def send_data(self, data):
self.transport.write(data)
def dataReceived(self, data):
response = self.responses.get(data)
if response is not None:
self.transport.write(response)
self.received_queue.put(data)
class ClientProtocol(LineReceiver):
def __init__(self):
self.queue = DeferredQueue()
self.connect_d = Deferred()
self.disconnect_d = Deferred()
def connectionMade(self):
self.connect_d.callback(None)
def lineReceived(self, line):
self.queue.put(line)
def connectionLost(self, reason):
self.queue.put("DONE")
self.disconnect_d.callback(None)
class QueuePoller(object):
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue()
@inlineCallbacks
def poll(self):
if not self.dq.waiting:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
class QueuePoller(object):
implements(IPoller)
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue(size=1)
@inlineCallbacks
def poll(self):
if self.dq.pending:
return
for p, q in self.queues.iteritems():
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
class QueuePoller(object):
def __init__(self, config):
self.config = config
self.update_projects()
self.dq = DeferredQueue()
@inlineCallbacks
def poll(self):
if not self.dq.waiting:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def next(self):
return self.dq.get()
def update_projects(self):
self.queues = get_spider_queues(self.config)
def _message(self, queue_msg, project):
d = queue_msg.copy()
d['_project'] = project
d['_spider'] = d.pop('name')
return d
class MockXmlOverTcpServer(MockServer):
def __init__(self):
self.responses = {}
self.received_queue = DeferredQueue()
def wait_for_data(self):
return self.received_queue.get()
def send_data(self, data):
self.transport.write(data)
def dataReceived(self, data):
response = self.responses.get(data)
if response is not None:
self.transport.write(response)
self.received_queue.put(data)
class TcpProxyProtocol(Protocol, object):
"""A simple TCP proxy"""
def __init__(self):
"""Create a new TCP proxy.
`self.server_queue` contains messages from end server to client.
`self.client_queue` contains messages from client to end server.
"""
self.server_queue = DeferredQueue()
self.client_queue = DeferredQueue()
self.server_queue.get().addCallback(self.serverQueueCallback)
def connectServer(self, hostname, port):
"""Tell the proxy what the end server is and start the connection.
The messages in `self.client_queue` will automatically be consumed.
This method should only be called once.
:param str hostname:
:param int port:
"""
endpoint = TCP4ClientEndpoint(reactor, hostname, port)
protocol = ServerProtocol(self.server_queue, self.client_queue)
connectProtocol(endpoint, protocol)
def serverQueueCallback(self, data):
"""A callback for `self.server_queue`
:param str data: data from server queue
"""
if data is False:
self.transport.loseConnection()
return
self.transport.write(data)
self.server_queue.get().addCallback(self.serverQueueCallback)
def dataReceived(self, data):
"""Received data from client, put into client queue
"""
self.client_queue.put(data)
def connectionLost(self, why):
"""Client closed connection, or some other issue. close connection
to server
"""
self.client_queue.put(False)
class ConsumerQueue(object):
def __init__(self, stop_on_error=False, empty=None):
self.stop_on_error = stop_on_error
self.empty = empty
self.queue = DeferredQueue()
self.size = 0
self.running = True
self._deferred = Deferred()
def _consume_next(self, *args):
if not self.running:
return
self._deferred = self.queue.get()
self._deferred.addCallbacks(self._consumer, self._error)
def _consumer(self, item):
self.size -= 1
r = self.consume(item)
if self.size == 0 and self.empty is not None:
self.empty()
if isinstance(r, Deferred):
r.addCallbacks(self._consume_next, self._consume_next)
else:
self._consume_next()
def _error(self, fail):
self.error(fail)
if not self.stop_on_error:
self._consume_next()
def add(self, item):
self.size += 1
self.queue.put(item)
def consume(self, item):
raise NotImplementedError
def error(self, fail):
raise NotImplementedError
def start(self):
self.running = True
self._consume_next()
def stop(self):
self.running = False
self._deferred.cancel()
class ToyXmlOverTcpClient(XmlOverTcpClient):
_PACKET_RECEIVED_HANDLERS = {'DummyPacket': 'dummy_packet_received'}
def __init__(self):
XmlOverTcpClient.__init__(self, 'root', 'toor', '1029384756')
self.PACKET_RECEIVED_HANDLERS.update(self._PACKET_RECEIVED_HANDLERS)
self.received_dummy_packets = []
self.received_data_request_packets = []
self.disconnected = False
self.session_id_counter = count()
self.generated_session_ids = []
self.request_id_counter = count()
self.generated_request_ids = []
self.received_queue = DeferredQueue()
def wait_for_data(self):
return self.received_queue.get()
def connectionMade(self):
pass
def dataReceived(self, data):
XmlOverTcpClient.dataReceived(self, data)
self.received_queue.put(data)
def dummy_packet_received(self, session_id, params):
self.received_dummy_packets.append((session_id, params))
def data_request_received(self, session_id, params):
self.received_data_request_packets.append((session_id, params))
def disconnect(self):
self.disconnected = True
@classmethod
def session_id_from_nr(cls, nr):
return cls.serialize_header_field(nr, cls.SESSION_ID_HEADER_SIZE)
def gen_session_id(self):
return self.session_id_from_nr(next(self.session_id_counter))
def gen_request_id(self):
return str(next(self.request_id_counter))
class DataConnection(Protocol):
def __init__(self):
self.q = DeferredQueue()
def connectionMade(self):
reactor.connectTCP('student02.cse.nd.edu', 22,
GenericClientFactory(ServiceConnection, self))
def dataReceived(self, data):
self.q.put(data)
def start_forwarding_data(self, service):
self.service = service
self.q.get().addCallback(self.forward_data)
def forward_data(self, data):
self.service.transport.write(data)
self.q.get().addCallback(self.forward_data) # Mayhaps this one
class ClientConnection(Protocol):
def __init__(self, cmd):
self.cmd = cmd
self.q = DeferredQueue()
def connectionMade(self):
reactor.listenTCP(DATA_PORT, GenericFactory(DataConnection, self))
self.cmd.transport.write('start data connection')
def dataReceived(self, data):
self.q.put(data)
def start_forwarding_client_data(self, data):
self.data = data
self.q.get().addCallback(self.forward_data)
def forward_data(self, data):
self.data.transport.write(data)
self.q.get().addCallback(self.forward_data)
class ProtocolP2(Protocol):
def __init__(self, friend):
self.friend = friend
self.data_queue = DeferredQueue()
def connectionMade(self):
logging.debug('P2 connection made')
def dataReceived(self, data):
logging.debug('P2 got data "%s"', data)
self.data_queue.put(data)
def start_forwarding(self):
self.data_queue.get().addCallback(self.send_to_friend)
def send_to_friend(self, data):
logging.debug('P2 sent "%s" to friend', data)
self.friend.transport.write(data)
self.data_queue.get().addCallback(self.send_to_friend)
