def test_full(self):
q = toro.Queue()
self.assertFalse(q.full())
self.assertEqual(q.maxsize, 0)
q = toro.Queue(1)
self.assertEqual(q.maxsize, 1)
self.assertFalse(q.full())
q.put('foo')
self.assertTrue(q.full())
def test_float_maxsize(self):
# Adapted from asyncio's test_float_maxsize.
q = toro.Queue(maxsize=1.3, io_loop=self.io_loop)
q.put_nowait(1)
q.put_nowait(2)
self.assertTrue(q.full())
self.assertRaises(Full, q.put_nowait, 3)
q = toro.Queue(maxsize=1.3, io_loop=self.io_loop)
yield q.put(1)
yield q.put(2)
self.assertTrue(q.full())
def test_multiple_waiters(self):
# tests that multiple waiters get their results back
q = toro.Queue()
@gen.coroutine
def waiter(q, evt):
evt.set((yield q.get()))
sendings = ['1', '2', '3', '4']
evts = [toro.AsyncResult() for x in sendings]
for i, x in enumerate(sendings):
waiter(q, evts[i]) # start task
@gen.coroutine
def collect_pending_results():
results = set()
for e in evts:
if e.ready():
# Won't block
x = yield e.get()
results.add(x)
raise gen.Return(len(results))
yield q.put(sendings[0])
self.assertEquals((yield collect_pending_results()), 1)
yield q.put(sendings[1])
self.assertEquals((yield collect_pending_results()), 2)
yield q.put(sendings[2])
yield q.put(sendings[3])
self.assertEquals((yield collect_pending_results()), 4)
def prepare(self):
logging.info('ProxyHandler.prepare')
self.chunks = toro.Queue(1)
self.fetch_future = AsyncHTTPClient().fetch(
'http://localhost:%d/upload' % options.port,
method='PUT',
body_producer=self.body_producer,
request_timeout=3600.0)
def test_str(self):
self.assertTrue('Queue' in str(toro.Queue()))
self.assertTrue('maxsize=11' in str(toro.Queue(11)))
q = toro.Queue()
for i in range(7):
q.get()
self.assertTrue('getters[7]' in str(q))
q = toro.Queue(1)
for i in range(5):
q.put('foo')
self.assertTrue('putters[4]' in str(q))
q = toro.Queue(1)
self.assertFalse('queue=' in str(q))
q.put('foo')
self.assertTrue('queue=' in str(q))
def test_senders_that_die(self):
q = toro.Queue()
@gen.coroutine
def do_send(q):
yield q.put('sent')
future = do_send(q)
self.assertEquals((yield q.get()), 'sent')
yield future
def __init__(self,
exe_path='phantomjs',
extra_args=None,
page_settings=None,
default_headers=None):
script_path = os.path.join(os.path.dirname(__file__), 'phantomjs.js')
self._in_queue = toro.Queue()
self._out_queue = toro.Queue()
self.page_event = wpull.actor.Event()
self._rpc_app = RPCApplication(self._out_queue, self._in_queue)
self._http_server = tornado.httpserver.HTTPServer(self._rpc_app)
http_socket, port = tornado.testing.bind_unused_port()
self._subproc = tornado.process.Subprocess(
[exe_path] + (extra_args or []) +
[script_path, str(port)],
stdout=tornado.process.Subprocess.STREAM,
)
self._rpc_reply_map = {}
self._setup(http_socket, page_settings, default_headers)
def __init__(self,
endpoint_url='http://localhost:8989',
io_loop=None,
*args,
**kwargs):
self._client = tornado.httpclient.AsyncHTTPClient()
self._wbuf = cStringIO.StringIO()
self.io_loop = io_loop or tornado.ioloop.IOLoop.instance()
self._endpoint = endpoint_url
self._response_queue = toro.Queue(io_loop=self.io_loop)
self._kwargs = kwargs
self._headers = {'Content-Type': 'application/x-thrift'}
def test_io_loop(self):
global_loop = self.io_loop
custom_loop = IOLoop()
self.assertNotEqual(global_loop, custom_loop)
q = toro.Queue(io_loop=custom_loop)
def callback(future):
assert future.result() == 'foo'
custom_loop.stop()
q.get().add_done_callback(callback)
q.put('foo')
custom_loop.start()
def test_send_last(self):
q = toro.Queue()
@gen.coroutine
def f():
val = yield q.get()
self.assertEqual('hi2', val)
yield q.put('ok')
# Start a task; blocks on get() until we do a put()
f()
yield q.put('hi2')
self.assertEqual('ok', (yield q.get()))
def _handle_request(self):
'''Process the request.'''
request = yield self._read_request_header()
if not request:
return
_logger.debug('Handling proxy request.')
if 'Content-Length' in request.fields:
yield self._read_request_body(request)
response_data_queue = toro.Queue()
recorder = ProxyRecorder(response_data_queue)
_logger.debug('Fetching.')
response_future = self._http_client.fetch(
request,
recorder=recorder,
)
touch_time = time.time()
while True:
try:
data = yield response_data_queue.get(
deadline=datetime.timedelta(seconds=0.1))
except toro.Timeout:
try:
if response_future.exception(timeout=0):
break
except concurrent.futures.TimeoutError:
pass
if time.time() - touch_time > 900:
break
else:
continue
if not data:
break
yield self._io_stream.write(data)
touch_time = time.time()
yield response_future
# TODO: determine whether the upstream connection was closed
self._io_stream.close()
def test_get_timeout(self):
q = toro.Queue()
st = time.time()
with assert_raises(toro.Timeout):
yield q.get(deadline=timedelta(seconds=0.1))
duration = time.time() - st
self.assertAlmostEqual(0.1, duration, places=1)
# Make sure that putting and getting a value returns Queue to initial
# state
q.put(1)
self.assertEqual(1, (yield q.get(deadline=timedelta(seconds=0.1))))
st = time.time()
with assert_raises(toro.Timeout):
yield q.get(deadline=timedelta(seconds=0.1))
duration = time.time() - st
self.assertAlmostEqual(0.1, duration, places=1)
def __init__(self,
address,
ssl_enable=False,
max_count=6,
connection_factory=Connection):
assert isinstance(address[0], str)
assert isinstance(address[1], int) and address[1]
self._address = address
self._request_queue = toro.Queue()
self._ssl = ssl_enable
self._connection_factory = connection_factory
self._connections = set()
self._max_count = max_count
self._max_count_semaphore = toro.BoundedSemaphore(max_count)
self._running = True
self._cleaner_timer = tornado.ioloop.PeriodicCallback(
self.clean, 300000)
tornado.ioloop.IOLoop.current().add_future(
self._run_loop(), lambda future: future.result())
self._cleaner_timer.start()
def __init__(self,
host,
port,
request_queue=None,
ssl=False,
max_count=6,
connection_factory=Connection):
assert isinstance(host, str)
assert isinstance(port, int) and port
self._host = host
self._port = port
self._request_queue = toro.Queue()
self._ssl = ssl
self._connection_factory = connection_factory
self._connections = set()
self._max_count = max_count
self._max_count_semaphore = toro.BoundedSemaphore(max_count)
self._running = True
self._cleaner_timer = tornado.ioloop.PeriodicCallback(
self.clean, 300000)
self._run()
self._cleaner_timer.start()
def test_max_size(self):
q = toro.Queue(2)
results = []
@gen.coroutine
def putter():
yield q.put('a')
results.append('a')
yield q.put('b')
results.append('b')
yield q.put('c')
results.append('c')
future = putter()
yield pause(timedelta(seconds=.01))
self.assertEquals(results, ['a', 'b'])
self.assertEquals((yield q.get()), 'a')
yield pause(timedelta(seconds=.01))
self.assertEquals(results, ['a', 'b', 'c'])
self.assertEquals((yield q.get()), 'b')
self.assertEquals((yield q.get()), 'c')
yield future
def test_callback_checking(self):
self.assertRaises(TypeError, toro.Queue().get, callback='foo')
self.assertRaises(TypeError, toro.Queue().get, callback=1)
def test_get_nowait_unblocks_putter(self):
q = toro.Queue(maxsize=1)
q.put_nowait(1)
future = q.put(2, deadline=timedelta(seconds=1))
self.assertEqual(1, q.get_nowait())
yield future
def test_put_nowait_unblocks_getter(self):
q = toro.Queue(maxsize=1)
future = q.get(deadline=timedelta(seconds=1))
q.put_nowait(1)
result = yield future
self.assertEqual(1, result)
def prepare(self):
self.key = self.request.uri.split('/')[2]
self.stripper = utils.BoundaryStripper()
queues[self.key] = toro.Queue(maxsize=1)
def test_send_first(self):
q = toro.Queue()
yield q.put('hi')
self.assertEqual('hi', (yield q.get()))
def test_repr(self):
# No exceptions
str(toro.Queue())
repr(toro.Queue())
def handle_connect(self, params):
"""
Authenticate client's connection, initialize required
variables in case of successful authentication.
"""
if self.application.collector:
self.application.collector.incr('connect')
self.application.collector.incr(self.sock.session.transport_name)
if self.is_authenticated:
raise Return((self.uid, None))
token = params["token"]
user = params["user"]
project_id = params["project"]
timestamp = params["timestamp"]
user_info = params.get("info")
project, error = yield self.application.get_project(project_id)
if error:
raise Return((None, error))
secret_key = project['secret_key']
try:
client_token = auth.get_client_token(secret_key,
project_id,
user,
timestamp,
user_info=user_info)
except Exception as err:
logger.error(err)
raise Return((None, "invalid connection parameters"))
if token != client_token:
raise Return((None, "invalid token"))
if user_info is not None:
try:
user_info = json_decode(user_info)
except Exception as err:
logger.error("malformed JSON data in user_info")
logger.error(err)
user_info = None
try:
timestamp = int(timestamp)
except ValueError:
raise Return((None, "invalid timestamp"))
now = time.time()
self.user = user
self.examined_at = timestamp
connection_check = project.get('connection_check', False)
if connection_check and self.examined_at + project.get(
"connection_lifetime", 24 * 365 * 3600) < now:
# connection expired - this is a rare case when Centrifuge went offline
# for a while or client turned on his computer from sleeping mode.
# put this client into the queue of connections waiting for
# permission to reconnect with expired credentials. To avoid waiting
# client must reconnect with actual credentials i.e. reload browser
# window.
if project_id not in self.application.expired_reconnections:
self.application.expired_reconnections[project_id] = []
self.application.expired_reconnections[project_id].append(self)
if project_id not in self.application.expired_connections:
self.application.expired_connections[project_id] = {
"users": set(),
"checked_at": None
}
self.application.expired_connections[project_id]["users"].add(user)
self.connect_queue = toro.Queue(maxsize=1)
value = yield self.connect_queue.get()
if not value:
yield self.close_sock()
raise Return((None, self.application.UNAUTHORIZED))
else:
self.connect_queue = None
# Welcome to Centrifuge dear Connection!
self.is_authenticated = True
self.project_id = project_id
self.token = token
self.default_user_info = {
'user_id': self.user,
'client_id': self.uid,
'default_info': user_info,
'channel_info': None
}
self.channels = {}
self.presence_ping_task = PeriodicCallback(
self.send_presence_ping,
self.application.engine.presence_ping_interval)
self.presence_ping_task.start()
self.application.add_connection(project_id, self.user, self.uid, self)
raise Return((self.uid, None))
