def test_full(self): q = toro.Queue() self.assertFalse(q.full()) self.assertEqual(q.maxsize, 0) q = toro.Queue(1) self.assertEqual(q.maxsize, 1) self.assertFalse(q.full()) q.put('foo') self.assertTrue(q.full())
def test_float_maxsize(self): # Adapted from asyncio's test_float_maxsize. q = toro.Queue(maxsize=1.3, io_loop=self.io_loop) q.put_nowait(1) q.put_nowait(2) self.assertTrue(q.full()) self.assertRaises(Full, q.put_nowait, 3) q = toro.Queue(maxsize=1.3, io_loop=self.io_loop) yield q.put(1) yield q.put(2) self.assertTrue(q.full())
def test_multiple_waiters(self): # tests that multiple waiters get their results back q = toro.Queue() @gen.coroutine def waiter(q, evt): evt.set((yield q.get())) sendings = ['1', '2', '3', '4'] evts = [toro.AsyncResult() for x in sendings] for i, x in enumerate(sendings): waiter(q, evts[i]) # start task @gen.coroutine def collect_pending_results(): results = set() for e in evts: if e.ready(): # Won't block x = yield e.get() results.add(x) raise gen.Return(len(results)) yield q.put(sendings[0]) self.assertEquals((yield collect_pending_results()), 1) yield q.put(sendings[1]) self.assertEquals((yield collect_pending_results()), 2) yield q.put(sendings[2]) yield q.put(sendings[3]) self.assertEquals((yield collect_pending_results()), 4)
def prepare(self): logging.info('ProxyHandler.prepare') self.chunks = toro.Queue(1) self.fetch_future = AsyncHTTPClient().fetch( 'http://localhost:%d/upload' % options.port, method='PUT', body_producer=self.body_producer, request_timeout=3600.0)
def test_str(self): self.assertTrue('Queue' in str(toro.Queue())) self.assertTrue('maxsize=11' in str(toro.Queue(11))) q = toro.Queue() for i in range(7): q.get() self.assertTrue('getters[7]' in str(q)) q = toro.Queue(1) for i in range(5): q.put('foo') self.assertTrue('putters[4]' in str(q)) q = toro.Queue(1) self.assertFalse('queue=' in str(q)) q.put('foo') self.assertTrue('queue=' in str(q))
def test_senders_that_die(self): q = toro.Queue() @gen.coroutine def do_send(q): yield q.put('sent') future = do_send(q) self.assertEquals((yield q.get()), 'sent') yield future
def __init__(self, exe_path='phantomjs', extra_args=None, page_settings=None, default_headers=None): script_path = os.path.join(os.path.dirname(__file__), 'phantomjs.js') self._in_queue = toro.Queue() self._out_queue = toro.Queue() self.page_event = wpull.actor.Event() self._rpc_app = RPCApplication(self._out_queue, self._in_queue) self._http_server = tornado.httpserver.HTTPServer(self._rpc_app) http_socket, port = tornado.testing.bind_unused_port() self._subproc = tornado.process.Subprocess( [exe_path] + (extra_args or []) + [script_path, str(port)], stdout=tornado.process.Subprocess.STREAM, ) self._rpc_reply_map = {} self._setup(http_socket, page_settings, default_headers)
def __init__(self, endpoint_url='http://localhost:8989', io_loop=None, *args, **kwargs): self._client = tornado.httpclient.AsyncHTTPClient() self._wbuf = cStringIO.StringIO() self.io_loop = io_loop or tornado.ioloop.IOLoop.instance() self._endpoint = endpoint_url self._response_queue = toro.Queue(io_loop=self.io_loop) self._kwargs = kwargs self._headers = {'Content-Type': 'application/x-thrift'}
def test_io_loop(self): global_loop = self.io_loop custom_loop = IOLoop() self.assertNotEqual(global_loop, custom_loop) q = toro.Queue(io_loop=custom_loop) def callback(future): assert future.result() == 'foo' custom_loop.stop() q.get().add_done_callback(callback) q.put('foo') custom_loop.start()
def test_send_last(self): q = toro.Queue() @gen.coroutine def f(): val = yield q.get() self.assertEqual('hi2', val) yield q.put('ok') # Start a task; blocks on get() until we do a put() f() yield q.put('hi2') self.assertEqual('ok', (yield q.get()))
def _handle_request(self): '''Process the request.''' request = yield self._read_request_header() if not request: return _logger.debug('Handling proxy request.') if 'Content-Length' in request.fields: yield self._read_request_body(request) response_data_queue = toro.Queue() recorder = ProxyRecorder(response_data_queue) _logger.debug('Fetching.') response_future = self._http_client.fetch( request, recorder=recorder, ) touch_time = time.time() while True: try: data = yield response_data_queue.get( deadline=datetime.timedelta(seconds=0.1)) except toro.Timeout: try: if response_future.exception(timeout=0): break except concurrent.futures.TimeoutError: pass if time.time() - touch_time > 900: break else: continue if not data: break yield self._io_stream.write(data) touch_time = time.time() yield response_future # TODO: determine whether the upstream connection was closed self._io_stream.close()
def test_get_timeout(self): q = toro.Queue() st = time.time() with assert_raises(toro.Timeout): yield q.get(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) # Make sure that putting and getting a value returns Queue to initial # state q.put(1) self.assertEqual(1, (yield q.get(deadline=timedelta(seconds=0.1)))) st = time.time() with assert_raises(toro.Timeout): yield q.get(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1)
def __init__(self, address, ssl_enable=False, max_count=6, connection_factory=Connection): assert isinstance(address[0], str) assert isinstance(address[1], int) and address[1] self._address = address self._request_queue = toro.Queue() self._ssl = ssl_enable self._connection_factory = connection_factory self._connections = set() self._max_count = max_count self._max_count_semaphore = toro.BoundedSemaphore(max_count) self._running = True self._cleaner_timer = tornado.ioloop.PeriodicCallback( self.clean, 300000) tornado.ioloop.IOLoop.current().add_future( self._run_loop(), lambda future: future.result()) self._cleaner_timer.start()
def __init__(self, host, port, request_queue=None, ssl=False, max_count=6, connection_factory=Connection): assert isinstance(host, str) assert isinstance(port, int) and port self._host = host self._port = port self._request_queue = toro.Queue() self._ssl = ssl self._connection_factory = connection_factory self._connections = set() self._max_count = max_count self._max_count_semaphore = toro.BoundedSemaphore(max_count) self._running = True self._cleaner_timer = tornado.ioloop.PeriodicCallback( self.clean, 300000) self._run() self._cleaner_timer.start()
def test_max_size(self): q = toro.Queue(2) results = [] @gen.coroutine def putter(): yield q.put('a') results.append('a') yield q.put('b') results.append('b') yield q.put('c') results.append('c') future = putter() yield pause(timedelta(seconds=.01)) self.assertEquals(results, ['a', 'b']) self.assertEquals((yield q.get()), 'a') yield pause(timedelta(seconds=.01)) self.assertEquals(results, ['a', 'b', 'c']) self.assertEquals((yield q.get()), 'b') self.assertEquals((yield q.get()), 'c') yield future
def test_callback_checking(self): self.assertRaises(TypeError, toro.Queue().get, callback='foo') self.assertRaises(TypeError, toro.Queue().get, callback=1)
def test_get_nowait_unblocks_putter(self): q = toro.Queue(maxsize=1) q.put_nowait(1) future = q.put(2, deadline=timedelta(seconds=1)) self.assertEqual(1, q.get_nowait()) yield future
def test_put_nowait_unblocks_getter(self): q = toro.Queue(maxsize=1) future = q.get(deadline=timedelta(seconds=1)) q.put_nowait(1) result = yield future self.assertEqual(1, result)
def prepare(self): self.key = self.request.uri.split('/')[2] self.stripper = utils.BoundaryStripper() queues[self.key] = toro.Queue(maxsize=1)
def test_send_first(self): q = toro.Queue() yield q.put('hi') self.assertEqual('hi', (yield q.get()))
def test_repr(self): # No exceptions str(toro.Queue()) repr(toro.Queue())
def handle_connect(self, params): """ Authenticate client's connection, initialize required variables in case of successful authentication. """ if self.application.collector: self.application.collector.incr('connect') self.application.collector.incr(self.sock.session.transport_name) if self.is_authenticated: raise Return((self.uid, None)) token = params["token"] user = params["user"] project_id = params["project"] timestamp = params["timestamp"] user_info = params.get("info") project, error = yield self.application.get_project(project_id) if error: raise Return((None, error)) secret_key = project['secret_key'] try: client_token = auth.get_client_token(secret_key, project_id, user, timestamp, user_info=user_info) except Exception as err: logger.error(err) raise Return((None, "invalid connection parameters")) if token != client_token: raise Return((None, "invalid token")) if user_info is not None: try: user_info = json_decode(user_info) except Exception as err: logger.error("malformed JSON data in user_info") logger.error(err) user_info = None try: timestamp = int(timestamp) except ValueError: raise Return((None, "invalid timestamp")) now = time.time() self.user = user self.examined_at = timestamp connection_check = project.get('connection_check', False) if connection_check and self.examined_at + project.get( "connection_lifetime", 24 * 365 * 3600) < now: # connection expired - this is a rare case when Centrifuge went offline # for a while or client turned on his computer from sleeping mode. # put this client into the queue of connections waiting for # permission to reconnect with expired credentials. To avoid waiting # client must reconnect with actual credentials i.e. reload browser # window. if project_id not in self.application.expired_reconnections: self.application.expired_reconnections[project_id] = [] self.application.expired_reconnections[project_id].append(self) if project_id not in self.application.expired_connections: self.application.expired_connections[project_id] = { "users": set(), "checked_at": None } self.application.expired_connections[project_id]["users"].add(user) self.connect_queue = toro.Queue(maxsize=1) value = yield self.connect_queue.get() if not value: yield self.close_sock() raise Return((None, self.application.UNAUTHORIZED)) else: self.connect_queue = None # Welcome to Centrifuge dear Connection! self.is_authenticated = True self.project_id = project_id self.token = token self.default_user_info = { 'user_id': self.user, 'client_id': self.uid, 'default_info': user_info, 'channel_info': None } self.channels = {} self.presence_ping_task = PeriodicCallback( self.send_presence_ping, self.application.engine.presence_ping_interval) self.presence_ping_task.start() self.application.add_connection(project_id, self.user, self.uid, self) raise Return((self.uid, None))