def __init__(self, proxy, thread_number): self.pool = eventlet.GreenPool(thread_number) self.input_queue = eventlet.Queue() self.output_queue = eventlet.Queue() self.proxy = proxy self.logger = Logger.create("log") self.timeout = 10
def __init__(self, path, disk_id, compress_transfer=None, compressor_count=3): self._offset = None self._session = None self._ip = None self._port = None self._crt = None self._key = None self._ca = None self._closing = False self._write_error = False self._id = None self._exception = None self._compressor_count = compressor_count self._comp_q = eventlet.Queue(maxsize=5) self._sender_q = eventlet.Queue(maxsize=5) self._sender_evt = None self._compressor_evt = None self._compress_transfer = compress_transfer if self._compress_transfer is None: self._compress_transfer = CONF.compress_transfers super(HTTPBackupWriterImpl, self).__init__(path, disk_id)
def __init__(self, create, destroy, size=3): if size == 0: raise RuntimeError("Zero size would create unbounded resources") self.ready = eventlet.Queue(maxsize=size) self.trash = eventlet.Queue() self.size = size self.create = create self.destroy = destroy
def __init__(self, handler, peers, routerid): self.logger = logging.getLogger('fbgp.exabgp_connect') self.handler = handler self.peers = peers self.routerid = routerid self.conn = None self.exabgp = None self.running = False self.recv_queue = eventlet.Queue(256) self.send_queue = eventlet.Queue(256)
def __init__(self): self.config = {} self.config['modules'] = {} self.config['services'] = {} # Dictionary mapping service name to a dict of arguments. # Those arguments are only passed to d6service by createservice if they # are not alreay present in the ARGS argument given to createservice. self.default_service_args = {} cageKeys = ['python.d6cage'] cageDesc = 'deepsix python cage' name = "d6cage" deepsix.deepSix.__init__(self, name, cageKeys) self.inbox = eventlet.Queue() self.dataPath = eventlet.Queue() self.table = amqprouter.routeTable() self.table.add("local.router", self.inbox) self.table.add(self.name, self.inbox) self.table.add("router", self.inbox) localname = "local." + self.name self.table.add(localname, self.inbox) self.modules = {} self.services = {} self.greenThreadPool = eventlet.GreenPool() self.greenThreads = [] self.unloadingServices = {} self.reloadingServices = set() self.services[self.name] = {} self.services[self.name]['service'] = self self.services[self.name]['name'] = self.name self.services[self.name]['description'] = cageDesc self.services[self.name]['inbox'] = self.inbox self.services[self.name]['keys'] = self.keys self.services[self.name]['type'] = None self.services[self.name]['id'] = None self.subscribe( "local.d6cage", "routeKeys", callback=self.updateRoutes, interval=5)
def handle_client_subscribe(data): """Handle client subscribing to a channel.""" channel = data['channel'] # make a queue to receive events from pubsub q = eventlet.Queue(maxsize=20) # save request context for eventlet ctx switch req_ctx_stack = _request_ctx_stack req_ctx = req_ctx_stack.top.copy() def emit_green(q, req_ctx): """Wait for events on the queue and emit them to client.""" while 1: n = q.get() # block on waiting for item from queue req_ctx.push() # restore request context emit( 'event', n ) # send event to client (has channel and payload fields) req_ctx.pop() # done with request context # subscribe and queue emit callbacks, async subscription = self.pubsub.subscribe(channel, q) listen_gthread = eventlet.spawn(emit_green, q, req_ctx) self.listen_gthreads[request.sid].append(listen_gthread) self.subscriptions[request.sid].append(subscription) log.info(f"Client {current_user} subscribed to {channel}") emit('subscribed', {'channel': channel})
def __init__(self, handler): self.proto = None self.running = False self.handler = handler self.server_addr = os.environ.get('FBGP_SERVER_ADDR') or 'localhost' self.server_port = int(os.environ.get('FBGP_SERVER_PORT') or 9999) self.send_q = eventlet.Queue(128)
def test_greenthread_singleton(metaclass: Type, repetitions: int): assert singletons.detect_greenthread_environment() == 'eventlet' class MySingleton(metaclass=metaclass): def __init__(self): self.uuid = uuid.uuid4() def inner_func(q: eventlet.Queue): a = MySingleton() b = MySingleton() q.put(( a.uuid, b.uuid, )) test_q = eventlet.Queue() greenthreads = [] for _ in range(repetitions): p = eventlet.spawn_n(inner_func, test_q) greenthreads.append(p) eventlet.sleep() # force execution context to switch seen_uuids = set() while len(seen_uuids) < repetitions: a, b = test_q.get(timeout=JOIN_TIMEOUT) assert a == b assert a not in seen_uuids seen_uuids.add(a)
def socket_created(tp): print "New socket type %s" % tp # Create a new mock socket. socket = mock.Mock() # Hook its bind and connect methods, so we can remember the address # that it binds or connects to. socket.bind.side_effect = make_socket_bound(socket) socket.connect.side_effect = make_socket_connect(socket) # Create a queue that we can use to deliver messages to be received # on this socket. socket.rcv_queue = eventlet.Queue(1) # Hook the socket's recv_multipart and poll methods, to wait on # this queue. socket.recv_multipart.side_effect = make_recv('multipart', socket) socket.recv_json.side_effect = make_recv('json', socket) socket.poll.side_effect = make_poll(socket) # Add this to the test code's list of known sockets. self.sockets.add(socket) return socket
def initialize(self, conf, context): LOG.info("API Spout is started") self.MAX_WORKERS = FLAGS.get('rabbit_read_workers') self.pid = os.getpid() self.worker_pool = eventlet.GreenPool(self.MAX_WORKERS) self.rpc = rpc.RemoteProcedureCall() self.queue = eventlet.Queue()
def benchmark_datasource_update(self, size): """Benchmark a datasource update. Time the propagation of a datasource update from datasource.poll() to ending up in the datasource.dataPath queue. """ LOG.info("%s:: benchmarking datasource update of %d rows", size) self.datasource.datarows = size # intercept the queue addition so it doesn't immediately get pulled off # by the d6cage received = eventlet.Queue() self.mox.StubOutWithMock(self.datasource.dataPath, "put_nowait") self.datasource.dataPath.put_nowait(mox.IgnoreArg()).WithSideEffects( received.put_nowait) self.mox.ReplayAll() # poll and then wait until we've got an item from our queue LOG.info("%s:: polling datasource", self.__class__.__name__) self.datasource.poll() result = received.get(timeout=30) self.assertTrue(result.body) self.assertEqual(len(result.body.data), size) self.mox.VerifyAll()
def bind(self): """ Binds the netlink socket and initializes data structures used for communication between the dispatcher and the server. """ if self.socket is not None: try: self.socket.close() except socket.error: # Ignore the error as we will try to rebind. pass self.__intfqueue = collections.defaultdict(eventlet.Queue) while self.__running: eventlet.sleep(1) self.__intfevent = eventlet.Queue() try: # pylint: disable=no-member self.socket = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, self.NETLINK_ROUTE) # Set rcv. buffer size to 30M (higher than the rmem_max of 8M). self.socket.setsockopt(socket.SOL_SOCKET, self.__SO_RCVBUFFORCE, self.__BUF_SIZE) except socket.error as ex: raise RuntimeError('open: socket err: %s' % ex) # Open a socket for receiving netlink msgs. try: # PID_MAX_LIMIT is 2^22 allowing 1024 sockets per-pid. We # start with 1 in the upper space (top 10 bits) instead of # 0 to avoid conflicts with netlink_autobind which always # attempts to bind with the pid (and on failure with # negative values -4097, -4098, -4099 etc.) self.socket.bind((os.getpid() | (1 << 22), Netlink.GROUPS)) except socket.error as ex: raise RuntimeError('bind: socket err: %s' % ex)
def __init__(self, members, name, server_address='tcp://*:5556', pubsub_address='tcp://*:5557'): """RPC server that runs on the master node. Note: Currently, the server is not threaded. A high volume of traffic is not expected, so its probably not necessary. Arguments: members: [list] the namespace/name of running servers in the cluster name: [string] the namespace/name of the node this is running on server_address: [string] the web address the main server runs on pubsub_address: [string] the web address the pub_sub runs on """ self.server = RPCServer(server_address) self.publisher = Publisher(pubsub_address) self.message_queue = eventlet.Queue() self.running = Switch() self.running.on() self.name = name self.phase = 0 self.num_repoted = 0 self.busy = False # TODO(egarcia): Make this thread safe self.members = {} for member in members: self.members[member] = {'phase': 0, 'data': {}} # Start the server self.__listen()
def test_channel_wait(self): channel = eventlet.Queue(0) events = [] def another_greenlet(): events.append('sending hello') channel.put('hello') events.append('sending world') channel.put('world') events.append('sent world') eventlet.spawn(another_greenlet) events.append('waiting') events.append(channel.get()) events.append(channel.get()) self.assertEqual( ['waiting', 'sending hello', 'hello', 'sending world', 'world'], events) eventlet.sleep(0) self.assertEqual([ 'waiting', 'sending hello', 'hello', 'sending world', 'world', 'sent world' ], events)
def simulated_time_sleep(secs=None): if secs is None: # Thread just wants to yield to any other waiting thread. self.give_way() return # Create a new queue. queue = eventlet.Queue(1) queue.stack = inspect.stack()[1][3] # Add it to the dict of sleepers, together with the waking up time. self.sleepers[queue] = self.current_time + secs _log.info("T=%s: %s: Start sleep for %ss until T=%s", self.current_time, queue.stack, secs, self.sleepers[queue]) # Do a zero time real sleep, to allow other threads to run. self.real_eventlet_sleep(REAL_EVENTLET_SLEEP_TIME) # Block until something is posted to the queue. queue.get(True) # Wake up. return None
def __init__(self, client, router, connection, message_handler): """ A Session between a Client and a Router. :Parameters: client : instance An instance of :class:`peers.Client` router : instance An instance of :class:`peers.Router` connection : instance An instance of ``wampy.transports``. Defaults to ``wampy.transports.WebSocket`` message_handler : instance An instance of ``wampy.message_handler.MessageHandler``, or a subclass of """ self.client = client self.router = router self.connection = connection self.message_handler = message_handler self.request_ids = {} self.subscription_map = {} self.registration_map = {} self.session_id = None # spawn a green thread to listen for incoming messages over # a connection and put them on a queue to be processed self._managed_thread = None self._message_queue = eventlet.Queue() self._listen(self.connection, self._message_queue)
def __init__(self): self.deleteQueue = eventlet.Queue() self.deleteStopped = False self.rebuildQueue = eventlet.Queue() self.rebuildStopped = False # Dictionary of weak references to Locks that govern file locking # of ThinAppFile objects. The mapping is fileId -> Lock. If no entry # is present, the file is assumed to be unlocked. self.fileLocks = weakref.WeakValueDictionary() # It's locks all the way down. self.lockMutex = threading.Lock() # By default this lets 1000 requests run so we may want to limit # it to a lower number. self.pool = eventlet.GreenPool()
def test_waiters_that_cancel(self): q = eventlet.Queue() gt = eventlet.spawn(do_bail, q) self.assertEqual(gt.wait(), 'timed out') q.put('hi') self.assertEqual(q.get(), 'hi')
def test_gt_link_callback_added_during_execution(self): pool = greenpool.GreenPool() q1 = eventlet.Queue() q2 = eventlet.Queue() def func(): q1.put(None) q2.get() link_callback = mock.Mock() thread = pool.spawn(func) q1.get() thread.link(link_callback) q2.put(None) pool.waitall() link_callback.assert_called_once_with(thread)
def handle_blocks(websocket): """Receive a connection and send it database notifications. """ queue = eventlet.Queue() eventlet.spawn(pg_listen, queue, 'blocks') while 1: item = queue.get() websocket.send(item.payload)
def test_gt_link_callback_exception_inside_thread(self): pool = greenpool.GreenPool() q1 = eventlet.Queue() q2 = eventlet.Queue() def func(): q1.put(None) q2.get() raise Exception() link_callback = mock.Mock() thread = pool.spawn(func) q1.get() thread.link(link_callback) q2.put(None) pool.waitall() link_callback.assert_called_once_with(thread)
def test_two_bogus_waiters(self): q = eventlet.Queue() gt1 = eventlet.spawn(do_bail, q) gt2 = eventlet.spawn(do_bail, q) eventlet.sleep(0) q.put('sent') self.assertEqual(gt1.wait(), 'timed out') self.assertEqual(gt2.wait(), 'timed out') self.assertEqual(q.get(), 'sent')
def __init__(self, *args, **kwargs): super(FlowBasedBGP, self).__init__(*args, **kwargs) self.logger = get_logger('fbgp', os.environ.get('FBGP_LOG', None), os.environ.get('FBGP_LOG_LEVEL', 'info')) self.faucet_api = kwargs['faucet_experimental_api'] self.nexthop_to_pathid = {} self.path_mapping = collections.defaultdict(set) self.vip_assignment = {} self.rcv_msg_q = eventlet.Queue(256)
def test_send_last(self): q = eventlet.Queue() def waiter(q): self.assertEqual(q.get(), 'hi2') gt = eventlet.spawn(eventlet.with_timeout, 0.1, waiter, q) eventlet.sleep(0) eventlet.sleep(0) q.put('hi2') gt.wait()
def test_resize_down(self): q = eventlet.Queue(5) for i in range(5): q.put(i) self.assertEqual(list(q.queue), list(range(5))) q.resize(1) eventlet.sleep(0) self.assertEqual(list(q.queue), list(range(5)))
def __init__(self, path, disk_id, compress_transfer=None, encoder_count=3): self._msg_id = None self._stdin = None self._stdout = None self._stderr = None self._offset = None self._ssh = None self._sender_q = eventlet.Queue(maxsize=5) self._enc_q = eventlet.Queue(maxsize=5) self._sender_evt = None self._encoder_evt = [] self._encoder_cnt = encoder_count self._exception = None self._closing = False self._compress_transfer = compress_transfer if self._compress_transfer is None: self._compress_transfer = CONF.compress_transfers super(SSHBackupWriterImpl, self).__init__(path, disk_id)
def test_waiting(self): q = eventlet.Queue() gt1 = eventlet.spawn(q.get) eventlet.sleep(0) self.assertEqual(1, q.getting()) q.put('hi') eventlet.sleep(0) self.assertEqual(0, q.getting()) self.assertEqual('hi', gt1.wait()) self.assertEqual(0, q.getting())
def multiple_greenthread_contexts(self): csm = EventletContextStackManager() def make_func(name): def func(csm, queue): csm.push_coroutine(name) queue.put(list(csm.iter_current_stack())) func.__name__ = name return func foo_queue = eventlet.Queue() bar_queue = eventlet.Queue() foo = eventlet.spawn(make_func('foo'), csm, foo_queue) bar = eventlet.spawn(make_func('bar'), csm, bar_queue) foo.wait() bar.wait() Assert(foo_queue.get()) == ['foo'] Assert(bar_queue.get()) == ['bar']
def handle(ws): """ Receive a connection and send it database notifications. """ q = eventlet.Queue() eventlet.spawn(dblisten, q) while 1: n = q.get() print(n) ws.send(n.payload)
def test_task_done(self): channel = eventlet.Queue(0) X = object() gt = eventlet.spawn(channel.put, X) result = channel.get() assert result is X, (result, X) assert channel.unfinished_tasks == 1, channel.unfinished_tasks channel.task_done() assert channel.unfinished_tasks == 0, channel.unfinished_tasks gt.wait()