class MongoThread(object): """A thread, or a greenlet, that uses a Connection""" def __init__(self, test_case): self.use_greenlets = test_case.use_greenlets self.connection = test_case.c self.db = self.connection[DB] self.ut = test_case self.passed = False def start(self): if self.use_greenlets: self.thread = Greenlet(self.run) else: self.thread = threading.Thread(target=self.run) self.thread.start() def join(self): self.thread.join() self.thread = None def run(self): self.run_mongo_thread() # No exceptions thrown self.passed = True def run_mongo_thread(self): raise NotImplementedError()
def ws(self): # open websocket websock = request.environ["ws4py.websocket"] # websock_version = request.environ['wsgi.websocket_version'] # sec_websocket_extensions = request.environ.get('HTTP_SEC_WEBSOCKET_EXTENSIONS') # sec_websocket_key = request.environ.get('HTTP_SEC_WEBSOCKET_KEY') # sec_websocket_version = request.environ.get('HTTP_SEC_WEBSOCKET_VERSION') endpoint = websock.sock.getpeername() # print 'connection established with endpoint %s:%s, version %s / %s, key %s, extensions %s' \ # % (endpoint[0], endpoint[1], websock_version, sec_websocket_version, sec_websocket_key, sec_websocket_extensions) from gevent import Greenlet g1 = Greenlet(websock.run) g1.start() websock.send("Hello dear Browser! I'll send you redis stuff when I get some") g2 = Greenlet(send_stuff_in_intervals, websock) # g2 = Greenlet(send_redis_stuff, websock) g2.start() g2.join() g1.join() print "connection closed to %s:%s" % endpoint
def thrFunc(s): print "%s start.\n" %s gevent.sleep(2) gThr = Greenlet(subFunc) gThr.start() gThr.join() print "%s end.\n" %s
def start(self): greenlet = Greenlet(self._start) greenlet.link_exception(self._logGreenletError) TIMER = gevent.greenlet.Greenlet(self._timer) TIMER.start() # Start and wait until the log server stops (main greenlet). greenlet.start() greenlet.join()
def StartGame(self): self.game_id = self.manager.CreateGame(self.game_name, self.players)[0] for p in self.players: CardNamespace.players[p].emit('go_to_game_table') while False in [CardNamespace.players[p].ready for p in self.players]: print >>sys.stderr, [CardNamespace.players[p].ready for p in self.players] sleep(0.05) g = Greenlet(self.manager.StartGame, self.game_id) g.start() g.join()
class MongoThread(object): """A thread, or a greenlet, that uses a MongoClient""" def __init__(self, test_case): self.use_greenlets = test_case.use_greenlets self.client = test_case.c self.db = self.client[DB] self.ut = test_case self.passed = False def start(self): if self.use_greenlets: # A Gevent extended Greenlet self.thread = Greenlet(self.run) else: self.thread = threading.Thread(target=self.run) self.thread.setDaemon(True) # Don't hang whole test if thread hangs self.thread.start() @property def alive(self): if self.use_greenlets: return not self.thread.dead else: return self.thread.isAlive() def join(self): self.thread.join(20) if self.use_greenlets: msg = "Greenlet timeout" else: msg = "Thread timeout" assert not self.alive, msg self.thread = None def run(self): self.run_mongo_thread() # No exceptions thrown self.passed = True def run_mongo_thread(self): raise NotImplementedError() def disconnect_client(self): if isinstance(self.client, MongoClient): self.client.close() else: # Don't kill the replica set monitor. self.client.disconnect()
def test_socket_reclamation(self): # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host,port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") force_reclaim_sockets(cx_pool, 1) # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
def check_stream(client, namespace, stream, start, end, limit, timeout, latency): def run(): for event in client.get(stream, start, end, limit=limit, timeout=latency): # Yeah, I'm useless. pass read_greenlet = Greenlet(run) read_greenlet.start() read_greenlet.join(timeout) if not read_greenlet.ready(): read_greenlet.kill() success = False else: success = read_greenlet.successful() return success
def test_reply_reconcile(db, config, message, sync_client): from inbox.server.models.tables.base import Message, SpoolMessage from inbox.server.models.tables.imap import ImapAccount from inbox.server.sendmail.base import reply, recipients to, subject, body = message attachment = None cc = '*****@*****.**' bcc = None account = db.session.query(ImapAccount).get(ACCOUNT_ID) # Create email message, store a local copy + send it: reply(NAMESPACE_ID, account, THREAD_ID, recipients(to, cc, bcc), subject, body, attachment) # Sync to verify reconciliation: synclet = Greenlet(sync_client.start_sync, ACCOUNT_ID) synclet.start() print '\nSyncing...' Greenlet.join(synclet, timeout=60) sync_client.stop_sync(ACCOUNT_ID) spool_messages = db.session.query(SpoolMessage).\ filter_by(subject=THREAD_TOPIC).all() assert len(spool_messages) == 1, 'spool message missing' resolved_message_id = spool_messages[0].resolved_message_id assert resolved_message_id, 'spool message not reconciled' inbox_uid = spool_messages[0].inbox_uid thread_id = spool_messages[0].thread_id g_thrid = spool_messages[0].g_thrid killall(synclet) reconciled_message = db.session.query(Message).get(resolved_message_id) assert reconciled_message.inbox_uid == inbox_uid,\ 'spool message, reconciled message have different inbox_uids' assert reconciled_message.thread_id == thread_id,\ 'spool message, reconciled message have different thread_ids' assert reconciled_message.g_thrid == g_thrid,\ 'spool message, reconciled message have different g_thrids'
def test_job_queue_guarantees_all_jobs(): queue = lets.JobQueue() xs = [] def f(x): gevent.sleep(0.01) xs.append(x) queue.put(Greenlet(f, 0)) queue.put(Greenlet(f, 1)) g = Greenlet(f, 2) queue.put(g) g.join() gevent.sleep(0) # before 0.0.23, the worker has done but the worker pool is still full. # before 0.0.12, the final job won't be scheduled. queue.put(Greenlet(f, 3)) queue.join() assert xs == [0, 1, 2, 3]
def test_job_queue_exited(): results = [] def f(x, delay=0): gevent.sleep(delay) results.append(x) return x queue = lets.JobQueue() g1 = Greenlet(f, 1, 0.1) g2 = Greenlet(f, 2, 0.1) queue.put(g1) queue.put(g2) g1.join() queue.kill() queue.join() assert results == [1] assert g1.get() == 1 assert isinstance(g2.get(), gevent.GreenletExit)
def start(self): print('Starting message server') for forwardAddress in self.forwardAddresses: client = MessageServerClient(forwardAddress) server.forwardClients.add(client) self._connect() self._storePidInPidFile() gevent.core.signal(signal.SIGHUP, self.stop) gevent.core.signal(signal.SIGINT, self.stop) gevent.core.signal(signal.SIGTERM, self.stop) greenlet = Greenlet(self.receiveMessages) greenlet.link_exception(self._logGreenletError) greenlet2 = Greenlet(self.processLogMessages) greenlet2.link_exception(self._logGreenletError) greenlet2.start() greenlet3 = Greenlet(self._timer) greenlet3.link_exception(self._logGreenletError) greenlet3.start() greenlet.start() storeLocallyStr = str(self.storeLocally) addresses = [client.address for client in self.forwardClients] addressesStr = ', '.join(addresses) print(('''\ Message server started listens on: %s stores locally: %s forwards to: %s pid: %d pid file: %s''' % (self._address, storeLocallyStr, addressesStr, self._pid, self._pidFile))) # Wait until the log server stops (main greenlet). try: greenlet.join() except KeyboardInterrupt: # Ignore this error. pass
class MongoThread(object): """A thread, or a greenlet, that uses a MongoClient""" def __init__(self, test_case): self.use_greenlets = test_case.use_greenlets self.client = test_case.c self.db = self.client[DB] self.ut = test_case self.passed = False def start(self): if self.use_greenlets: # A Gevent extended Greenlet self.thread = Greenlet(self.run) else: self.thread = threading.Thread(target=self.run) self.thread.setDaemon( True) # Don't hang whole test if thread hangs self.thread.start() @property def alive(self): if self.use_greenlets: return not self.thread.dead else: return self.thread.isAlive() def join(self): self.thread.join(10) if self.use_greenlets: msg = "Greenlet timeout" else: msg = "Thread timeout" assert not self.alive, msg self.thread = None def run(self): self.run_mongo_thread() # No exceptions thrown self.passed = True def run_mongo_thread(self): raise NotImplementedError()
def test_send_reconcile(db, config, message, sync_client): from inbox.server.models.tables.base import Message, SpoolMessage from inbox.server.models.tables.imap import ImapAccount from inbox.server.sendmail.base import send, recipients to, subject, body = message attachment = None cc = '*****@*****.**' bcc = None # Create email message, store a local copy + send it: account = db.sesson.query(ImapAccount).get(ACCOUNT_ID) send(account, recipients(to, cc, bcc), subject, body, attachment) # Sync to verify reconciliation: synclet = Greenlet(sync_client.start_sync, ACCOUNT_ID) synclet.start() Greenlet.join(synclet, timeout=60) sync_client.stop_sync(ACCOUNT_ID) spool_messages = db.session.query(SpoolMessage).\ filter_by(subject=subject).all() assert len(spool_messages) == 1, 'spool message missing' resolved_message_id = spool_messages[0].resolved_message_id assert resolved_message_id, 'spool message not reconciled' inbox_uid = spool_messages[0].inbox_uid thread_id = spool_messages[0].thread_id killall(synclet) reconciled_message = db.session.query(Message).get(resolved_message_id) assert reconciled_message.inbox_uid == inbox_uid,\ 'spool message, reconciled message have different inbox_uids' assert reconciled_message.thread_id == thread_id,\ 'spool message, reconciled message have different thread_ids'
class TemperatureMonitor(Service): __service__ = 'temperature_monitor' def start(self): patch_all() # :D self.background = Background() gevent.signal(signal.SIGTERM, self.background.kill) gevent.signal(signal.SIGINT, self.background.kill) self.service_greenlet = Greenlet(super(TemperatureMonitor, self).start) gevent.signal(signal.SIGTERM, self.service_greenlet.kill) gevent.signal(signal.SIGINT, self.service_greenlet.kill) self.background.start() self.service_greenlet.start() self.service_greenlet.join() def sleep(self, seconds): sleep(seconds) def handle(self, obj): self.logger.debug(u"Request {0}".format(obj.metadata))
class MongoThread(object): """A thread, or a greenlet, that uses a Connection""" def __init__(self, test_case): self.use_greenlets = test_case.use_greenlets self.connection = test_case.c self.db = self.connection[DB] self.ut = test_case self.passed = False def start(self): if self.use_greenlets: # A Gevent extended Greenlet self.thread = Greenlet(self.run) else: self.thread = threading.Thread(target=self.run) self.thread.setDaemon( True) # Don't hang whole test if thread hangs self.thread.start() def join(self): self.thread.join(300) if self.use_greenlets: assert self.thread.dead, "Greenlet timeout" else: assert not self.thread.isAlive(), "Thread timeout" self.thread = None def run(self): self.run_mongo_thread() # No exceptions thrown self.passed = True def run_mongo_thread(self): raise NotImplementedError()
class MongoThread(object): """A thread, or a greenlet, that uses a Connection""" def __init__(self, test_case): self.use_greenlets = test_case.use_greenlets self.connection = test_case.c self.db = self.connection[DB] self.ut = test_case self.passed = False def start(self): if self.use_greenlets: # A Gevent extended Greenlet self.thread = Greenlet(self.run) else: self.thread = threading.Thread(target=self.run) self.thread.setDaemon(True) # Don't hang whole test if thread hangs self.thread.start() def join(self): self.thread.join(300) if self.use_greenlets: assert self.thread.dead, "Greenlet timeout" else: assert not self.thread.isAlive(), "Thread timeout" self.thread = None def run(self): self.run_mongo_thread() # No exceptions thrown self.passed = True def run_mongo_thread(self): raise NotImplementedError()
def join(self): self._source.join() self._pool.join() Greenlet.join(self)
class FakeClient(object): """ A fake client with persistent connection. Driven by a dedicated greenlet, it will die trying to operate by the rules from the script orderly, round and round. """ def __init__(self, swarm, server, script_): self._swarm = swarm self._server = server self._socket = None self._greenlet = Greenlet(self._run) self._status = INITIAL self._prev_status = None self._script = script_ self._id = id(self) def _report(self, status): """ Report to swarm immediately on status change """ if status != self._status: self._swarm.commit(CommandSTATUS(self._id, status, self._status)) self._status, self._prev_status = (status, self._status) def _reconnect_server(self): """ Die trying """ while True: try: # To scatter connect requests time.sleep(randint(1, 20)) self._report(CONNECT) self._disconnect_server() self._socket = create_connection(self._server, 3) self._socket.setsockopt(SOL_SOCKET, SO_RCVBUF, 128) self._socket.setsockopt(SOL_SOCKET, SO_SNDBUF, 1024) break except socket.error as e: # A fact: `socket.timeout`, `socket.herror`, and # `socket.gaierror` are all subclasses of `socket.error`. self._report(e.args[0]) continue def _disconnect_server(self): if self._socket: self._socket.close() self._socket = None def _run(self): try: self._report(STARTED) self._reconnect_server() while True: try: self._report(ACTING) script.execute(self, self._script) self._report(STANDBY) except (socket.error, BenchException) as e: self._report(e.args[0]) self._reconnect_server() except GreenletExit: self._report(KILLED) except: self._report(FATAL) # let gevent print this exception raise finally: self._disconnect_server() def start(self): self._greenlet.start() def stop(self): self._greenlet.kill() self._greenlet.join() def send_for_reply(self, data, reply_parser): """ Called by object of Script. Exceptions raised here should be handled in `_run`. """ self._socket.sendall(data) need = reply_parser(None) while need > 0: data = self._socket.recv(need) if not data: raise ServerClosed("server closed") need = reply_parser(data) def send_noreply(self, data): self._socket.sendall(data) def close_connection(self): raise CloseForcibly("client closed")
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ results = { 'find_fast_result': None, 'find_slow_result': None, } cx = get_connection( use_greenlets=self.use_greenlets, auto_start_request=False ) db = cx.pymongo_test db.test.remove(safe=True) db.test.insert({'_id': 1}, safe=True) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" results['find_fast_result'] = list(db.test.find()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses where = delay(2) results['find_slow_result'] = list(db.test.find( {'$where': where} )) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr1 = threading.Thread(target=find_fast) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([{'_id': 1}], results['find_slow_result']) # Fails if there's a bug in socket allocation, since find_fast won't # complete self.assertEqual([{'_id': 1}], results['find_fast_result']) self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
def run_badger_node(myID, N, f, sPK, sSK, ePK, eSK, sendPath, receivePath): ''' Test for the client with random delay channels :param i: the current node index :param N: the number of parties :param t: the number of malicious parties toleranted :return None: ''' assert type(sPK) is boldyreva.TBLSPublicKey assert type(sSK) is boldyreva.TBLSPrivateKey assert type(ePK) is tpke.TPKEPublicKey assert type(eSK) is tpke.TPKEPrivateKey # Create the listening channel recv_queue = listen_to_channel(BASE_PORT + myID) recv = recv_queue.get print 'server started' # Create the sending channels send_queues = [] for i in range(N): port = BASE_PORT + i send_queues.append(connect_to_channel('127.0.0.1', port, myID)) def send(j, obj): send_queues[j].put(obj) # Start the honeybadger instance tx_submit = Queue() def send_to_hyperledger(transactions): global sendConnection for tx in transactions: if os.path.exists(sendPath): if sendConnection is None: print "Opening sending socket at path " + sendPath sendConnection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sendConnection.connect(sendPath) print "sending length " + str(len(tx)) sendConnection.send(struct.pack('!Q', len(tx))) print "sending tx " + tx sendConnection.send(tx) hbbft = HoneyBadgerBFT("sid", myID, 8, N, f, sPK, sSK, ePK, eSK, send, recv, tx_submit.get, send_to_hyperledger, encode=repr, decode=ast.literal_eval) th = Greenlet(hbbft.run) th.parent_args = (N, f) th.name = __file__ + '.honestParty(%d)' % i th.start() if os.path.exists(receivePath): os.remove(receivePath) print "Opening listening socket at path " + receivePath server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) server.bind(receivePath) # Listen for incoming connections server.listen(1) while True: # Wait for a connection connection, client_address = server.accept() try: while True: message = connection.recv(8) if message: print "Message " + message length, = struct.unpack('!Q', message) print length message = connection.recv(length) print message tx_submit.put([message]) else: print >> sys.stderr, 'no more data from', client_address break finally: # Clean up the connection connection.close() os.remove(receivePath) th.join()
import os, time from gevent import Greenlet, sleep l = [] t1 = time.time() for i in xrange(0, 100000): # also try a milion :) t = Greenlet(sleep, 30) t.start() l.append(t) t2 = time.time() print "%i coroutines created in %d seconds." % (len(l), t2 - t1) for t in l: t.join()
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ cx = get_client( use_greenlets=self.use_greenlets, auto_start_request=False ) db = cx.pymongo_test db.test.remove() db.test.insert({'_id': 1}) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With greenlets and the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" self.assertEqual({'_id': 1}, db.test.find_one()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses N seconds per document fn = delay(10) if (is_mongos(db.connection) or not version.at_least(db.connection, (1, 7, 2))): # mongos doesn't support eval so we have to use $where # which is less reliable in this context. self.assertEqual(1, db.test.find({"$where": fn}).count()) else: # 'nolock' allows find_fast to start and finish while we're # waiting for this to complete. self.assertEqual({'ok': 1.0, 'retval': True}, db.command('eval', fn, nolock=True)) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr0.setDaemon(True) gr1 = threading.Thread(target=find_fast) gr1.setDaemon(True) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
def _test_pool(self, use_greenlets, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ if use_greenlets: try: from gevent import monkey, Greenlet except ImportError: raise SkipTest('gevent not installed') # Note we don't do patch_thread() or patch_all() - we're # testing here that patch_thread() is unnecessary for # the connection pool to work properly. monkey.patch_socket() try: results = { 'find_fast_result': None, 'find_slow_result': None, } cx = get_connection( use_greenlets=use_greenlets, auto_start_request=False ) db = cx.pymongo_test db.test.remove(safe=True) db.test.insert({'_id': 1}) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" results['find_fast_result'] = list(db.test.find()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses where = delay(2) results['find_slow_result'] = list(db.test.find( {'$where': where} )) history.append('find_slow done') if use_request: cx.end_request() if use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr1 = threading.Thread(target=find_fast) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([{'_id': 1}], results['find_slow_result']) # Fails, since find_fast doesn't complete self.assertEqual([{'_id': 1}], results['find_fast_result']) self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history) finally: # Undo Gevent patching reload(socket)
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ cx = get_connection(use_greenlets=self.use_greenlets, auto_start_request=False) db = cx.pymongo_test if not version.at_least(db.connection, (1, 7, 2)): raise SkipTest("Need at least MongoDB version 1.7.2 to use" " db.eval(nolock=True)") db.test.remove(safe=True) db.test.insert({'_id': 1}, safe=True) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With greenlets and the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" self.assertEqual({'_id': 1}, db.test.find_one()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses 5 sec. 'nolock' allows find_fast # to start and finish while we're waiting for this. fn = delay(5) self.assertEqual({ 'ok': 1.0, 'retval': True }, db.command('eval', fn, nolock=True)) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr0.setDaemon(True) gr1 = threading.Thread(target=find_fast) gr1.setDaemon(True) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
# Block until all threads complete. gevent.joinall(threads) """ def foo(): print "foo" print gevent.getcurrent() gevent.sleep(2) return "foo" def foo2(green): print("foo2") print gevent.getcurrent() return "foo2" print gevent.getcurrent() t = Greenlet(foo) print t.ready() t.start() t.link(foo2) t.join(0) #t.kill() print "yes" print t.ready() print t.successful() #print t.get() print t.value
#!/usr/bin/python import random import gevent from gevent import Greenlet def thrFunc(s): print "%s start.\n" % s gevent.sleep(2) print "%s end.\n" % s gThr = Greenlet(thrFunc, "startjoin1") gThr.start() gThr.join() gThr = Greenlet(thrFunc, "startjoin2") gThr.start() gThr.join() print "+++++++++" gThr = Greenlet(thrFunc, "start1") gThr.start() gThr = Greenlet(thrFunc, "start2") gThr.start() gThr = Greenlet(thrFunc, "start3") gThr.start() gThr = Greenlet(thrFunc, "start4") gThr.start() print "+++++++++" gThr1 = Greenlet.spawn(thrFunc, "spawn1")
import gevent from gevent import Greenlet import time def foo(): print 'foo' print 'foo again' return "haha" def bar(): print 'bar' gevent.sleep(0) print 'bar again' g = Greenlet(foo) class Study(object): def __call__(self, args): print "callback" case = Study() g.link(case) g.start() g.join()
class BaseMailSyncMonitor(Greenlet): """ The SYNC_MONITOR_CLS for all mail sync providers should subclass this. Parameters ---------- account_id : int Which account to sync. email_address : str Email address for `account_id`. provider : str Provider for `account_id`. heartbeat : int How often to check for commands. """ def __init__(self, account, heartbeat=1): bind_context(self, 'mailsyncmonitor', account.id) self.shutdown = event.Event() # how often to check inbox, in seconds self.heartbeat = heartbeat self.log = log.new(component='mail sync', account_id=account.id) self.account_id = account.id self.namespace_id = account.namespace.id self.email_address = account.email_address self.provider_name = account.verbose_provider Greenlet.__init__(self) def _run(self): try: return retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=self.log) except GreenletExit: self._cleanup() raise def _run_impl(self): self.sync = Greenlet(retry_with_logging, self.sync, account_id=self.account_id, provider=self.provider_name, logger=self.log) self.sync.start() self.sync.join() if self.sync.successful(): return self._cleanup() self.log.error('mail sync should run forever', provider=self.provider_name, account_id=self.account_id, exc=self.sync.exception) raise self.sync.exception def sync(self): raise NotImplementedError def _cleanup(self): self.sync.kill() with session_scope(self.namespace_id) as mailsync_db_session: map(lambda x: x.set_stopped(mailsync_db_session), self.folder_monitors) self.folder_monitors.kill()
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ cx = get_connection( use_greenlets=self.use_greenlets, auto_start_request=False ) db = cx.pymongo_test if not version.at_least(db.connection, (1, 7, 2)): raise SkipTest("Need at least MongoDB version 1.7.2 to use" " db.eval(nolock=True)") db.test.remove(safe=True) db.test.insert({'_id': 1}, safe=True) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With greenlets and the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" self.assertEqual({'_id': 1}, db.test.find_one()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses 5 sec. 'nolock' allows find_fast # to start and finish while we're waiting for this. fn = delay(5) self.assertEqual( {'ok': 1.0, 'retval': True}, db.command('eval', fn, nolock=True)) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr0.setDaemon(True) gr1 = threading.Thread(target=find_fast) gr1.setDaemon(True) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
def do_test(): if use_greenlets: try: from gevent import Greenlet from gevent import monkey # Note we don't do patch_thread() or patch_all() - we're # testing here that patch_thread() is unnecessary for # the connection pool to work properly. monkey.patch_socket() except ImportError: outcome.value = SKIP return cx = get_connection( use_greenlets=use_greenlets, auto_start_request=False ) db = cx.pymongo_test db.test.remove(safe=True) db.test.insert({'_id': 1}) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" results['find_fast_result'] = list(db.test.find()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses for half a second where = delay(0.5) results['find_slow_result'] = list(db.test.find( {'$where': where} )) history.append('find_slow done') if use_request: cx.end_request() if use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr1 = threading.Thread(target=find_fast) gr0.start() time.sleep(0.1) gr1.start() gr0.join() gr1.join() self.assertEqual([{'_id': 1}], results['find_slow_result']) # Fails, since find_fast doesn't complete self.assertEqual([{'_id': 1}], results['find_fast_result']) self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history) outcome.value = SUCCESS
def _test_pool(self, use_request): """ Test that the connection pool prevents both threads and greenlets from using a socket at the same time. Sequence: gr0: start a slow find() gr1: start a fast find() gr1: get results gr0: get results """ cx = get_client(use_greenlets=self.use_greenlets, auto_start_request=False) db = cx.pymongo_test db.test.remove() db.test.insert({'_id': 1}) history = [] def find_fast(): if use_request: cx.start_request() history.append('find_fast start') # With greenlets and the old connection._Pool, this would throw # AssertionError: "This event is already used by another # greenlet" self.assertEqual({'_id': 1}, db.test.find_one()) history.append('find_fast done') if use_request: cx.end_request() def find_slow(): if use_request: cx.start_request() history.append('find_slow start') # Javascript function that pauses N seconds per document fn = delay(10) if (is_mongos(db.connection) or not version.at_least(db.connection, (1, 7, 2))): # mongos doesn't support eval so we have to use $where # which is less reliable in this context. self.assertEqual(1, db.test.find({"$where": fn}).count()) else: # 'nolock' allows find_fast to start and finish while we're # waiting for this to complete. self.assertEqual({ 'ok': 1.0, 'retval': True }, db.command('eval', fn, nolock=True)) history.append('find_slow done') if use_request: cx.end_request() if self.use_greenlets: gr0, gr1 = Greenlet(find_slow), Greenlet(find_fast) gr0.start() gr1.start_later(.1) else: gr0 = threading.Thread(target=find_slow) gr0.setDaemon(True) gr1 = threading.Thread(target=find_fast) gr1.setDaemon(True) gr0.start() time.sleep(.1) gr1.start() gr0.join() gr1.join() self.assertEqual([ 'find_slow start', 'find_fast start', 'find_fast done', 'find_slow done', ], history)
class BaseMailSyncMonitor(Greenlet): """ The SYNC_MONITOR_CLS for all mail sync providers should subclass this. Parameters ---------- account_id : int Which account to sync. email_address : str Email address for `account_id`. provider : str Provider for `account_id`. heartbeat : int How often to check for commands. """ def __init__(self, account, heartbeat=1): bind_context(self, "mailsyncmonitor", account.id) self.shutdown = event.Event() # how often to check inbox, in seconds self.heartbeat = heartbeat self.log = log.new(component="mail sync", account_id=account.id) self.account_id = account.id self.namespace_id = account.namespace.id self.email_address = account.email_address self.provider_name = account.verbose_provider Greenlet.__init__(self) def _run(self): try: return retry_with_logging( self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=self.log, ) except GreenletExit: self._cleanup() raise def _run_impl(self): self.sync = Greenlet( retry_with_logging, self.sync, account_id=self.account_id, provider=self.provider_name, logger=self.log, ) self.sync.start() self.sync.join() if self.sync.successful(): return self._cleanup() self.log.error( "mail sync should run forever", provider=self.provider_name, account_id=self.account_id, exc=self.sync.exception, ) raise self.sync.exception def sync(self): raise NotImplementedError def _cleanup(self): self.sync.kill() with session_scope(self.namespace_id) as mailsync_db_session: map(lambda x: x.set_stopped(mailsync_db_session), self.folder_monitors) self.folder_monitors.kill()
class ConsoleAdapter(object): """ This adapter will run Jeev in console mode, listening to stdin for messages, and writing outgoing messages to stdout. """ def __init__(self, jeev, opts): self._jeev = jeev self._opts = opts self._stdin = None self._stdout = None self._reader = None self._channel = opts.get('console_channel', 'console') self._user = opts.get('console_user', 'user') def _read_stdin(self): self._stdout.write(">>> Jeev Console Adapater\n") self._stdout.write(">>> Switch channel using \c channel_name\n") self._stdout.write(">>> Switch user using \u user_name\n") self._stdout.flush() while True: self._stdout.write('[%s@%s] > ' % (self._user, self._channel)) self._stdout.flush() line = self._stdin.readline() if not line: break if line.startswith('\c'): self._channel = line[2:].strip().lstrip('#') self._stdout.write("Switched channel to #%s\n" % self._channel) self._stdout.flush() elif line.startswith('\u'): self._user = line[2:].strip() self._stdout.write("Switched user %s\n" % self._user) self._stdout.flush() else: message = Message({}, self._channel, self._user, line.strip()) self._jeev._handle_message(message) def start(self): self._reader = Greenlet(self._read_stdin) self._stdin = FileObject(sys.stdin) self._stdout = FileObject(sys.stdout) self._reader.start() def stop(self): self._reader.kill() self._reader = None def join(self): self._reader.join() def send_message(self, channel, message): self._stdout.write('\r< [#%s] %s\n' % (channel, message)) self._stdout.write('[%s@%s] > ' % (self._user, self._channel)) self._stdout.flush() def send_messages(self, channel, *messages): for message in messages: self.send_message(channel, message)
def test_socket_reclamation(self): if sys.platform.startswith('java'): raise SkipTest("Jython can't do socket reclamation") # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host, port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) cx_pool.maybe_return_socket(sock_info) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") # In Gevent after 0.13.8, join() returns before the Greenlet.link # callback fires. Give it a moment to reclaim the socket. gevent.sleep(0.1) else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") # Make sure thread is really gone time.sleep(1) if 'PyPy' in sys.version: gc.collect() # Access the thread local from the main thread to trigger the # ThreadVigil's delete callback, returning the request socket to # the pool. # In Python 2.7.0 and lesser, a dead thread's locals are deleted # and those locals' weakref callbacks are fired only when another # thread accesses the locals and finds the thread state is stale, # see http://bugs.python.org/issue1868. Accessing the thread # local from the main thread is a necessary part of this test, and # realistic: in a multithreaded web server a new thread will access # Pool._ident._local soon after an old thread has died. cx_pool._ident.get() # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock)) self.assertEqual(0, len(cx_pool._tid_to_sock))
import os, time from gevent import Greenlet, sleep l = [] t1 = time.time() for i in xrange(0, 100000): # also try a milion :) t = Greenlet(sleep, 30) t.start() l.append(t) t2 = time.time() print "%i coroutines created in %d seconds." % (len(l), t2-t1) for t in l: t.join()
def test_socket_reclamation(self): if sys.platform.startswith('java'): raise SkipTest("Jython can't do socket reclamation") # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host,port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) cx_pool.maybe_return_socket(sock_info) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") # In Gevent after 0.13.8, join() returns before the Greenlet.link # callback fires. Give it a moment to reclaim the socket. gevent.sleep(0.1) else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") # Make sure thread is really gone time.sleep(1) if 'PyPy' in sys.version: gc.collect() # Access the thread local from the main thread to trigger the # ThreadVigil's delete callback, returning the request socket to # the pool. # In Python 2.7.0 and lesser, a dead thread's locals are deleted # and those locals' weakref callbacks are fired only when another # thread accesses the locals and finds the thread state is stale, # see http://bugs.python.org/issue1868. Accessing the thread # local from the main thread is a necessary part of this test, and # realistic: in a multithreaded web server a new thread will access # Pool._ident._local soon after an old thread has died. cx_pool._ident.get() # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock)) self.assertEqual(0, len(cx_pool._tid_to_sock))
#!/usr/bin/python import random import gevent from gevent import Greenlet def thrFunc(s): print "%s start.\n" %s gevent.sleep(2) print "%s end.\n" %s gThr = Greenlet(thrFunc,"startjoin1") gThr.start() gThr.join() gThr = Greenlet(thrFunc,"startjoin2") gThr.start() gThr.join() print "+++++++++" gThr = Greenlet(thrFunc,"start1") gThr.start() gThr = Greenlet(thrFunc,"start2") gThr.start() gThr = Greenlet(thrFunc,"start3") gThr.start() gThr = Greenlet(thrFunc,"start4") gThr.start() print "+++++++++" gThr1 = Greenlet.spawn(thrFunc,"spawn1") gThr2 = Greenlet.spawn(thrFunc,"spawn2") gThr2.join()