def _run_impl(self): sync = Greenlet(retry_and_report_killed, self.sync, account_id=self.account_id, logger=self.log, fail_classes=self.retry_fail_classes) sync.start() while not sync.ready(): try: cmd = self.inbox.get_nowait() if not self.process_command(cmd): # ctrl-c, basically! self.log.info("Stopping sync", email=self.email_address) # make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) self.folder_monitors.kill() return except Empty: sleep(self.heartbeat) if sync.successful(): self.folder_monitors.kill() return # We just want the name of the exception so don't bother with # sys.exc_info() self.log.error('mail sync should run forever', provider=self.provider_name, account_id=self.account_id, exception=type(sync.exception).__name__) raise sync.exception
def _run_impl(self): sync = Greenlet(retry_with_logging, self.sync, account_id=self.account_id, logger=self.log) sync.start() while not sync.ready(): if self.shutdown.is_set(): # Ctrl-c, basically! self.log.info('Stopping sync', email=self.email_address, account_id=self.account_id) # Make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) return self._cleanup() else: sleep(self.heartbeat) if sync.successful(): return self._cleanup() # We just want the name of the exception so don't bother with # sys.exc_info() self.log.error('mail sync should run forever', provider=self.provider_name, account_id=self.account_id, exception=type(sync.exception).__name__) raise sync.exception
def _run_impl(self): sync = Greenlet(retry_and_report_killed, self.sync, account_id=self.account_id, logger=self.log) sync.link_value(lambda _: report_stopped(account_id=self.account_id)) sync.start() while not sync.ready(): try: cmd = self.inbox.get_nowait() if not self.process_command(cmd): # ctrl-c, basically! self.log.info("Stopping sync", email=self.email_address) # make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) self.folder_monitors.kill() return except Empty: sleep(self.heartbeat) if sync.successful(): self.folder_monitors.kill() return self.log.error("mail sync should run forever", provider=self.provider_name, account_id=self.account_id) raise sync.exception
class Channel(object): def __init__(self, socket): self.socket = socket self.running = False self.incoming = Queue(None) self.outgoing = Queue(None) self.reader = Greenlet(self.do_read) self.writer = Greenlet(self.do_write) def do_read(self): while self.running: data = self.socket.receive() if not data: break if data: self.incoming.put(parse_message(data)) def do_write(self): while self.running: msg = self.outgoing.get() self.socket.send(flatten_message(msg)) def is_running(): return self.running and not any( [self.reader.ready(), self.writier.ready()]) def run(self): self.running = True self.reader.start() self.writer.start() def wait(self): self.running = False gevent.killall([self.reader, self.writer]) def receive(self): return self.incoming.get() def send(self, type_, data): return self.outgoing.put(message(type_, data)) def send_ping(self): return self.send('ping', time.time()) def send_spawn(self, avatar): return self.send('spawn', avatar.stat()) def send_die(self, avatar): return self.send('die', avatar.uid) def send_state(self, avatars): return self.send('state', [i.stat() for i in avatars]) def send_update(self, avatar): return self.send('update', avatar.stat())
def test_socket_reclamation(self): # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host,port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") force_reclaim_sockets(cx_pool, 1) # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
def check_stream(client, namespace, stream, start, end, limit, timeout, latency): def run(): for event in client.get(stream, start, end, limit=limit, timeout=latency): # Yeah, I'm useless. pass read_greenlet = Greenlet(run) read_greenlet.start() read_greenlet.join(timeout) if not read_greenlet.ready(): read_greenlet.kill() success = False else: success = read_greenlet.successful() return success
def _run_impl(self): sync = Greenlet(retry_and_report_killed, self.sync, account_id=self.account_id, logger=self.log) sync.link_value(lambda _: report_stopped(account_id=self.account_id)) sync.start() while not sync.ready(): try: cmd = self.inbox.get_nowait() if not self.process_command(cmd): # ctrl-c, basically! self.log.info("Stopping sync for {0}".format( self.email_address)) # make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) self.folder_monitors.kill() return except Empty: sleep(self.heartbeat) assert not sync.successful(), \ "mail sync for {} account {} should run forever!"\ .format(self.provider, self.account_id) raise sync.exception
def test_socket_reclamation(self): if sys.platform.startswith('java'): raise SkipTest("Jython can't do socket reclamation") # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host,port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) cx_pool.maybe_return_socket(sock_info) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") # In Gevent after 0.13.8, join() returns before the Greenlet.link # callback fires. Give it a moment to reclaim the socket. gevent.sleep(0.1) else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") # Make sure thread is really gone time.sleep(1) if 'PyPy' in sys.version: gc.collect() # Access the thread local from the main thread to trigger the # ThreadVigil's delete callback, returning the request socket to # the pool. # In Python 2.7.0 and lesser, a dead thread's locals are deleted # and those locals' weakref callbacks are fired only when another # thread accesses the locals and finds the thread state is stale, # see http://bugs.python.org/issue1868. Accessing the thread # local from the main thread is a necessary part of this test, and # realistic: in a multithreaded web server a new thread will access # Pool._ident._local soon after an old thread has died. cx_pool._ident.get() # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock)) self.assertEqual(0, len(cx_pool._tid_to_sock))
def test_socket_reclamation(self): if sys.platform.startswith('java'): raise SkipTest("Jython can't do socket reclamation") # Check that if a thread starts a request and dies without ending # the request, that the socket is reclaimed into the pool. cx_pool = self.get_pool( pair=(host, port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, ) self.assertEqual(0, len(cx_pool.sockets)) lock = None the_sock = [None] def leak_request(): self.assertEqual(NO_REQUEST, cx_pool._get_request_state()) cx_pool.start_request() self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state()) sock_info = cx_pool.get_socket() self.assertEqual(sock_info, cx_pool._get_request_state()) the_sock[0] = id(sock_info.sock) cx_pool.maybe_return_socket(sock_info) if not self.use_greenlets: lock.release() if self.use_greenlets: g = Greenlet(leak_request) g.start() g.join(1) self.assertTrue(g.ready(), "Greenlet is hung") # In Gevent after 0.13.8, join() returns before the Greenlet.link # callback fires. Give it a moment to reclaim the socket. gevent.sleep(0.1) else: lock = thread.allocate_lock() lock.acquire() # Start a thread WITHOUT a threading.Thread - important to test that # Pool can deal with primitive threads. thread.start_new_thread(leak_request, ()) # Join thread acquired = lock.acquire() self.assertTrue(acquired, "Thread is hung") # Make sure thread is really gone time.sleep(1) if 'PyPy' in sys.version: gc.collect() # Access the thread local from the main thread to trigger the # ThreadVigil's delete callback, returning the request socket to # the pool. # In Python 2.7.0 and lesser, a dead thread's locals are deleted # and those locals' weakref callbacks are fired only when another # thread accesses the locals and finds the thread state is stale, # see http://bugs.python.org/issue1868. Accessing the thread # local from the main thread is a necessary part of this test, and # realistic: in a multithreaded web server a new thread will access # Pool._ident._local soon after an old thread has died. cx_pool._ident.get() # Pool reclaimed the socket self.assertEqual(1, len(cx_pool.sockets)) self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock)) self.assertEqual(0, len(cx_pool._tid_to_sock))
# Block until all threads complete. gevent.joinall(threads) """ def foo(): print "foo" print gevent.getcurrent() gevent.sleep(2) return "foo" def foo2(green): print("foo2") print gevent.getcurrent() return "foo2" print gevent.getcurrent() t = Greenlet(foo) print t.ready() t.start() t.link(foo2) t.join(0) #t.kill() print "yes" print t.ready() print t.successful() #print t.get() print t.value
def _run_impl(self): sync = Greenlet(retry_and_report_killed, self.sync, account_id=self.account_id, logger=self.log) sync.start() while not sync.ready(): try: cmd = self.inbox.get_nowait() if not self.process_command(cmd): # ctrl-c, basically! self.log.info("Stopping sync", email=self.email_address) # make sure the parent can't start/stop any folder monitors # first sync.kill(block=True) self.folder_monitors.kill() return except Empty: sleep(self.heartbeat) if sync.successful(): self.folder_monitors.kill() return self.log.error("mail sync should run forever", provider=self.provider_name, account_id=self.account_id) raise sync.exception