def _run_impl(self):
sync = Greenlet(retry_and_report_killed, self.sync,
account_id=self.account_id, logger=self.log,
fail_classes=self.retry_fail_classes)
sync.start()
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync", email=self.email_address)
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
self.folder_monitors.kill()
return
except Empty:
sleep(self.heartbeat)
if sync.successful():
self.folder_monitors.kill()
return
# We just want the name of the exception so don't bother with
# sys.exc_info()
self.log.error('mail sync should run forever',
provider=self.provider_name,
account_id=self.account_id,
exception=type(sync.exception).__name__)
raise sync.exception
def _run_impl(self):
sync = Greenlet(retry_and_report_killed, self.sync,
account_id=self.account_id, logger=self.log)
sync.start()
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync", email=self.email_address)
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
self.folder_monitors.kill()
return
except Empty:
sleep(self.heartbeat)
if sync.successful():
self.folder_monitors.kill()
return
self.log.error("mail sync should run forever",
provider=self.provider_name,
account_id=self.account_id)
raise sync.exception
def _run_impl(self):
sync = Greenlet(retry_with_logging, self.sync,
account_id=self.account_id, logger=self.log)
sync.start()
while not sync.ready():
if self.shutdown.is_set():
# Ctrl-c, basically!
self.log.info('Stopping sync', email=self.email_address,
account_id=self.account_id)
# Make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
return self._cleanup()
else:
sleep(self.heartbeat)
if sync.successful():
return self._cleanup()
# We just want the name of the exception so don't bother with
# sys.exc_info()
self.log.error('mail sync should run forever',
provider=self.provider_name,
account_id=self.account_id,
exception=type(sync.exception).__name__)
raise sync.exception
def _run_impl(self):
sync = Greenlet(retry_and_report_killed,
self.sync,
account_id=self.account_id,
logger=self.log)
sync.link_value(lambda _: report_stopped(account_id=self.account_id))
sync.start()
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync", email=self.email_address)
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
self.folder_monitors.kill()
return
except Empty:
sleep(self.heartbeat)
if sync.successful():
self.folder_monitors.kill()
return
self.log.error("mail sync should run forever",
provider=self.provider_name,
account_id=self.account_id)
raise sync.exception
def _run_impl(self):
sync = Greenlet(retry_with_logging,
self.sync,
account_id=self.account_id,
logger=self.log)
sync.start()
while not sync.ready():
if self.shutdown.is_set():
# Ctrl-c, basically!
self.log.info('Stopping sync',
email=self.email_address,
account_id=self.account_id)
# Make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
return self._cleanup()
else:
sleep(self.heartbeat)
if sync.successful():
return self._cleanup()
# We just want the name of the exception so don't bother with
# sys.exc_info()
self.log.error('mail sync should run forever',
provider=self.provider_name,
account_id=self.account_id,
exception=type(sync.exception).__name__)
raise sync.exception
class Channel(object):
def __init__(self, socket):
self.socket = socket
self.running = False
self.incoming = Queue(None)
self.outgoing = Queue(None)
self.reader = Greenlet(self.do_read)
self.writer = Greenlet(self.do_write)
def do_read(self):
while self.running:
data = self.socket.receive()
if not data:
break
if data:
self.incoming.put(parse_message(data))
def do_write(self):
while self.running:
msg = self.outgoing.get()
self.socket.send(flatten_message(msg))
def is_running():
return self.running and not any(
[self.reader.ready(), self.writier.ready()])
def run(self):
self.running = True
self.reader.start()
self.writer.start()
def wait(self):
self.running = False
gevent.killall([self.reader, self.writer])
def receive(self):
return self.incoming.get()
def send(self, type_, data):
return self.outgoing.put(message(type_, data))
def send_ping(self):
return self.send('ping', time.time())
def send_spawn(self, avatar):
return self.send('spawn', avatar.stat())
def send_die(self, avatar):
return self.send('die', avatar.uid)
def send_state(self, avatars):
return self.send('state', [i.stat() for i in avatars])
def send_update(self, avatar):
return self.send('update', avatar.stat())
def _run_impl(self):
sync = Greenlet(retry_and_report_killed,
self.sync,
account_id=self.account_id,
logger=self.log,
fail_classes=self.retry_fail_classes)
sync.start()
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync", email=self.email_address)
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
self.folder_monitors.kill()
return
except Empty:
sleep(self.heartbeat)
if sync.successful():
self.folder_monitors.kill()
return
# We just want the name of the exception so don't bother with
# sys.exc_info()
self.log.error('mail sync should run forever',
provider=self.provider_name,
account_id=self.account_id,
exception=type(sync.exception).__name__)
raise sync.exception
def test_socket_reclamation(self):
# Check that if a thread starts a request and dies without ending
# the request, that the socket is reclaimed into the pool.
cx_pool = self.get_pool(
pair=(host,port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
)
self.assertEqual(0, len(cx_pool.sockets))
lock = None
the_sock = [None]
def leak_request():
self.assertEqual(NO_REQUEST, cx_pool._get_request_state())
cx_pool.start_request()
self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state())
sock_info = cx_pool.get_socket()
self.assertEqual(sock_info, cx_pool._get_request_state())
the_sock[0] = id(sock_info.sock)
if not self.use_greenlets:
lock.release()
if self.use_greenlets:
g = Greenlet(leak_request)
g.start()
g.join(1)
self.assertTrue(g.ready(), "Greenlet is hung")
else:
lock = thread.allocate_lock()
lock.acquire()
# Start a thread WITHOUT a threading.Thread - important to test that
# Pool can deal with primitive threads.
thread.start_new_thread(leak_request, ())
# Join thread
acquired = lock.acquire()
self.assertTrue(acquired, "Thread is hung")
force_reclaim_sockets(cx_pool, 1)
# Pool reclaimed the socket
self.assertEqual(1, len(cx_pool.sockets))
self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
def check_stream(client, namespace, stream, start, end, limit, timeout,
latency):
def run():
for event in client.get(stream, start, end, limit=limit, timeout=latency):
# Yeah, I'm useless.
pass
read_greenlet = Greenlet(run)
read_greenlet.start()
read_greenlet.join(timeout)
if not read_greenlet.ready():
read_greenlet.kill()
success = False
else:
success = read_greenlet.successful()
return success
def check_stream(client, namespace, stream, start, end, limit, timeout,
latency):
def run():
for event in client.get(stream,
start,
end,
limit=limit,
timeout=latency):
# Yeah, I'm useless.
pass
read_greenlet = Greenlet(run)
read_greenlet.start()
read_greenlet.join(timeout)
if not read_greenlet.ready():
read_greenlet.kill()
success = False
else:
success = read_greenlet.successful()
return success
def _run_impl(self):
sync = Greenlet(retry_and_report_killed, self.sync,
account_id=self.account_id, logger=self.log)
sync.link_value(lambda _: report_stopped(account_id=self.account_id))
sync.start()
while not sync.ready():
try:
cmd = self.inbox.get_nowait()
if not self.process_command(cmd):
# ctrl-c, basically!
self.log.info("Stopping sync for {0}".format(
self.email_address))
# make sure the parent can't start/stop any folder monitors
# first
sync.kill(block=True)
self.folder_monitors.kill()
return
except Empty:
sleep(self.heartbeat)
assert not sync.successful(), \
"mail sync for {} account {} should run forever!"\
.format(self.provider, self.account_id)
raise sync.exception
def test_socket_reclamation(self):
if sys.platform.startswith('java'):
raise SkipTest("Jython can't do socket reclamation")
# Check that if a thread starts a request and dies without ending
# the request, that the socket is reclaimed into the pool.
cx_pool = self.get_pool(
pair=(host,port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
)
self.assertEqual(0, len(cx_pool.sockets))
lock = None
the_sock = [None]
def leak_request():
self.assertEqual(NO_REQUEST, cx_pool._get_request_state())
cx_pool.start_request()
self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state())
sock_info = cx_pool.get_socket()
self.assertEqual(sock_info, cx_pool._get_request_state())
the_sock[0] = id(sock_info.sock)
cx_pool.maybe_return_socket(sock_info)
if not self.use_greenlets:
lock.release()
if self.use_greenlets:
g = Greenlet(leak_request)
g.start()
g.join(1)
self.assertTrue(g.ready(), "Greenlet is hung")
# In Gevent after 0.13.8, join() returns before the Greenlet.link
# callback fires. Give it a moment to reclaim the socket.
gevent.sleep(0.1)
else:
lock = thread.allocate_lock()
lock.acquire()
# Start a thread WITHOUT a threading.Thread - important to test that
# Pool can deal with primitive threads.
thread.start_new_thread(leak_request, ())
# Join thread
acquired = lock.acquire()
self.assertTrue(acquired, "Thread is hung")
# Make sure thread is really gone
time.sleep(1)
if 'PyPy' in sys.version:
gc.collect()
# Access the thread local from the main thread to trigger the
# ThreadVigil's delete callback, returning the request socket to
# the pool.
# In Python 2.7.0 and lesser, a dead thread's locals are deleted
# and those locals' weakref callbacks are fired only when another
# thread accesses the locals and finds the thread state is stale,
# see http://bugs.python.org/issue1868. Accessing the thread
# local from the main thread is a necessary part of this test, and
# realistic: in a multithreaded web server a new thread will access
# Pool._ident._local soon after an old thread has died.
cx_pool._ident.get()
# Pool reclaimed the socket
self.assertEqual(1, len(cx_pool.sockets))
self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
self.assertEqual(0, len(cx_pool._tid_to_sock))
def test_socket_reclamation(self):
if sys.platform.startswith('java'):
raise SkipTest("Jython can't do socket reclamation")
# Check that if a thread starts a request and dies without ending
# the request, that the socket is reclaimed into the pool.
cx_pool = self.get_pool(
pair=(host, port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
)
self.assertEqual(0, len(cx_pool.sockets))
lock = None
the_sock = [None]
def leak_request():
self.assertEqual(NO_REQUEST, cx_pool._get_request_state())
cx_pool.start_request()
self.assertEqual(NO_SOCKET_YET, cx_pool._get_request_state())
sock_info = cx_pool.get_socket()
self.assertEqual(sock_info, cx_pool._get_request_state())
the_sock[0] = id(sock_info.sock)
cx_pool.maybe_return_socket(sock_info)
if not self.use_greenlets:
lock.release()
if self.use_greenlets:
g = Greenlet(leak_request)
g.start()
g.join(1)
self.assertTrue(g.ready(), "Greenlet is hung")
# In Gevent after 0.13.8, join() returns before the Greenlet.link
# callback fires. Give it a moment to reclaim the socket.
gevent.sleep(0.1)
else:
lock = thread.allocate_lock()
lock.acquire()
# Start a thread WITHOUT a threading.Thread - important to test that
# Pool can deal with primitive threads.
thread.start_new_thread(leak_request, ())
# Join thread
acquired = lock.acquire()
self.assertTrue(acquired, "Thread is hung")
# Make sure thread is really gone
time.sleep(1)
if 'PyPy' in sys.version:
gc.collect()
# Access the thread local from the main thread to trigger the
# ThreadVigil's delete callback, returning the request socket to
# the pool.
# In Python 2.7.0 and lesser, a dead thread's locals are deleted
# and those locals' weakref callbacks are fired only when another
# thread accesses the locals and finds the thread state is stale,
# see http://bugs.python.org/issue1868. Accessing the thread
# local from the main thread is a necessary part of this test, and
# realistic: in a multithreaded web server a new thread will access
# Pool._ident._local soon after an old thread has died.
cx_pool._ident.get()
# Pool reclaimed the socket
self.assertEqual(1, len(cx_pool.sockets))
self.assertEqual(the_sock[0], id(one(cx_pool.sockets).sock))
self.assertEqual(0, len(cx_pool._tid_to_sock))
# Block until all threads complete.
gevent.joinall(threads)
"""
def foo():
print "foo"
print gevent.getcurrent()
gevent.sleep(2)
return "foo"
def foo2(green):
print("foo2")
print gevent.getcurrent()
return "foo2"
print gevent.getcurrent()
t = Greenlet(foo)
print t.ready()
t.start()
t.link(foo2)
t.join(0)
#t.kill()
print "yes"
print t.ready()
print t.successful()
#print t.get()
print t.value
