def test_greenlet_sockets(self):
# Check that Pool gives two sockets to two greenlets
try:
import greenlet
except ImportError:
raise SkipTest('greenlet not installed')
cx_pool = pool.Pool(pair=(host, port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
use_greenlets=True)
socks = []
def get_socket():
cx_pool.start_request()
socks.append(cx_pool.get_socket())
looplet([
greenlet.greenlet(get_socket),
greenlet.greenlet(get_socket),
])
self.assertEqual(2, len(socks))
self.assertNotEqual(socks[0], socks[1])
def test_greenlet_sockets(self):
# Check that Pool gives two sockets to two greenlets
try:
import greenlet
import gevent
except ImportError:
raise SkipTest('gevent not installed')
from pymongo import thread_util_gevent
cx_pool = pool.Pool(
pair=(host,port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
thread_support_module=thread_util_gevent)
socks = []
def get_socket():
cx_pool.start_request()
socks.append(cx_pool.get_socket())
looplet([
greenlet.greenlet(get_socket),
greenlet.greenlet(get_socket),
])
self.assertEqual(2, len(socks))
self.assertNotEqual(socks[0], socks[1])
def _test_counter(self, use_greenlets):
if use_greenlets:
from pymongo import thread_util_gevent
thread_support_module = thread_util_gevent
else:
from pymongo import thread_util_threading
thread_support_module = thread_util_threading
counter = thread_util.Counter(thread_support_module)
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
done = set()
def f(n):
for i in xrange(n):
self.assertEqual(i, counter.get())
self.assertEqual(i + 1, counter.inc())
for i in xrange(n, 0, -1):
self.assertEqual(i, counter.get())
self.assertEqual(i - 1, counter.dec())
self.assertEqual(0, counter.get())
# Extra decrements have no effect
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
done.add(n)
if use_greenlets:
greenlets = [
greenlet.greenlet(my_partial(f, i)) for i in xrange(10)]
looplet(greenlets)
else:
threads = [
threading.Thread(target=my_partial(f, i)) for i in xrange(10)]
for t in threads:
t.start()
for t in threads:
t.join()
self.assertEqual(10, len(done))
def _test_counter(self, use_greenlets):
counter = thread_util.Counter(use_greenlets)
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
done = set()
def f(n):
for i in xrange(n):
self.assertEqual(i, counter.get())
self.assertEqual(i + 1, counter.inc())
for i in xrange(n, 0, -1):
self.assertEqual(i, counter.get())
self.assertEqual(i - 1, counter.dec())
self.assertEqual(0, counter.get())
# Extra decrements have no effect
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
done.add(n)
if use_greenlets:
greenlets = [
greenlet.greenlet(my_partial(f, i)) for i in xrange(10)
]
looplet(greenlets)
else:
threads = [
threading.Thread(target=my_partial(f, i)) for i in xrange(10)
]
for t in threads:
t.start()
for t in threads:
t.join()
self.assertEqual(10, len(done))
def _test_counter(self, use_greenlets):
counter = thread_util.Counter(use_greenlets)
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
done = set()
def f(n):
for i in xrange(n):
self.assertEqual(i, counter.get())
self.assertEqual(i + 1, counter.inc())
for i in xrange(n, 0, -1):
self.assertEqual(i, counter.get())
self.assertEqual(i - 1, counter.dec())
self.assertEqual(0, counter.get())
# Extra decrements have no effect
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
self.assertEqual(0, counter.dec())
self.assertEqual(0, counter.get())
done.add(n)
if use_greenlets:
greenlets = [
greenlet.greenlet(partial(f, i)) for i in xrange(10)]
looplet(greenlets)
else:
threads = [
threading.Thread(target=partial(f, i)) for i in xrange(10)]
for t in threads:
t.start()
for t in threads:
t.join()
self.assertEqual(10, len(done))
def test_greenlet_sockets(self):
# Check that Pool gives two sockets to two greenlets
try:
import greenlet
except ImportError:
raise SkipTest("greenlet not installed")
cx_pool = pool.Pool(
pair=(host, port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, use_greenlets=True
)
socks = []
def get_socket():
cx_pool.start_request()
socks.append(cx_pool.get_socket())
looplet([greenlet.greenlet(get_socket), greenlet.greenlet(get_socket)])
self.assertEqual(2, len(socks))
self.assertNotEqual(socks[0], socks[1])
def _test_ident(self, use_greenlets):
if 'java' in sys.platform:
raise SkipTest("Can't rely on weakref callbacks in Jython")
ident = thread_util.create_ident(use_greenlets)
ids = set([ident.get()])
unwatched_id = []
done = set([ident.get()]) # Start with main thread's / greenlet's id.
died = set()
class Watched(object):
def __init__(self, ident):
self._my_ident = ident
def before_rendezvous(self):
self.my_id = self._my_ident.get()
ids.add(self.my_id)
def after_rendezvous(self):
assert not self._my_ident.watching()
self._my_ident.watch(lambda ref: died.add(self.my_id))
assert self._my_ident.watching()
done.add(self.my_id)
class Unwatched(Watched):
def before_rendezvous(self):
Watched.before_rendezvous(self)
unwatched_id.append(self.my_id)
def after_rendezvous(self):
Watched.after_rendezvous(self)
self._my_ident.unwatch(self.my_id)
assert not self._my_ident.watching()
if use_greenlets:
class WatchedGreenlet(Watched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
class UnwatchedGreenlet(Unwatched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
looplet([t_watched, t_unwatched])
else:
class WatchedThread(Watched, RendezvousThread):
def __init__(self, ident, state):
Watched.__init__(self, ident)
RendezvousThread.__init__(self, state)
class UnwatchedThread(Unwatched, RendezvousThread):
def __init__(self, ident, state):
Unwatched.__init__(self, ident)
RendezvousThread.__init__(self, state)
state = RendezvousThread.create_shared_state(2)
t_watched = WatchedThread(ident, state)
t_watched.start()
t_unwatched = UnwatchedThread(ident, state)
t_unwatched.start()
RendezvousThread.wait_for_rendezvous(state)
RendezvousThread.resume_after_rendezvous(state)
t_watched.join()
t_unwatched.join()
self.assertTrue(t_watched.passed)
self.assertTrue(t_unwatched.passed)
# Remove references, let weakref callbacks run
del t_watched
del t_unwatched
# Trigger final cleanup in Python <= 2.7.0.
# http://bugs.python.org/issue1868
ident.get()
self.assertEqual(3, len(ids))
self.assertEqual(3, len(done))
# Make sure thread is really gone
slept = 0
while not died and slept < 10:
time.sleep(1)
gc.collect()
slept += 1
self.assertEqual(1, len(died))
self.assertFalse(unwatched_id[0] in died)
def test_greenlet_sockets_with_request(self):
# Verify two assumptions: that start_request() with two greenlets but
# not use_greenlets fails, meaning that the two greenlets will
# share one socket. Also check that start_request() with use_greenlets
# succeeds, meaning that two greenlets will get different sockets.
try:
import greenlet
import gevent
except ImportError:
raise SkipTest('gevent not installed')
from pymongo import thread_util_threading, thread_util_gevent
pool_args = dict(
pair=(host,port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
)
for use_greenlets, use_request, expect_success in [
(True, True, True),
(True, False, False),
(False, True, False),
(False, False, False),
]:
pool_args_cp = pool_args.copy()
if use_greenlets:
pool_args_cp['thread_support_module'] = thread_util_gevent
else:
pool_args_cp['thread_support_module'] = thread_util_threading
cx_pool = pool.Pool(**pool_args_cp)
# Map: greenlet -> socket
greenlet2socks = {}
main = greenlet.getcurrent()
def get_socket_in_request():
# Get a socket from the pool twice, switching contexts each time
if use_request:
cx_pool.start_request()
main.switch()
for _ in range(2):
sock = cx_pool.get_socket()
cx_pool.maybe_return_socket(sock)
greenlet2socks.setdefault(
greenlet.getcurrent(), []
).append(id(sock))
main.switch()
cx_pool.end_request()
greenlets = [
greenlet.greenlet(get_socket_in_request),
greenlet.greenlet(get_socket_in_request),
]
# Run both greenlets to completion
looplet(greenlets)
socks_for_gr0 = greenlet2socks[greenlets[0]]
socks_for_gr1 = greenlet2socks[greenlets[1]]
# Whether we expect requests to work or not, we definitely expect
# greenlet2socks to have the same number of keys and values
self.assertEqual(2, len(greenlet2socks))
self.assertEqual(2, len(socks_for_gr0))
self.assertEqual(2, len(socks_for_gr1))
# If we started a request, then there was a point at which we had
# 2 active sockets, otherwise we always used one.
if use_request and use_greenlets:
self.assertEqual(2, len(cx_pool.sockets))
else:
self.assertEqual(1, len(cx_pool.sockets))
# Again, regardless of whether requests work, a greenlet will get
# the same socket each time it calls get_socket() within a request.
# What we're really testing is that the two *different* greenlets
# get *different* sockets from each other.
self.assertEqual(
socks_for_gr0[0], socks_for_gr0[1],
"Expected greenlet 0 to get the same socket for each call "
"to get_socket()"
)
self.assertEqual(
socks_for_gr1[0], socks_for_gr1[1],
"Expected greenlet 1 to get the same socket for each call "
"to get_socket()"
)
if expect_success:
# We passed use_greenlets=True, so start_request successfully
# distinguished between the two greenlets.
self.assertNotEqual(
socks_for_gr0[0], socks_for_gr1[0],
"Expected two greenlets to get two different sockets"
)
else:
# We passed use_greenlets=False, so start_request didn't
# distinguish between the two greenlets, and it gave them both
# the same socket.
self.assertEqual(
socks_for_gr0[0], socks_for_gr1[0],
"Expected two greenlets to get same socket"
)
def _test_ident(self, use_greenlets):
ident = thread_util.create_ident(use_greenlets)
ids = set([ident.get()])
unwatched_id = []
done = set([ident.get()]) # Start with main thread's / greenlet's id.
died = set()
class Watched(object):
def __init__(self, ident):
self._my_ident = ident
def before_rendezvous(self):
self.my_id = self._my_ident.get()
ids.add(self.my_id)
def after_rendezvous(self):
assert not self._my_ident.watching()
self._my_ident.watch(lambda ref: died.add(self.my_id))
assert self._my_ident.watching()
done.add(self.my_id)
class Unwatched(Watched):
def before_rendezvous(self):
Watched.before_rendezvous(self)
unwatched_id.append(self.my_id)
def after_rendezvous(self):
Watched.after_rendezvous(self)
self._my_ident.unwatch(self.my_id)
assert not self._my_ident.watching()
if use_greenlets:
class WatchedGreenlet(Watched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
class UnwatchedGreenlet(Unwatched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
looplet([t_watched, t_unwatched])
else:
class WatchedThread(Watched, RendezvousThread):
def __init__(self, ident, state):
Watched.__init__(self, ident)
RendezvousThread.__init__(self, state)
class UnwatchedThread(Unwatched, RendezvousThread):
def __init__(self, ident, state):
Unwatched.__init__(self, ident)
RendezvousThread.__init__(self, state)
state = RendezvousThread.create_shared_state(2)
t_watched = WatchedThread(ident, state)
t_watched.start()
t_unwatched = UnwatchedThread(ident, state)
t_unwatched.start()
RendezvousThread.wait_for_rendezvous(state)
RendezvousThread.resume_after_rendezvous(state)
t_watched.join()
t_unwatched.join()
self.assertTrue(t_watched.passed)
self.assertTrue(t_unwatched.passed)
# Remove references, let weakref callbacks run
del t_watched
del t_unwatched
# Trigger final cleanup in Python <= 2.7.0.
# http://bugs.python.org/issue1868
ident.get()
self.assertEqual(3, len(ids))
self.assertEqual(3, len(done))
# Make sure thread is really gone
slept = 0
while not died and slept < 10:
time.sleep(1)
gc.collect()
slept += 1
self.assertEqual(1, len(died))
self.assertFalse(unwatched_id[0] in died)
def test_greenlet_sockets_with_request(self):
# Verify two assumptions: that start_request() with two greenlets but
# not use_greenlets fails, meaning that the two greenlets will
# share one socket. Also check that start_request() with use_greenlets
# succeeds, meaning that two greenlets will get different sockets.
try:
import greenlet
except ImportError:
raise SkipTest('greenlet not installed')
pool_args = dict(
pair=(host, port),
max_size=10,
net_timeout=1000,
conn_timeout=1000,
use_ssl=False,
)
for use_greenlets, use_request, expect_success in [
(True, True, True),
(True, False, False),
(False, True, False),
(False, False, False),
]:
pool_args_cp = pool_args.copy()
pool_args_cp['use_greenlets'] = use_greenlets
cx_pool = pool.Pool(**pool_args_cp)
# Map: greenlet -> socket
greenlet2socks = {}
main = greenlet.getcurrent()
def get_socket_in_request():
# Get a socket from the pool twice, switching contexts each time
if use_request:
cx_pool.start_request()
main.switch()
for _ in range(2):
sock = cx_pool.get_socket()
cx_pool.maybe_return_socket(sock)
greenlet2socks.setdefault(greenlet.getcurrent(),
[]).append(id(sock))
main.switch()
cx_pool.end_request()
greenlets = [
greenlet.greenlet(get_socket_in_request),
greenlet.greenlet(get_socket_in_request),
]
# Run both greenlets to completion
looplet(greenlets)
socks_for_gr0 = greenlet2socks[greenlets[0]]
socks_for_gr1 = greenlet2socks[greenlets[1]]
# Whether we expect requests to work or not, we definitely expect
# greenlet2socks to have the same number of keys and values
self.assertEqual(2, len(greenlet2socks))
self.assertEqual(2, len(socks_for_gr0))
self.assertEqual(2, len(socks_for_gr1))
# If we started a request, then there was a point at which we had
# 2 active sockets, otherwise we always used one.
if use_request and use_greenlets:
self.assertEqual(2, len(cx_pool.sockets))
else:
self.assertEqual(1, len(cx_pool.sockets))
# Again, regardless of whether requests work, a greenlet will get
# the same socket each time it calls get_socket() within a request.
# What we're really testing is that the two *different* greenlets
# get *different* sockets from each other.
self.assertEqual(
socks_for_gr0[0], socks_for_gr0[1],
"Expected greenlet 0 to get the same socket for each call "
"to get_socket()")
self.assertEqual(
socks_for_gr1[0], socks_for_gr1[1],
"Expected greenlet 1 to get the same socket for each call "
"to get_socket()")
if expect_success:
# We passed use_greenlets=True, so start_request successfully
# distinguished between the two greenlets.
self.assertNotEqual(
socks_for_gr0[0], socks_for_gr1[0],
"Expected two greenlets to get two different sockets")
else:
# We passed use_greenlets=False, so start_request didn't
# distinguish between the two greenlets, and it gave them both
# the same socket.
self.assertEqual(socks_for_gr0[0], socks_for_gr1[0],
"Expected two greenlets to get same socket")