def test_primary_failure(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
primary = c.primary
secondaries = c.secondaries
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
yield self.pause(1)
# Wait for new primary to step up, and for MotorReplicaSetClient
# to detect it.
for _ in range(30):
if c.primary != primary and c.secondaries != secondaries:
break
yield self.pause(1)
else:
self.fail("New primary not detected")
def test_monitor_removes_recovering_member(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
yield assert_read_from_all(self, self.c, partitioned_secondaries,
mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
yield self.pause(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
yield assert_read_from(self, self.c, _partition_node(secondary),
mode)
def test_cert_ssl_validation_hostname_fail(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests e.g.:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
if not test.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
client = motor.MotorClient(host,
port,
ssl=True,
ssl_certfile=CLIENT_PEM)
response = yield client.admin.command('ismaster')
try:
# Create client with hostname 'localhost' or whatever, not
# the name 'server', which is what the server cert presents.
client = motor.MotorClient(host,
port,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM)
yield client.db.collection.find_one()
self.fail("Invalid hostname should have failed")
except ConnectionFailure:
pass
if 'setName' in response:
try:
client = motor.MotorReplicaSetClient(
'%s:%d' % (host, port),
replicaSet=response['setName'],
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM)
yield client.db.collection.find_one()
self.fail("Invalid hostname should have failed")
except ConnectionFailure:
pass
def get_connection_async(
host="127.0.0.1",
port=27017,
database="trunk",
user="******",
password="******",
replica=None,
**kwargs
):
"""Возвращает асинхронное подключение к базе.
:param host: Хост, к которому выполняется подключение
:type host: str
:param port: Порт базы данных
:type port: int
:param user: Имя пользователя базы данных
:type user: str
:param password: Пароль пользователя базы данных
:type password: str
:param replica: Название replicaSet (при наличии)
:type replica: str
:param database: Имя базы данных
:type database: str
"""
if not replica:
con = motor.MotorClient(
"mongodb://{}:{}@{}:{}/{}".format(
user, password,
host, port,
database
))
else:
con = motor.MotorReplicaSetClient(
"mongodb://{}:{}@{}:{}/{}".format(
user, password,
host, port,
database
),
replicaSet=replica,
connectTimeoutMS=1500,
socketTimeoutMS=1500
)
return con[database]
def test_cert_ssl_validation(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests e.g.:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
#
# Also requires an /etc/hosts entry where "server" is resolvable.
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
if not test.env.server_is_resolvable:
raise SkipTest("No hosts entry for 'server'. Cannot validate "
"hostname in the certificate")
if test.env.auth:
raise SkipTest("can't test with auth")
client = motor.MotorClient(test.env.fake_hostname_uri,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
yield client.db.collection.find_one()
response = yield client.admin.command('ismaster')
if 'setName' in response:
if response['primary'].split(":")[0] != 'server':
raise SkipTest("No hosts in the replicaset for 'server'. "
"Cannot validate hostname in the certificate")
client = motor.MotorReplicaSetClient(
test.env.fake_hostname_uri,
replicaSet=response['setName'],
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
yield client.db.collection.find_one()
def __init__(self,
mongo_uri,
max_pool_size,
mongo_database,
memcache_pool=None):
if motor.version_tuple == (0, 1, 2):
self.db = motor.MotorClient(
mongo_uri,
max_pool_size=max_pool_size).open_sync()[mongo_database]
else:
if mongo_uri.find('replicaSet') > 0:
self.db = motor.MotorReplicaSetClient(
mongo_uri,
max_pool_size=max_pool_size,
read_preference=pymongo.ReadPreference.SECONDARY_PREFERRED
)[mongo_database]
else:
self.db = motor.MotorClient(
mongo_uri, max_pool_size=max_pool_size)[mongo_database]
if memcache_pool:
self.mem_client = pylibmc.Client(memcache_pool, binary=True)
def test_open_sync(self):
cx = motor.MotorReplicaSetClient(host,
port,
replicaSet=self.name,
io_loop=self.io_loop)
self.assertFalse(cx.connected)
# open_sync() creates a special IOLoop just to run the connection
# code to completion
self.assertEqual(cx, cx.open_sync())
self.assertTrue(cx.connected)
# IOLoop was restored?
self.assertEqual(self.io_loop, cx.io_loop)
# Really connected?
result = yield cx.pymongo_test.test_collection.find_one({'_id': 0})
self.assertEqual(0, result['_id'])
cx.close()
def test_replica_set_client(self):
cx = motor.MotorReplicaSetClient('%s:%s' % (host, port),
replicaSet=self.name,
io_loop=self.io_loop)
# Can't access databases before connecting
self.assertRaises(pymongo.errors.InvalidOperation,
lambda: cx.some_database_name)
self.assertRaises(pymongo.errors.InvalidOperation,
lambda: cx['some_database_name'])
result = yield cx.open()
self.assertEqual(result, cx)
self.assertTrue(cx.connected)
self.assertTrue(
isinstance(cx.delegate._MongoReplicaSetClient__monitor,
motor.MotorReplicaSetMonitor))
self.assertEqual(self.io_loop,
cx.delegate._MongoReplicaSetClient__monitor.io_loop)
def test_read_with_failover(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
db = c.motor_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
yield db.test.insert([{'foo': i} for i in range(10)], w=w)
self.assertEqual(10, (yield db.test.count()))
db.read_preference = SECONDARY
cursor = db.test.find().batch_size(5)
yield cursor.fetch_next
self.assertEqual(5, cursor.delegate._Cursor__retrieved)
for i in range(5):
cursor.next_object()
ha_tools.kill_primary()
yield self.pause(2)
# Primary failure shouldn't interrupt the cursor
yield cursor.fetch_next
self.assertEqual(10, cursor.delegate._Cursor__retrieved)
def test_secondary_failure(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
primary = c.primary
killed = ha_tools.kill_secondary()
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
yield self.pause(2 * MONITOR_INTERVAL)
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
# Wait for secondary to join, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.secondaries != secondaries:
break
yield self.pause(1)
else:
self.fail("Dead secondary not detected")
def test_secondary_connection(self):
self.c = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.motor_test
yield db.test.remove({}, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
yield db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A connection succeeds no matter the read preference
for kwargs in [
{
'read_preference': PRIMARY
},
{
'read_preference': PRIMARY_PREFERRED
},
{
'read_preference': SECONDARY
},
{
'read_preference': SECONDARY_PREFERRED
},
{
'read_preference': NEAREST
},
]:
client = yield motor.MotorClient(primary_host, primary_port,
**kwargs).open()
self.assertEqual(primary_host, client.host)
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue((yield client.motor_test.test.find_one()))
client = yield motor.MotorClient(secondary_host, secondary_port,
**kwargs).open()
self.assertEqual(secondary_host, client.host)
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue((yield client.motor_test.test.find_one()))
else:
with assert_raises(AutoReconnect):
yield client.motor_test.test.find_one()
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MotorClient
# should do the same for unacknowledged writes.
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % client)
# Test direct connection to an arbiter
client = yield motor.MotorClient(arbiter_host, arbiter_port,
**kwargs).open()
self.assertEqual(arbiter_host, client.host)
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into arbiter connection %s should'
'have raised exception' % client)
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
@gen.coroutine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
):
db = rsc.motor_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
raise gen.Return(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
raise gen.Return(None)
@gen.coroutine
def assert_read_from(member, *args, **kwargs):
for _ in range(10):
used = yield read_from_which_host(c, *args, **kwargs)
self.assertEqual(member, used)
@gen.coroutine
def assert_read_from_all(members, *args, **kwargs):
members = set(members)
all_used = set()
for _ in range(100):
used = yield read_from_which_host(c, *args, **kwargs)
all_used.add(used)
if members == all_used:
raise gen.Return() # Success
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Multiple tags
yield assert_read_from(secondary, SECONDARY_PREFERRED,
self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'primary'})
# No matching secondaries
yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}],
latency=9999999)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from_all([primary, other_secondary],
NEAREST, [bad_tag, {
'dc': 'ny'
}],
latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 milliseconds.
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield assert_read_from(primary, NEAREST)
# Tags override nearness
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
yield assert_read_from(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 milliseconds.
self.set_ping_time(secondary, 0)
yield assert_read_from(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield assert_read_from(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
self.clear_ping_times()
yield assert_read_from_all([primary, other_secondary],
NEAREST, [{
'dc': 'ny'
}],
latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield assert_read_from_all([secondary, other_secondary],
PRIMARY_PREFERRED,
latency=9999999)
yield assert_read_from(secondary, PRIMARY_PREFERRED,
{'name': 'secondary'})
# No primary or matching secondary
yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# Only primary matches
yield assert_read_from(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Mode and tags both match
yield assert_read_from(secondary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([secondary, other_secondary],
NEAREST,
latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield self.pause(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
# SECONDARY
yield assert_read_from(other_secondary, SECONDARY)
yield assert_read_from(other_secondary, SECONDARY,
self.other_secondary_dc)
# Only the down secondary matches
yield assert_read_from(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
yield assert_read_from_all([primary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from(other_secondary, NEAREST,
{'name': 'other_secondary'})
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield assert_read_from(None, SECONDARY)
yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
yield assert_read_from(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield assert_read_from(primary, SECONDARY_PREFERRED)
yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
yield assert_read_from(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_io_loop(self):
with assert_raises(TypeError):
motor.MotorReplicaSetClient(test.env.rs_uri, io_loop='foo')
def test_connect(self):
with assert_raises(pymongo.errors.ConnectionFailure):
yield motor.MotorReplicaSetClient(
'%s:%s' % (host, port),
replicaSet='anything',
connectTimeoutMS=600).test.test.find_one()
def test_io_loop(self):
with assert_raises(TypeError):
motor.MotorReplicaSetClient('%s:%s' % (host, port),
replicaSet=test.rs_name,
io_loop='foo')
def test_open_callback(self):
cx = motor.MotorReplicaSetClient('%s:%s' % (host, port),
replicaSet=self.name,
io_loop=self.io_loop)
yield self.check_optional_callback(cx.open)
cx.close()