def setUp(self):
members = [
{},
{
'priority': 0
},
{
'priority': 0
},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = ReplicaSetConnection(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
try:
self.c.admin.add_user('admin', 'adminpass')
except:
# SERVER-4225
pass
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def test_passive_and_hidden(self):
self.c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
db = self.c.pymongo_test
w = len(self.c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
db.test.insert({'foo': 'bar'}, safe=True, w=w)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
hidden = ha_tools.get_hidden_members()
hidden = [_partition_node(member) for member in hidden]
self.assertEqual(self.c.secondaries, set(passives))
for mode in (ReadPreference.SECONDARY,
ReadPreference.SECONDARY_PREFERRED):
db.read_preference = mode
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id in passives)
self.assertTrue(cursor._Cursor__connection_id not in hidden)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
db.read_preference = ReadPreference.SECONDARY_PREFERRED
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, self.c.primary)
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{},
{'priority': 0},
{'arbiterOnly': True},
{'priority': 0, 'hidden': True},
{'priority': 0, 'slaveDelay': 5}
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestMonitorRemovesRecoveringMember(unittest.TestCase):
# Members in STARTUP2 or RECOVERING states are shown in the primary's
# isMaster response, but aren't secondaries and shouldn't be read from.
# Verify that if a secondary goes into RECOVERING mode, the Monitor removes
# it from the set of readers.
def setUp(self):
members = [{}, {'priority': 0}, {'priority': 0}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_monitor_removes_recovering_member(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def getMongoDB():
conn = ReplicaSetConnection(
"192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017",
replicaSet='ats_rs')
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
db = conn['bugreporter']
collection = db['original_data']
return collection
class TestReplicaSetRequest(unittest.TestCase):
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = ReplicaSetConnection(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
primary_pool = self.c._MongoReplicaSetClient__members[primary].pool
secondary_pool = self.c._MongoReplicaSetClient__members[secondary].pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
patience_seconds = 60
for _ in range(patience_seconds):
sleep(1)
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
else:
self.fail("Problem with test: No new primary after %s seconds"
% patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{}, {
'priority': 0
}, {
'arbiterOnly': True
}, {
'priority': 0,
'hidden': True
}, {
'priority': 0,
'slaveDelay': 5
}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
db = self.c.pymongo_test
w = len(self.c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
db.test.insert({'foo': 'bar'}, safe=True, w=w)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
hidden = ha_tools.get_hidden_members()
hidden = [_partition_node(member) for member in hidden]
self.assertEqual(self.c.secondaries, set(passives))
for mode in (ReadPreference.SECONDARY,
ReadPreference.SECONDARY_PREFERRED):
db.read_preference = mode
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id in passives)
self.assertTrue(cursor._Cursor__connection_id not in hidden)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
db.read_preference = ReadPreference.SECONDARY_PREFERRED
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, self.c.primary)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestReplicaSetAuth(unittest.TestCase):
def setUp(self):
members = [
{},
{
'priority': 0
},
{
'priority': 0
},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = ReplicaSetConnection(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
try:
self.c.admin.add_user('admin', 'adminpass')
except:
# SERVER-4225
pass
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def test_auth_during_failover(self):
self.assertTrue(self.db.authenticate('user', 'userpass'))
self.assertTrue(
self.db.foo.insert({'foo': 'bar'}, safe=True, w=3, wtimeout=1000))
self.db.logout()
self.assertRaises(OperationFailure, self.db.foo.find_one)
primary = '%s:%d' % self.c.primary
ha_tools.kill_members([primary], 2)
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# Make sure we can still authenticate
self.assertTrue(self.db.authenticate('user', 'userpass'))
# And still query.
self.db.read_preference = ReadPreference.PRIMARY_PREFERRED
self.assertEqual('bar', self.db.foo.find_one()['foo'])
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{},
{'priority': 0},
{'arbiterOnly': True},
{'priority': 0, 'hidden': True},
{'priority': 0, 'slaveDelay': 5}
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = self.c.pymongo_test
w = len(self.c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
db.test.insert({'foo': 'bar'}, safe=True, w=w)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
hidden = ha_tools.get_hidden_members()
hidden = [_partition_node(member) for member in hidden]
self.assertEqual(self.c.secondaries, set(passives))
for mode in (
ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED
):
db.read_preference = mode
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id in passives)
self.assertTrue(cursor._Cursor__connection_id not in hidden)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
db.read_preference = ReadPreference.SECONDARY_PREFERRED
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, self.c.primary)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def test_secondary_failure(self):
c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def readers_changed():
for _ in xrange(20):
if c.secondaries != secondaries:
return True
sleep(1)
return False
killed = replset_tools.kill_secondary()
sleep(5) # Let monitor thread notice change
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
secondaries = c.secondaries
replset_tools.restart_members(killed)
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
def test_secondary_connection(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Force replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, safe=True, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.NEAREST).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY).is_primary)
for kwargs in [
{'read_preference': ReadPreference.PRIMARY_PREFERRED},
{'read_preference': ReadPreference.SECONDARY},
{'read_preference': ReadPreference.SECONDARY_PREFERRED},
{'read_preference': ReadPreference.NEAREST},
{'slave_okay': True},
]:
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
self.assert_(conn.pymongo_test.test.find_one())
# Test direct connection to an arbiter
secondary_host = ha_tools.get_arbiters()[0]
host, port = ha_tools.get_arbiters()[0].split(':')
port = int(port)
conn = Connection(host, port)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
def test_passive_and_hidden(self):
c = ReplicaSetConnection(self.seed, replicaSet=self.name)
db = c.pymongo_test
db.test.remove({}, safe=True, w=len(c.secondaries))
w = len(c.secondaries) + 1
db.test.insert({'foo': 'bar'}, safe=True, w=w)
db.read_preference = ReadPreference.SECONDARY
passives = replset_tools.get_passives()
passives = [_partition_node(member) for member in passives]
hidden = replset_tools.get_hidden_members()
hidden = [_partition_node(member) for member in hidden]
self.assertEqual(c.secondaries, set(passives))
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id not in hidden)
replset_tools.kill_members(replset_tools.get_passives(), 2)
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, c.primary)
def test_secondary_failure(self):
c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def readers_changed():
for _ in xrange(20):
if c.secondaries != secondaries:
return True
sleep(1)
return False
killed = ha_tools.kill_secondary()
sleep(2 * MONITOR_INTERVAL)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
def test_passive_and_hidden(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = self.c.pymongo_test
w = len(self.c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
db.test.insert({'foo': 'bar'}, safe=True, w=w)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
hidden = ha_tools.get_hidden_members()
hidden = [_partition_node(member) for member in hidden]
self.assertEqual(self.c.secondaries, set(passives))
for mode in (
ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED
):
db.read_preference = mode
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id in passives)
self.assertTrue(cursor._Cursor__connection_id not in hidden)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
db.read_preference = ReadPreference.SECONDARY_PREFERRED
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, self.c.primary)
def test_writes_with_failover(self):
c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
db.test.insert({'foo': 'bar'}, safe=True, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
def try_write():
for _ in xrange(30):
try:
db.test.insert({'bar': 'baz'}, safe=True)
return True
except AutoReconnect:
sleep(1)
return False
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
self.assertTrue(try_write())
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_read_with_failover(self):
c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets,
auto_start_request=False)
self.assertTrue(bool(len(c.secondaries)))
def iter_cursor(cursor):
for _ in cursor:
pass
return True
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
# Force replication
db.test.insert([{'foo': i} for i in xrange(10)], safe=True, w=w)
self.assertEqual(10, db.test.count())
db.read_preference = ReadPreference.SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
cursor.next()
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
class TestReplicaSetAuth(unittest.TestCase):
def setUp(self):
members = [
{},
{'priority': 0},
{'priority': 0},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = ReplicaSetConnection(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
try:
self.c.admin.add_user('admin', 'adminpass')
except:
# SERVER-4225
pass
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def test_auth_during_failover(self):
self.assertTrue(self.db.authenticate('user', 'userpass'))
self.assertTrue(self.db.foo.insert({'foo': 'bar'},
safe=True, w=3, wtimeout=1000))
self.db.logout()
self.assertRaises(OperationFailure, self.db.foo.find_one)
primary = '%s:%d' % self.c.primary
ha_tools.kill_members([primary], 2)
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# Make sure we can still authenticate
self.assertTrue(self.db.authenticate('user', 'userpass'))
# And still query.
self.db.read_preference = ReadPreference.PRIMARY_PREFERRED
self.assertEqual('bar', self.db.foo.find_one()['foo'])
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = Connection(primary, use_greenlets=use_greenlets)
secondary_cx = Connection(secondary, use_greenlets=use_greenlets)
rsc = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = Connection(primary, use_greenlets=use_greenlets)
secondary_cx = Connection(secondary, use_greenlets=use_greenlets)
rsc = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def setUp(self):
members = [{}, {"priority": 0}, {"priority": 0}]
res = ha_tools.start_replica_set(members, auth=True)
self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets)
# Add an admin user to enable auth
self.c.admin.add_user("admin", "adminpass")
self.c.admin.authenticate("admin", "adminpass")
self.db = self.c.pymongo_ha_auth
self.db.add_user("user", "userpass")
self.c.admin.logout()
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def test_passive_and_hidden(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def getConn(self):
''' 获取 primary 的连接信息 '''
if self._repl:
client = ReplicaSetConnection(
self._addr,
replicaSet=self._repl,
read_preference=pymongo.read_preferences.ReadPreference.PRIMARY
)
else:
client = MongoClient(self._addr)
if self._user and self._pwd:
client.admin.authenticate(self._user, self._pwd)
return client
def test_primary_stepdown(self):
c = ReplicaSetConnection(self.seed, replicaSet=self.name)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def primary_changed():
for _ in xrange(30):
if c.primary != primary:
return True
time.sleep(1)
return False
replset_tools.stepdown_primary()
self.assertTrue(primary_changed())
self.assertTrue(secondaries != c.secondaries)
def init(self):
logger.info(
"[Trending broker] I init the %s server connection to %s:%s (%s)" %
(self.get_name(), self.backend, self.uri, self.replica_set))
if self.replica_set:
self.con = ReplicaSetConnection(self.uri,
replicaSet=self.replica_set,
safe=True)
else:
# Old versions of pymongo do not known about fsync
if ReplicaSetConnection:
self.con = Connection(self.uri, safe=True)
else:
self.con = Connection(self.uri, safe=True)
# Open a connection
self.db = getattr(self.con, self.database)
self.col = self.db['trending']
def __init__(self, conn_str, db_name):
try:
if conn_str.find("replicaSet") == -1:
_conn = Connection(
conn_str,
max_pool_size=30,
safe=True,
read_preference=ReadPreference.SECONDARY_ONLY)
else:
_conn = ReplicaSetConnection(
conn_str,
max_pool_size=30,
safe=True,
read_preference=ReadPreference.SECONDARY_ONLY)
self._db = _conn[db_name]
except Exception, e:
logger.exception('Can not connect to mongodb: %s' % e)
raise e
def test_monitor_removes_recovering_member(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def test_primary_failure(self):
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
# Wait for new primary to be elected
def primary_changed():
for _ in xrange(30):
if c.primary and c.primary != primary:
return True
sleep(1)
return False
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
self.assertTrue(primary_changed())
self.assertNotEqual(secondaries, c.secondaries)
def test_simple_ops(self):
if not have_ssl:
raise SkipTest()
try:
conn = Connection(connectTimeoutMS=100, ssl=True)
# MongoDB not configured for SSL?
except ConnectionFailure:
raise SkipTest()
response = conn.admin.command('ismaster')
if 'setName' in response:
conn = ReplicaSetConnection(replicaSet=response['setName'],
w=len(response['hosts']),
ssl=True)
db = conn.pymongo_ssl_test
self.assert_(db.test.insert({'ssl': True}, safe=True))
self.assertTrue(db.test.find_one()['ssl'])
conn.drop_database(db)
def setUp(self):
members = [
# primary
{"tags": {"dc": "ny", "name": "primary"}},
# secondary
{"tags": {"dc": "la", "name": "secondary"}, "priority": 0},
# other_secondary
{"tags": {"dc": "ny", "name": "other_secondary"}, "priority": 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual("primary", self.primary_tags["name"])
self.primary_dc = {"dc": self.primary_tags["dc"]}
secondaries = ha_tools.get_secondaries()
(secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "secondary"]
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {"dc": self.secondary_tags["dc"]}
(other_secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "other_secondary"]
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {"dc": self.other_secondary_tags["dc"]}
self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert([{"foo": i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def test_primary_failure(self):
c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def primary_changed():
for _ in xrange(30):
if c.primary != primary:
return True
sleep(1)
return False
killed = replset_tools.kill_primary()
sleep(5) # Let monitor thread notice change
self.assertTrue(bool(len(killed)))
self.assertTrue(primary_changed())
self.assertTrue(secondaries != c.secondaries)
def init(self):
"""
Called by Scheduler to say 'let's prepare yourself guy'
"""
logger.debug("Initialization of the mongodb module")
if self.replica_set:
self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, fsync=True)
else:
# Old versions of pymongo do not known about fsync
if ReplicaSetConnection:
self.con = Connection(self.uri, fsync=True)
else:
self.con = Connection(self.uri)
#self.con = Connection(self.uri)
# Open a gridfs connection
self.db = getattr(self.con, self.database)
self.hosts_fs = GridFS(self.db, collection='retention_hosts')
self.services_fs = GridFS(self.db, collection='retention_services')
def test_primary_stepdown(self):
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries.copy()
def primary_changed():
for _ in xrange(30):
if c.primary != primary:
return True
sleep(1)
return False
ha_tools.stepdown_primary()
self.assertTrue(primary_changed())
# There can be a delay between finding the primary and updating
# secondaries
sleep(5)
self.assertNotEqual(secondaries, c.secondaries)
def init(self):
"""
Called by broker to say 'let's prepare yourself guy'
"""
logger.debug("Initialization of the mongodb snapshot module")
if self.replica_set:
self.con = ReplicaSetConnection(self.uri,
replicaSet=self.replica_set,
fsync=False)
else:
# Old versions of pymongo do not known about fsync
if ReplicaSetConnection:
self.con = Connection(self.uri, fsync=False)
else:
self.con = Connection(self.uri)
self.db = getattr(self.con, self.database)
self.snapshots = getattr(self.db, 'snapshots')
self.snapshots.ensure_index([('host_name', pymongo.DESCENDING),
('snapshot_time', pymongo.DESCENDING)])
def getConnection(self, dbName, isSECONDARYONLY=True):
'''
@summary: 获取连接
@param dbName: 要连接的数据库
'''
tryMaxNum = 100
tryNum = 0
while tryNum < tryMaxNum:
tryNum = tryNum + 1
try:
conn = ReplicaSetConnection(self.mongoDbHost,
replicaSet=self.replicaSetName)
self.DB = conn[dbName]
if isSECONDARYONLY:
self.DB.read_preference = ReadPreference.PRIMARY
else:
self.DB.read_preference = ReadPreference.PRIMARY
self.DB.authenticate(self.mongoDbUser, self.mongoDbPassword)
break
except Exception, e:
time.sleep(5)
def setUp(self):
members = [
{},
{'priority': 0},
{'priority': 0},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = ReplicaSetConnection(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
try:
self.c.admin.add_user('admin', 'adminpass')
except:
# SERVER-4225
pass
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def getMongoDB():
conn=ReplicaSetConnection("192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs')
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
db = conn['bugreporter']
collection = db['original_data']
return collection
def test_secondary_connection(self):
self.c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Force replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, safe=True, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
self.assertTrue(
Connection(primary_host, primary_port,
use_greenlets=use_greenlets).is_primary)
self.assertTrue(
Connection(
primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary)
self.assertTrue(
Connection(
primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary)
self.assertTrue(
Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.NEAREST).is_primary)
self.assertTrue(
Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY).is_primary)
for kwargs in [
{
'read_preference': ReadPreference.PRIMARY_PREFERRED
},
{
'read_preference': ReadPreference.SECONDARY
},
{
'read_preference': ReadPreference.SECONDARY_PREFERRED
},
{
'read_preference': ReadPreference.NEAREST
},
{
'slave_okay': True
},
]:
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
self.assert_(conn.pymongo_test.test.find_one())
# Test direct connection to an arbiter
host, port = ha_tools.get_arbiters()[0].split(':')
port = int(port)
conn = Connection(host, port)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
class TestReadPreference(unittest.TestCase):
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def set_ping_time(self, host, ping_time_seconds):
Member._host_to_ping_time[host] = ping_time_seconds
def clear_ping_times(self):
Member._host_to_ping_time.clear()
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy modes and hosts into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll(
[primary, secondary, other_secondary],
NEAREST, secondary_acceptable_latency_ms=1000*1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(Connection(
unpartition_node(primary), use_greenlets=use_greenlets,
slave_okay=True
).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(Connection(
unpartition_node(primary), use_greenlets=use_greenlets,
slave_okay=True
).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
self.clear_ping_times()
def getDB(self):
#return Connection('192.168.7.201',27017)['bugreporter']
#return Connection('localhost',27017)['bugreporter']
conn=ReplicaSetConnection("192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs')
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
return conn['bugreporter']
class TestReadPreference(unittest.TestCase):
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def set_ping_time(self, host, ping_time_seconds):
Member._host_to_ping_time[host] = ping_time_seconds
def clear_ping_times(self):
Member._host_to_ping_time.clear()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy modes and hosts into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll(
[primary, secondary, other_secondary],
NEAREST, secondary_acceptable_latency_ms=1000*1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(Connection(
unpartition_node(primary), use_greenlets=use_greenlets,
slave_okay=True
).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(Connection(
unpartition_node(primary), use_greenlets=use_greenlets,
slave_okay=True
).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
self.clear_ping_times()
def getConn(self):
#return Connection('192.168.7.201',27017)
conn=ReplicaSetConnection("192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs')
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
return conn
def test_secondary_connection(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A Connection succeeds no matter the read preference
for kwargs in [
{'read_preference': ReadPreference.PRIMARY},
{'read_preference': ReadPreference.PRIMARY_PREFERRED},
{'read_preference': ReadPreference.SECONDARY},
{'read_preference': ReadPreference.SECONDARY_PREFERRED},
{'read_preference': ReadPreference.NEAREST},
{'slave_okay': True}
]:
conn = Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(primary_host, conn.host)
self.assertEqual(primary_port, conn.port)
self.assertTrue(conn.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue(conn.pymongo_test.test.find_one())
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != ReadPreference.PRIMARY:
self.assertTrue(conn.pymongo_test.test.find_one())
else:
self.assertRaises(
AutoReconnect, conn.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), Connection
# should do the same for unacknowledged writes.
try:
conn.pymongo_test.test.insert({}, safe=False)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
else:
self.fail(
'Unacknowledged insert into secondary connection %s should'
'have raised exception' % conn)
# Test direct connection to an arbiter
conn = Connection(arbiter_host, arbiter_port, **kwargs)
self.assertEqual(arbiter_host, conn.host)
self.assertEqual(arbiter_port, conn.port)
self.assertFalse(conn.is_primary)
# See explanation above
try:
conn.pymongo_test.test.insert({}, safe=False)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
def test_read_preference(self):
c = ReplicaSetConnection(self.seed, replicaSet=self.name)
self.assertTrue(bool(len(c.secondaries)))
db = c.pymongo_test
db.test.remove({}, safe=True, w=len(c.secondaries))
# Force replication...
w = len(c.secondaries) + 1
db.test.insert({'foo': 'bar'}, safe=True, w=w)
# Test direct connection to a secondary
host, port = replset_tools.get_secondaries()[0].split(':')
port = int(port)
conn = Connection(host, port, slave_okay=True)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
self.assert_(conn.pymongo_test.test.find_one())
conn = Connection(host, port, read_preference=ReadPreference.SECONDARY)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
self.assert_(conn.pymongo_test.test.find_one())
# Test direct connection to an arbiter
host = replset_tools.get_arbiters()[0]
self.assertRaises(ConnectionFailure, Connection, host)
# Test PRIMARY
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, c.primary)
# Test SECONDARY with a secondary
db.read_preference = ReadPreference.SECONDARY
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
# Test SECONDARY_ONLY with a secondary
db.read_preference = ReadPreference.SECONDARY_ONLY
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
# Test SECONDARY with no secondary
killed = replset_tools.kill_all_secondaries()
self.assertTrue(bool(len(killed)))
db.read_preference = ReadPreference.SECONDARY
for _ in xrange(10):
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, c.primary)
# Test SECONDARY_ONLY with no secondary
db.read_preference = ReadPreference.SECONDARY_ONLY
for _ in xrange(10):
cursor = db.test.find()
self.assertRaises(AutoReconnect, cursor.next)
replset_tools.restart_members(killed)
# Test PRIMARY with no primary (should raise an exception)
db.read_preference = ReadPreference.PRIMARY
cursor = db.test.find()
cursor.next()
self.assertEqual(cursor._Cursor__connection_id, c.primary)
killed = replset_tools.kill_primary()
self.assertTrue(bool(len(killed)))
self.assertRaises(AutoReconnect, db.test.find_one)
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets,
auto_start_request=False)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy modes and hosts into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED,
[bad_tag, {}])
assertReadFrom(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary], NEAREST,
[bad_tag, {
'dc': 'ny'
}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll([primary, secondary, other_secondary],
NEAREST,
secondary_acceptable_latency_ms=1000 * 1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
Connection(unpartition_node(primary),
use_greenlets=use_greenlets,
slave_okay=True).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
Connection(unpartition_node(primary),
use_greenlets=use_greenlets,
slave_okay=True).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
import urllib2
from bs4 import BeautifulSoup
import time
if __name__ == '__main__':
# The Mongo database to connect to
db_to_use = sys.argv[1]
# The collection to use
collection_to_use = sys.argv[2]
# The LinkedIn URL field in your Mongo database
linkedin_url_field = sys.argv[3]
# Connect to the database
# client = MongoClient()
client = ReplicaSetConnection('localhost:27017', replicaSet='rep01')
db = client[db_to_use]
collection = db[collection_to_use]
# Get a list of all the records in the database - we need to update everything
# Fetch the record and get the linkedin url
for item in collection.find():
# Fetch the record and get the linkedin url
record_id = ObjectId(item["_id"])
url_to_get = item[linkedin_url_field]
print "Current ObjectID: " + str(record_id)
print "Current URL: " + url_to_get
if url_to_get != "N/A":
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = ReplicaSetConnection(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
class TestSecondaryConnection(unittest.TestCase):
def setUp(self):
members = [{}, {}, {"arbiterOnly": True}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_secondary_connection(self):
self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Force replication...
w = len(self.c.secondaries) + 1
db.test.insert({"foo": "bar"}, safe=True, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(":")
primary_port = int(primary_port)
(secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(":")
secondary_port = int(secondary_port)
self.assertTrue(Connection(primary_host, primary_port, use_greenlets=use_greenlets).is_primary)
self.assertTrue(
Connection(
primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.PRIMARY_PREFERRED,
).is_primary
)
self.assertTrue(
Connection(
primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY_PREFERRED,
).is_primary
)
self.assertTrue(
Connection(
primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.NEAREST
).is_primary
)
self.assertTrue(
Connection(
primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY
).is_primary
)
for kwargs in [
{"read_preference": ReadPreference.PRIMARY_PREFERRED},
{"read_preference": ReadPreference.SECONDARY},
{"read_preference": ReadPreference.SECONDARY_PREFERRED},
{"read_preference": ReadPreference.NEAREST},
{"slave_okay": True},
]:
conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
self.assert_(conn.pymongo_test.test.find_one())
# Test direct connection to an arbiter
secondary_host = ha_tools.get_arbiters()[0]
host, port = ha_tools.get_arbiters()[0].split(":")
port = int(port)
conn = Connection(host, port)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
f.close()
@classmethod
def loadGlobal(cls, filename):
"""
Loads a previously created global_storage file
"""
import json
f = open(filename, 'r')
cls.global_storage = json.load(f)
f.close()
if __name__ == "__main__":
import json
from pymongo import ReplicaSetConnection, ReadPreference
connection = ReplicaSetConnection("clutch:27017", replicaSet="geo")
connection.read_preference = ReadPreference.SECONDARY
f = open("words.json",'r')
word_dict = json.loads(f.read())
f.close()
#GEOvector.loadGlobal("gv_test.json" )
gv = GEOvector('GSE2395',word_dict, mongo_conn=connection, cache=False)
print gv.getSparse()
print "1"
print sum(gv)
print sum(gv)
