def test_init_disconnected_with_auth(self):
c = self._get_client()
c.admin.system.users.remove({})
c.pymongo_test.system.users.remove({})
try:
c.admin.add_user("admin", "pass")
c.admin.authenticate("admin", "pass")
c.pymongo_test.add_user("user", "pass")
# Auth with lazy connection.
host = one(self.hosts)
uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % (
host[0], host[1], self.name)
authenticated_client = MongoReplicaSetClient(uri, _connect=False)
authenticated_client.pymongo_test.test.find_one()
# Wrong password.
bad_uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % (
host[0], host[1], self.name)
bad_client = MongoReplicaSetClient(bad_uri, _connect=False)
self.assertRaises(
OperationFailure, bad_client.pymongo_test.test.find_one)
finally:
# Clean up.
c.admin.system.users.remove({})
c.pymongo_test.system.users.remove({})
def test_ipv6(self):
try:
client = MongoReplicaSetClient("[::1]:%d" % (port, ),
replicaSet=self.name)
except:
# Either mongod was started without --ipv6
# or the OS doesn't support it (or both).
raise SkipTest("No IPv6")
# Try a few simple things
client = MongoReplicaSetClient("mongodb://[::1]:%d" % (port, ),
replicaSet=self.name)
client = MongoReplicaSetClient("mongodb://[::1]:%d/?w=0;"
"replicaSet=%s" % (port, self.name))
client = MongoReplicaSetClient("[::1]:%d,localhost:"
"%d" % (port, port),
replicaSet=self.name)
client = MongoReplicaSetClient("localhost:%d,[::1]:"
"%d" % (port, port),
replicaSet=self.name)
client.pymongo_test.test.save({"dummy": u"object"})
client.pymongo_test_bernie.test.save({"dummy": u"object"})
dbs = client.database_names()
self.assertTrue("pymongo_test" in dbs)
self.assertTrue("pymongo_test_bernie" in dbs)
client.close()
def test_use_greenlets(self):
self.assertFalse(
MongoReplicaSetClient(pair, replicaSet=self.name).use_greenlets)
if have_gevent:
self.assertTrue(MongoReplicaSetClient(
pair, replicaSet=self.name, use_greenlets=True).use_greenlets)
def test_init_disconnected_with_auth(self):
c = self._get_client()
if not server_started_with_auth(c):
raise SkipTest('Authentication is not enabled on server')
c.admin.add_user("admin", "pass")
c.admin.authenticate("admin", "pass")
try:
c.pymongo_test.add_user("user",
"pass",
roles=['readWrite', 'userAdmin'])
# Auth with lazy connection.
host = one(self.hosts)
uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % (
host[0], host[1], self.name)
authenticated_client = MongoReplicaSetClient(uri, _connect=False)
authenticated_client.pymongo_test.test.find_one()
# Wrong password.
bad_uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % (
host[0], host[1], self.name)
bad_client = MongoReplicaSetClient(bad_uri, _connect=False)
self.assertRaises(OperationFailure,
bad_client.pymongo_test.test.find_one)
finally:
# Clean up.
remove_all_users(c.pymongo_test)
remove_all_users(c.admin)
def test_mongo_replica_set_client(self):
c = Connection(pair)
ismaster = c.admin.command('ismaster')
if 'setName' in ismaster:
setname = str(ismaster.get('setName'))
else:
raise SkipTest("Not connected to a replica set.")
m = MongoReplicaSetClient(pair, replicaSet=setname, w=0)
coll = m.pymongo_test.write_concern_test
coll.drop()
doc = {"_id": ObjectId()}
coll.insert(doc)
self.assertTrue(coll.insert(doc, safe=False))
self.assertTrue(coll.insert(doc, w=0))
self.assertTrue(coll.insert(doc))
self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
self.assertRaises(OperationFailure, coll.insert, doc, w=1)
m = MongoReplicaSetClient(pair, replicaSet=setname)
coll = m.pymongo_test.write_concern_test
self.assertTrue(coll.insert(doc, safe=False))
self.assertTrue(coll.insert(doc, w=0))
self.assertRaises(OperationFailure, coll.insert, doc)
self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
self.assertRaises(OperationFailure, coll.insert, doc, w=1)
m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname))
self.assertTrue(m.safe)
coll = m.pymongo_test.write_concern_test
self.assertRaises(OperationFailure, coll.insert, doc)
m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname))
self.assertFalse(m.safe)
coll = m.pymongo_test.write_concern_test
self.assertTrue(coll.insert(doc))
def open_spider(self, spider):
self.load_spider(spider)
self.configure()
if self.config['replica_set'] is not None:
connection = MongoReplicaSetClient(
self.config['uri'],
replicaSet=self.config['replica_set'],
w=self.config['write_concern'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY_PREFERRED)
shop_connection = MongoReplicaSetClient(
self.config['shop_uri'],
replicaSet=self.config['replica_set'],
w=self.config['write_concern'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY_PREFERRED)
else:
# Connecting to a stand alone MongoDB
connection = MongoClient(
self.config['uri'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY)
shop_connection = MongoClient(
self.config['shop_uri'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY)
# Set up db
self.db = connection[self.config['database']]
self.db_category = connection['item_categories']
self.shop_db = shop_connection[self.config['shop_database']]
self.shop_db_category = shop_connection['item_categories']
# self.image_handle_queue_redis = redis.Redis(host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port'])
# self.image_hamdle_queue = RedisqClient("crawler", host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port'])
if 'image_handle_queue_redis_password' in self.config.keys():
self.image_handle_queue = BeeQueueClient("bq:crawler_handle_image:", host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port'], password=self.config['image_handle_queue_redis_password'])
else:
self.image_handle_queue = BeeQueueClient("bq:crawler_handle_image:", host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port'])
self.need_notify_redis = (os.environ.get('item_update_notify_redis', 'False') == 'True')
if self.need_notify_redis:
if os.environ.get('single_item_mode', 'None') == 'realtime':
if 'realtime_image_handle_queue_redis_password' in self.config.keys():
self.realtime_image_handle_queue = BeeQueueClient("bq:realtime_handle_image:", host=self.config['realtime_image_handle_queue_redis_host'], port=self.config['realtime_image_handle_queue_redis_port'], password=self.config['realtime_image_handle_queue_redis_password'])
else:
self.realtime_image_handle_queue = BeeQueueClient("bq:realtime_handle_image:", host=self.config['realtime_image_handle_queue_redis_host'], port=self.config['realtime_image_handle_queue_redis_port'])
if 'item_notify_redis_password' in self.config.keys():
self.item_redis_connection = redis.Redis(host=self.config['item_notify_redis_host'], port=self.config['item_notify_redis_port'], password=self.config['item_notify_redis_password'])
else:
self.item_redis_connection = redis.Redis(host=self.config['item_notify_redis_host'], port=self.config['item_notify_redis_port'])
def test_read_mode_secondary(self):
client = MongoReplicaSetClient(
pair, replicaSet=self.name, _connect=False,
read_preference=ReadPreference.SECONDARY)
# No error.
client.pymongo_test.test_collection.find_one()
def test_repr(self):
with ignore_deprecations():
client = MongoReplicaSetClient(host, port,
replicaSet=self.name)
self.assertIn("MongoReplicaSetClient(host=[", repr(client))
self.assertIn(pair, repr(client))
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def delete(self, message_id):
with statsd.timer('%s.%s' % (__name__, self.__class__.__name__)):
try:
client = MongoReplicaSetClient(self.host,
replicaSet=self.replicaSet,
w=self.writeConcern,
j=self.journal,
slave_okay=True,
connectTimeoutMS=200)
except ConnectionFailure as e:
self.log.error("Connection falure error reached: %r" %
e, exc_info=True)
raise Exception(e)
db = client[self.mongodb['database']]
collection = db[self.account]
try:
results = collection.remove({'id': {'$in': message_id}})
ret = json.loads(dumps(results))
client.close()
return ret
except PyMongoError as e:
self.log.error(e, exc_info=True)
raise Exception(e)
def test_read_with_failover(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
def iter_cursor(cursor):
for _ in cursor:
pass
return True
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
db.test.insert([{'foo': i} for i in xrange(10)], w=w)
self.assertEqual(10, db.test.count())
db.read_preference = SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
cursor.next()
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
def test_writes_with_failover(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
def try_write():
for _ in xrange(30):
try:
db.test.insert({'bar': 'baz'})
return True
except AutoReconnect:
sleep(1)
return False
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
self.assertTrue(try_write())
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def __init__(self, *args, **kwargs):
super(MdbClient, self).__init__('mongodb')
auto = kwargs.get('auto')
if auto:
del kwargs['auto']
kwauto = mongoConfToPy()
kwauto.update(kwargs)
#@note: update and overwrite from origina kwargs
kwargs = kwauto
dbName = kwargs.get('dbName')
if dbName:
del kwargs['dbName']
hosts_or_uri = kwargs.get('hosts_or_uri', False)
replicaSet = kwargs.get('replicaSet', False)
if replicaSet is None:
del kwargs['replicaSet']
#@note: delete if None so we can call it with None
self.client = None
if hosts_or_uri:
self.client = MongoReplicaSetClient(*args, **kwargs)
self.rs_last_primary = self.client.primary
self.rs_last_secondaries = self.client.secondaries
else:
#kwargs['host']=kwargs.get('host') or kwargs.get('hosts_or_uri')
#if kwargs.get('hosts_or_uri'):del kwargs['hosts_or_uri']
self.client = MongoClient(*args, **kwargs)
self.db = self.client[dbName] if dbName else None
self.setUpCollections()
def test_init_disconnected(self):
c = self._get_client(_connect=False)
# No errors
c.seeds
c.hosts
c.arbiters
c.is_mongos
c.max_pool_size
c.use_greenlets
c.get_document_class
c.tz_aware
c.max_bson_size
c.auto_start_request
self.assertFalse(c.primary)
self.assertFalse(c.secondaries)
c.pymongo_test.test.find_one() # Auto-connect for read.
self.assertTrue(c.primary)
self.assertTrue(c.secondaries)
c = self._get_client(_connect=False)
c.pymongo_test.test.update({}, {}) # Auto-connect for write.
self.assertTrue(c.primary)
c = MongoReplicaSetClient("somedomainthatdoesntexist.org",
replicaSet="rs",
connectTimeoutMS=1,
_connect=False)
self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = [
MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets,
read_preference=SECONDARY) for _ in xrange(2)
]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
self.assertEqual(one(c.secondaries), _partition_node(secondary))
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
self.assertRaises(AutoReconnect, self.c_count.test.test.count)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
sleep(1)
for c in self.c_find_one, self.c_count:
self.assertFalse(c.secondaries)
self.assertEqual(_partition_node(primary), c.primary)
def open_spider(self, spider):
self._build_unique_key()
if self._replica_set is not None:
self.connection = MongoReplicaSetClient(
self._uri,
replicaSet=self._replica_set,
w=self._write_concern,
fsync=self._fsync,
read_preference=ReadPreference.PRIMARY_PREFERRED)
else:
self.connection = MongoClient(
self._uri,
fsync=self._fsync,
read_preference=ReadPreference.PRIMARY)
self.database = self.connection[self._database]
self.collection = self.database[self._collection]
log.msg('Connected to MongoDB "%s", using "%s/%s"' %
self._uri, self._database, self._collection)
# ensure index
if self._unique_key:
log.msg('Creating index for key %s' % self._unique_key)
self.collection.ensure_index(self._unique_key.items(), unique=True, sparse=True)
def connect(self, database, timezone=None, cache_size=0, auto_ensure=True, replica_set=None, *args, **kwds):
''' `connect` is a thin wrapper around __init__ which creates the
database connection that the session will use.
:param database: the database name to use. Should be an instance of \
:class:`basestring`
:param safe: The value for the "safe" parameter of the Session \
init function
:param auto_ensure: Whether to implicitly call ensure_indexes on all write \
operations.
:param replica_set: The replica-set to use (as a string). If specified, \
:class:`pymongo.mongo_replica_set_client.MongoReplicaSetClient` is used \
instead of :class:`pymongo.mongo_client.MongoClient`
:param args: arguments for :class:`pymongo.mongo_client.MongoClient`
:param kwds: keyword arguments for :class:`pymongo.mongo_client.MongoClient`
'''
safe = kwds.get('safe', False)
if 'safe' in kwds:
del kwds['safe']
if timezone is not None:
kwds['tz_aware'] = True
if replica_set is not None:
if 'MongoReplicaSetClient' in globals():
conn = MongoReplicaSetClient(*args, replicaSet=replica_set, **kwds)
else: # pragma: no cover
conn = MongoClient(*args, replicaSet=replica_set, **kwds)
else:
conn = MongoClient(*args, **kwds)
db = conn[database]
return Session(db, timezone=timezone, safe=safe, cache_size=cache_size, auto_ensure=auto_ensure)
def test_writes_with_failover(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
# Wait past pool's check interval, so it throws an error from
# get_socket().
sleep(1)
# Verify that we only raise AutoReconnect, not some other error,
# while we wait for new primary.
for _ in xrange(10000):
try:
db.test.insert({'bar': 'baz'})
# No error, found primary.
break
except AutoReconnect:
sleep(.01)
else:
self.fail("Couldn't connect to new primary")
# Found new primary.
self.assertTrue(c.primary)
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_primary_stepdown(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
ha_tools.stepdown_primary()
# Wait for new primary
patience_seconds = 30
for _ in xrange(patience_seconds):
sleep(1)
rs_state = c._MongoReplicaSetClient__rs_state
if rs_state.writer and rs_state.writer != primary:
if ha_tools.get_primary():
# New primary stepped up
new_primary = _partition_node(ha_tools.get_primary())
self.assertEqual(new_primary, rs_state.writer)
new_secondaries = partition_nodes(
ha_tools.get_secondaries())
self.assertEqual(set(new_secondaries),
rs_state.secondaries)
break
else:
self.fail(
"No new primary after %s seconds. Old primary was %s, current"
" is %s" % (patience_seconds, primary, ha_tools.get_primary()))
def setUp(self):
members = [{}, {}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
def test_stepdown_triggers_refresh(self):
c_find_one = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(one(c_find_one.secondaries),
_partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
sleep(1)
# Trigger a refresh
self.assertRaises(AutoReconnect, c_find_one.test.test.find_one)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
sleep(1)
# We've detected the stepdown
self.assertTrue(not c_find_one.primary
or _partition_node(primary) != c_find_one.primary)
def count(self, filters):
with statsd.timer('%s.%s' % (__name__, self.__class__.__name__)):
try:
client = MongoReplicaSetClient(self.host,
replicaSet=self.replicaSet,
w=self.writeConcern,
j=self.journal,
slave_okay=True,
connectTimeoutMS=200)
except ConnectionFailure as e:
self.log.error("Connection falure error reached: %r" %
e, exc_info=True)
raise Exception(e)
db = client[self.mongodb['database']]
collection = db[self.account]
try:
if filters:
results = collection.find(filters).count()
else:
results = db.command('collstats', self.account)['count']
client.close()
return results
except OperationFailure as e:
self.log.error(e, exc_info=True)
return 0
def __init__(self):
""" Constructor """
# Configure the connection
self.configure()
if self.config['replica_set'] is not None:
connection = MongoReplicaSetClient(
self.config['uri'],
replicaSet=self.config['replica_set'],
w=self.config['write_concern'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY_PREFERRED)
else:
# Connecting to a stand alone MongoDB
connection = MongoClient(
self.config['uri'],
fsync=self.config['fsync'],
read_preference=ReadPreference.PRIMARY)
# Set up the collection
database = connection[self.config['database']]
self.collection = database[self.config['collection']]
log.msg('Connected to MongoDB {0}, using "{1}/{2}"'.format(
self.config['uri'],
self.config['database'],
self.config['collection']))
# Ensure unique index
if self.config['unique_key']:
self.collection.ensure_index(self.config['unique_key'], unique=True)
log.msg('Ensuring index for key {0}'.format(
self.config['unique_key']))
def test_secondary_failure(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def readers_changed():
for _ in xrange(20):
if c.secondaries != secondaries:
return True
sleep(1)
return False
killed = ha_tools.kill_secondary()
sleep(2 * MONITOR_INTERVAL)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = MongoClient(primary, use_greenlets=use_greenlets)
secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets)
rsc = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def get(self, filename):
if not filename:
raise Exception("No filename given: %r" % filename)
try:
client = MongoReplicaSetClient(self.host,
replicaSet=self.replicaSet,
w=self.writeConcern,
j=self.journal,
slave_okay=True,
read_preference=self.r_p,
connectTimeoutMS=200)
except ConnectionFailure as e:
self.log.error("Connection falure error reached: %r" % e,
exc_info=True)
raise Exception(e)
db = client[self.current_db]
self.fs = GridFS(db)
try:
result = self.fs.get_last_version(filename=filename)
info = self.info(filename=filename)
info['content'] = result.read()
return info
except PyMongoError as e:
self.log.error(e, exc_info=True)
raise Exception(e)
def test_get_default_database(self):
host = one(self.hosts)
uri = "mongodb://%s:%d/foo?replicaSet=%s" % (host[0], host[1],
self.name)
c = MongoReplicaSetClient(uri, _connect=False)
self.assertEqual(Database(c, 'foo'), c.get_default_database())
def test_get_default_database_error(self):
host = one(self.hosts)
# URI with no database.
uri = "mongodb://%s:%d/?replicaSet=%s" % (host[0], host[1], self.name)
c = MongoReplicaSetClient(uri, _connect=False)
self.assertRaises(ConfigurationError, c.get_default_database)
def test_get_default_database_with_authsource(self):
# Ensure we distinguish database name from authSource.
host = one(self.hosts)
uri = "mongodb://%s:%d/foo?replicaSet=%s&authSource=src" % (
host[0], host[1], self.name)
c = MongoReplicaSetClient(uri, _connect=False)
self.assertEqual(Database(c, 'foo'), c.get_default_database())
def test_init_disconnected_with_auth_failure(self):
c = MongoReplicaSetClient(
"mongodb://*****:*****@somedomainthatdoesntexist",
replicaSet="rs",
connectTimeoutMS=1,
_connect=False)
self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
