def test_init_disconnected_with_auth(self): c = self._get_client() c.admin.system.users.remove({}) c.pymongo_test.system.users.remove({}) try: c.admin.add_user("admin", "pass") c.admin.authenticate("admin", "pass") c.pymongo_test.add_user("user", "pass") # Auth with lazy connection. host = one(self.hosts) uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % ( host[0], host[1], self.name) authenticated_client = MongoReplicaSetClient(uri, _connect=False) authenticated_client.pymongo_test.test.find_one() # Wrong password. bad_uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % ( host[0], host[1], self.name) bad_client = MongoReplicaSetClient(bad_uri, _connect=False) self.assertRaises( OperationFailure, bad_client.pymongo_test.test.find_one) finally: # Clean up. c.admin.system.users.remove({}) c.pymongo_test.system.users.remove({})
def test_ipv6(self): try: client = MongoReplicaSetClient("[::1]:%d" % (port, ), replicaSet=self.name) except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest("No IPv6") # Try a few simple things client = MongoReplicaSetClient("mongodb://[::1]:%d" % (port, ), replicaSet=self.name) client = MongoReplicaSetClient("mongodb://[::1]:%d/?w=0;" "replicaSet=%s" % (port, self.name)) client = MongoReplicaSetClient("[::1]:%d,localhost:" "%d" % (port, port), replicaSet=self.name) client = MongoReplicaSetClient("localhost:%d,[::1]:" "%d" % (port, port), replicaSet=self.name) client.pymongo_test.test.save({"dummy": u"object"}) client.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) client.close()
def test_use_greenlets(self): self.assertFalse( MongoReplicaSetClient(pair, replicaSet=self.name).use_greenlets) if have_gevent: self.assertTrue(MongoReplicaSetClient( pair, replicaSet=self.name, use_greenlets=True).use_greenlets)
def test_init_disconnected_with_auth(self): c = self._get_client() if not server_started_with_auth(c): raise SkipTest('Authentication is not enabled on server') c.admin.add_user("admin", "pass") c.admin.authenticate("admin", "pass") try: c.pymongo_test.add_user("user", "pass", roles=['readWrite', 'userAdmin']) # Auth with lazy connection. host = one(self.hosts) uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % ( host[0], host[1], self.name) authenticated_client = MongoReplicaSetClient(uri, _connect=False) authenticated_client.pymongo_test.test.find_one() # Wrong password. bad_uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % ( host[0], host[1], self.name) bad_client = MongoReplicaSetClient(bad_uri, _connect=False) self.assertRaises(OperationFailure, bad_client.pymongo_test.test.find_one) finally: # Clean up. remove_all_users(c.pymongo_test) remove_all_users(c.admin)
def test_mongo_replica_set_client(self): c = Connection(pair) ismaster = c.admin.command('ismaster') if 'setName' in ismaster: setname = str(ismaster.get('setName')) else: raise SkipTest("Not connected to a replica set.") m = MongoReplicaSetClient(pair, replicaSet=setname, w=0) coll = m.pymongo_test.write_concern_test coll.drop() doc = {"_id": ObjectId()} coll.insert(doc) self.assertTrue(coll.insert(doc, safe=False)) self.assertTrue(coll.insert(doc, w=0)) self.assertTrue(coll.insert(doc)) self.assertRaises(OperationFailure, coll.insert, doc, safe=True) self.assertRaises(OperationFailure, coll.insert, doc, w=1) m = MongoReplicaSetClient(pair, replicaSet=setname) coll = m.pymongo_test.write_concern_test self.assertTrue(coll.insert(doc, safe=False)) self.assertTrue(coll.insert(doc, w=0)) self.assertRaises(OperationFailure, coll.insert, doc) self.assertRaises(OperationFailure, coll.insert, doc, safe=True) self.assertRaises(OperationFailure, coll.insert, doc, w=1) m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname)) self.assertTrue(m.safe) coll = m.pymongo_test.write_concern_test self.assertRaises(OperationFailure, coll.insert, doc) m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname)) self.assertFalse(m.safe) coll = m.pymongo_test.write_concern_test self.assertTrue(coll.insert(doc))
def open_spider(self, spider): self.load_spider(spider) self.configure() if self.config['replica_set'] is not None: connection = MongoReplicaSetClient( self.config['uri'], replicaSet=self.config['replica_set'], w=self.config['write_concern'], fsync=self.config['fsync'], read_preference=ReadPreference.PRIMARY_PREFERRED) shop_connection = MongoReplicaSetClient( self.config['shop_uri'], replicaSet=self.config['replica_set'], w=self.config['write_concern'], fsync=self.config['fsync'], read_preference=ReadPreference.PRIMARY_PREFERRED) else: # Connecting to a stand alone MongoDB connection = MongoClient( self.config['uri'], fsync=self.config['fsync'], read_preference=ReadPreference.PRIMARY) shop_connection = MongoClient( self.config['shop_uri'], fsync=self.config['fsync'], read_preference=ReadPreference.PRIMARY) # Set up db self.db = connection[self.config['database']] self.db_category = connection['item_categories'] self.shop_db = shop_connection[self.config['shop_database']] self.shop_db_category = shop_connection['item_categories'] # self.image_handle_queue_redis = redis.Redis(host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port']) # self.image_hamdle_queue = RedisqClient("crawler", host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port']) if 'image_handle_queue_redis_password' in self.config.keys(): self.image_handle_queue = BeeQueueClient("bq:crawler_handle_image:", host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port'], password=self.config['image_handle_queue_redis_password']) else: self.image_handle_queue = BeeQueueClient("bq:crawler_handle_image:", host=self.config['image_handle_queue_redis_host'], port=self.config['image_handle_queue_redis_port']) self.need_notify_redis = (os.environ.get('item_update_notify_redis', 'False') == 'True') if self.need_notify_redis: if os.environ.get('single_item_mode', 'None') == 'realtime': if 'realtime_image_handle_queue_redis_password' in self.config.keys(): self.realtime_image_handle_queue = BeeQueueClient("bq:realtime_handle_image:", host=self.config['realtime_image_handle_queue_redis_host'], port=self.config['realtime_image_handle_queue_redis_port'], password=self.config['realtime_image_handle_queue_redis_password']) else: self.realtime_image_handle_queue = BeeQueueClient("bq:realtime_handle_image:", host=self.config['realtime_image_handle_queue_redis_host'], port=self.config['realtime_image_handle_queue_redis_port']) if 'item_notify_redis_password' in self.config.keys(): self.item_redis_connection = redis.Redis(host=self.config['item_notify_redis_host'], port=self.config['item_notify_redis_port'], password=self.config['item_notify_redis_password']) else: self.item_redis_connection = redis.Redis(host=self.config['item_notify_redis_host'], port=self.config['item_notify_redis_port'])
def test_read_mode_secondary(self): client = MongoReplicaSetClient( pair, replicaSet=self.name, _connect=False, read_preference=ReadPreference.SECONDARY) # No error. client.pymongo_test.test_collection.find_one()
def test_repr(self): with ignore_deprecations(): client = MongoReplicaSetClient(host, port, replicaSet=self.name) self.assertIn("MongoReplicaSetClient(host=[", repr(client)) self.assertIn(pair, repr(client))
def setUp(self): members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.c = MongoReplicaSetClient(res[0], replicaSet=res[1], use_greenlets=use_greenlets, auto_start_request=True)
def delete(self, message_id): with statsd.timer('%s.%s' % (__name__, self.__class__.__name__)): try: client = MongoReplicaSetClient(self.host, replicaSet=self.replicaSet, w=self.writeConcern, j=self.journal, slave_okay=True, connectTimeoutMS=200) except ConnectionFailure as e: self.log.error("Connection falure error reached: %r" % e, exc_info=True) raise Exception(e) db = client[self.mongodb['database']] collection = db[self.account] try: results = collection.remove({'id': {'$in': message_id}}) ret = json.loads(dumps(results)) client.close() return ret except PyMongoError as e: self.log.error(e, exc_info=True) raise Exception(e)
def test_read_with_failover(self): c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) def iter_cursor(cursor): for _ in cursor: pass return True db = c.pymongo_test w = len(c.secondaries) + 1 db.test.remove({}, w=w) # Force replication db.test.insert([{'foo': i} for i in xrange(10)], w=w) self.assertEqual(10, db.test.count()) db.read_preference = SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) cursor.next() self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor._Cursor__connection_id in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved)
def test_writes_with_failover(self): c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) primary = c.primary db = c.pymongo_test w = len(c.secondaries) + 1 db.test.remove({}, w=w) db.test.insert({'foo': 'bar'}, w=w) self.assertEqual('bar', db.test.find_one()['foo']) def try_write(): for _ in xrange(30): try: db.test.insert({'bar': 'baz'}) return True except AutoReconnect: sleep(1) return False killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) self.assertTrue(try_write()) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def __init__(self, *args, **kwargs): super(MdbClient, self).__init__('mongodb') auto = kwargs.get('auto') if auto: del kwargs['auto'] kwauto = mongoConfToPy() kwauto.update(kwargs) #@note: update and overwrite from origina kwargs kwargs = kwauto dbName = kwargs.get('dbName') if dbName: del kwargs['dbName'] hosts_or_uri = kwargs.get('hosts_or_uri', False) replicaSet = kwargs.get('replicaSet', False) if replicaSet is None: del kwargs['replicaSet'] #@note: delete if None so we can call it with None self.client = None if hosts_or_uri: self.client = MongoReplicaSetClient(*args, **kwargs) self.rs_last_primary = self.client.primary self.rs_last_secondaries = self.client.secondaries else: #kwargs['host']=kwargs.get('host') or kwargs.get('hosts_or_uri') #if kwargs.get('hosts_or_uri'):del kwargs['hosts_or_uri'] self.client = MongoClient(*args, **kwargs) self.db = self.client[dbName] if dbName else None self.setUpCollections()
def test_init_disconnected(self): c = self._get_client(_connect=False) # No errors c.seeds c.hosts c.arbiters c.is_mongos c.max_pool_size c.use_greenlets c.get_document_class c.tz_aware c.max_bson_size c.auto_start_request self.assertFalse(c.primary) self.assertFalse(c.secondaries) c.pymongo_test.test.find_one() # Auto-connect for read. self.assertTrue(c.primary) self.assertTrue(c.secondaries) c = self._get_client(_connect=False) c.pymongo_test.test.update({}, {}) # Auto-connect for write. self.assertTrue(c.primary) c = MongoReplicaSetClient("somedomainthatdoesntexist.org", replicaSet="rs", connectTimeoutMS=1, _connect=False) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = [ MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets, read_preference=SECONDARY) for _ in xrange(2) ] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: self.assertEqual(one(c.secondaries), _partition_node(secondary)) ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one) self.assertRaises(AutoReconnect, self.c_count.test.test.count) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled sleep(1) for c in self.c_find_one, self.c_count: self.assertFalse(c.secondaries) self.assertEqual(_partition_node(primary), c.primary)
def open_spider(self, spider): self._build_unique_key() if self._replica_set is not None: self.connection = MongoReplicaSetClient( self._uri, replicaSet=self._replica_set, w=self._write_concern, fsync=self._fsync, read_preference=ReadPreference.PRIMARY_PREFERRED) else: self.connection = MongoClient( self._uri, fsync=self._fsync, read_preference=ReadPreference.PRIMARY) self.database = self.connection[self._database] self.collection = self.database[self._collection] log.msg('Connected to MongoDB "%s", using "%s/%s"' % self._uri, self._database, self._collection) # ensure index if self._unique_key: log.msg('Creating index for key %s' % self._unique_key) self.collection.ensure_index(self._unique_key.items(), unique=True, sparse=True)
def connect(self, database, timezone=None, cache_size=0, auto_ensure=True, replica_set=None, *args, **kwds): ''' `connect` is a thin wrapper around __init__ which creates the database connection that the session will use. :param database: the database name to use. Should be an instance of \ :class:`basestring` :param safe: The value for the "safe" parameter of the Session \ init function :param auto_ensure: Whether to implicitly call ensure_indexes on all write \ operations. :param replica_set: The replica-set to use (as a string). If specified, \ :class:`pymongo.mongo_replica_set_client.MongoReplicaSetClient` is used \ instead of :class:`pymongo.mongo_client.MongoClient` :param args: arguments for :class:`pymongo.mongo_client.MongoClient` :param kwds: keyword arguments for :class:`pymongo.mongo_client.MongoClient` ''' safe = kwds.get('safe', False) if 'safe' in kwds: del kwds['safe'] if timezone is not None: kwds['tz_aware'] = True if replica_set is not None: if 'MongoReplicaSetClient' in globals(): conn = MongoReplicaSetClient(*args, replicaSet=replica_set, **kwds) else: # pragma: no cover conn = MongoClient(*args, replicaSet=replica_set, **kwds) else: conn = MongoClient(*args, **kwds) db = conn[database] return Session(db, timezone=timezone, safe=safe, cache_size=cache_size, auto_ensure=auto_ensure)
def test_writes_with_failover(self): c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) primary = c.primary db = c.pymongo_test w = len(c.secondaries) + 1 db.test.remove({}, w=w) db.test.insert({'foo': 'bar'}, w=w) self.assertEqual('bar', db.test.find_one()['foo']) killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) # Wait past pool's check interval, so it throws an error from # get_socket(). sleep(1) # Verify that we only raise AutoReconnect, not some other error, # while we wait for new primary. for _ in xrange(10000): try: db.test.insert({'bar': 'baz'}) # No error, found primary. break except AutoReconnect: sleep(.01) else: self.fail("Couldn't connect to new primary") # Found new primary. self.assertTrue(c.primary) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_primary_stepdown(self): c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary ha_tools.stepdown_primary() # Wait for new primary patience_seconds = 30 for _ in xrange(patience_seconds): sleep(1) rs_state = c._MongoReplicaSetClient__rs_state if rs_state.writer and rs_state.writer != primary: if ha_tools.get_primary(): # New primary stepped up new_primary = _partition_node(ha_tools.get_primary()) self.assertEqual(new_primary, rs_state.writer) new_secondaries = partition_nodes( ha_tools.get_secondaries()) self.assertEqual(set(new_secondaries), rs_state.secondaries) break else: self.fail( "No new primary after %s seconds. Old primary was %s, current" " is %s" % (patience_seconds, primary, ha_tools.get_primary()))
def setUp(self): members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res self.c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
def test_stepdown_triggers_refresh(self): c_find_one = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual(one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes sleep(1) # Trigger a refresh self.assertRaises(AutoReconnect, c_find_one.test.test.find_one) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled sleep(1) # We've detected the stepdown self.assertTrue(not c_find_one.primary or _partition_node(primary) != c_find_one.primary)
def count(self, filters): with statsd.timer('%s.%s' % (__name__, self.__class__.__name__)): try: client = MongoReplicaSetClient(self.host, replicaSet=self.replicaSet, w=self.writeConcern, j=self.journal, slave_okay=True, connectTimeoutMS=200) except ConnectionFailure as e: self.log.error("Connection falure error reached: %r" % e, exc_info=True) raise Exception(e) db = client[self.mongodb['database']] collection = db[self.account] try: if filters: results = collection.find(filters).count() else: results = db.command('collstats', self.account)['count'] client.close() return results except OperationFailure as e: self.log.error(e, exc_info=True) return 0
def __init__(self): """ Constructor """ # Configure the connection self.configure() if self.config['replica_set'] is not None: connection = MongoReplicaSetClient( self.config['uri'], replicaSet=self.config['replica_set'], w=self.config['write_concern'], fsync=self.config['fsync'], read_preference=ReadPreference.PRIMARY_PREFERRED) else: # Connecting to a stand alone MongoDB connection = MongoClient( self.config['uri'], fsync=self.config['fsync'], read_preference=ReadPreference.PRIMARY) # Set up the collection database = connection[self.config['database']] self.collection = database[self.config['collection']] log.msg('Connected to MongoDB {0}, using "{1}/{2}"'.format( self.config['uri'], self.config['database'], self.config['collection'])) # Ensure unique index if self.config['unique_key']: self.collection.ensure_index(self.config['unique_key'], unique=True) log.msg('Ensuring index for key {0}'.format( self.config['unique_key']))
def test_secondary_failure(self): c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries def readers_changed(): for _ in xrange(20): if c.secondaries != secondaries: return True sleep(1) return False killed = ha_tools.kill_secondary() sleep(2 * MONITOR_INTERVAL) self.assertTrue(bool(len(killed))) self.assertEqual(primary, c.primary) self.assertTrue(readers_changed()) secondaries = c.secondaries ha_tools.restart_members([killed]) self.assertEqual(primary, c.primary) self.assertTrue(readers_changed())
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = MongoClient(primary, use_greenlets=use_greenlets) secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets) rsc = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
def get(self, filename): if not filename: raise Exception("No filename given: %r" % filename) try: client = MongoReplicaSetClient(self.host, replicaSet=self.replicaSet, w=self.writeConcern, j=self.journal, slave_okay=True, read_preference=self.r_p, connectTimeoutMS=200) except ConnectionFailure as e: self.log.error("Connection falure error reached: %r" % e, exc_info=True) raise Exception(e) db = client[self.current_db] self.fs = GridFS(db) try: result = self.fs.get_last_version(filename=filename) info = self.info(filename=filename) info['content'] = result.read() return info except PyMongoError as e: self.log.error(e, exc_info=True) raise Exception(e)
def test_get_default_database(self): host = one(self.hosts) uri = "mongodb://%s:%d/foo?replicaSet=%s" % (host[0], host[1], self.name) c = MongoReplicaSetClient(uri, _connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database())
def test_get_default_database_error(self): host = one(self.hosts) # URI with no database. uri = "mongodb://%s:%d/?replicaSet=%s" % (host[0], host[1], self.name) c = MongoReplicaSetClient(uri, _connect=False) self.assertRaises(ConfigurationError, c.get_default_database)
def test_get_default_database_with_authsource(self): # Ensure we distinguish database name from authSource. host = one(self.hosts) uri = "mongodb://%s:%d/foo?replicaSet=%s&authSource=src" % ( host[0], host[1], self.name) c = MongoReplicaSetClient(uri, _connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database())
def test_init_disconnected_with_auth_failure(self): c = MongoReplicaSetClient( "mongodb://*****:*****@somedomainthatdoesntexist", replicaSet="rs", connectTimeoutMS=1, _connect=False) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)