def test_auto_ref_and_deref(self): # Test same functionality as in PyMongo's test_database.py; the # implementation for Motor for async is a little complex so we test # that it works here, and we don't just rely on synchrotest # to cover it. db = self.db # We test a special hack where add_son_manipulator corrects our mistake # if we pass a MotorDatabase, instead of Database, to AutoReference. db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) a = {"hello": "world"} b = {"test": a} c = {"another test": b} yield db.a.delete_many({}) yield db.b.delete_many({}) yield db.c.delete_many({}) yield db.a.save(a) yield db.b.save(b) yield db.c.save(c) a["hello"] = "mike" yield db.a.save(a) result_a = yield db.a.find_one() result_b = yield db.b.find_one() result_c = yield db.c.find_one() self.assertEqual(a, result_a) self.assertEqual(a, result_b["test"]) self.assertEqual(a, result_c["another test"]["test"]) self.assertEqual(b, result_b) self.assertEqual(b, result_c["another test"]) self.assertEqual(c, result_c)
def test_auto_ref_and_deref(self): # Legacy API. db = self.client.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.test.a.remove({}) db.test.b.remove({}) db.test.c.remove({}) a = {"hello": u"world"} db.test.a.save(a) b = {"test": a} db.test.b.save(b) c = {"another test": b} db.test.c.save(c) a["hello"] = "mike" db.test.a.save(a) self.assertEqual(db.test.a.find_one(), a) self.assertEqual(db.test.b.find_one()["test"], a) self.assertEqual(db.test.c.find_one()["another test"]["test"], a) self.assertEqual(db.test.b.find_one(), b) self.assertEqual(db.test.c.find_one()["another test"], b) self.assertEqual(db.test.c.find_one(), c)
def initialize(name=None, seeds=None, max_pool_size=None): """ Initialize the connection pool and top-level database for pulp. """ global _CONNECTION, _DATABASE try: if name is None: name = config.config.get('database', 'name') if seeds is None: seeds = config.config.get('database', 'seeds') if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE _LOG.info("Attempting Database connection with seeds = %s" % (seeds)) _CONNECTION = pymongo.Connection(seeds, max_pool_size=max_pool_size) _DATABASE = getattr(_CONNECTION, name) _DATABASE.add_son_manipulator(NamespaceInjector()) _DATABASE.add_son_manipulator(AutoReference(_DATABASE)) _LOG.info( "Database connection established with: seeds = %s, name = %s" % (seeds, name)) except: _LOG.critical('Database initialization failed') _CONNECTION = None _DATABASE = None raise
def __init__(self): host = MONGODB_SETTINGS['host'] port = MONGODB_SETTINGS['port'] max_pool = MONGODB_SETTINGS['max_pool'] self.connection = motor.MotorClient(host, port, max_pool).open_sync() self.db = self.connection[MONGODB_SETTINGS["database"]] self.db.add_son_manipulator(NamespaceInjector()) self.db.add_son_manipulator(AutoReference(self.db))
def connect(db_name='pulp_database'): """ Connect to a test database. Optional argument, db_name, specifies the name of the database. """ global _connection, _db _connection = pymongo.Connection() _db = getattr(_connection, db_name) _db.add_son_manipulator(NamespaceInjector())
def database(self): """ Returns a connection to the given database. """ connection = self.connection database = getattr(connection, self.db_name) database.add_son_manipulator(NamespaceInjector()) database.add_son_manipulator(AutoReference(database)) return database
def __init__(cls): uri = 'mongodb://' + MONGODB_USERNAME + ':' + MONGODB_PWD + '@' + MONGODB_SERVER + ':' + MONGODB_PORT \ + '/' + MONGODB_DB client = MongoClient(uri) cls.database = client.get_database(MONGODB_DB) database = client.get_database(MONGODB_DB) # 自动创建与解引用 database.add_son_manipulator(NamespaceInjector()) database.add_son_manipulator(AutoReference(database)) cls.blogItemCollection = database.get_collection( MONGODB_BLOG_ITEM_COLLECTION) cls.authorItemCollection = database.get_collection( MONGODB_AUTHOR_ITEM_COLLECTION)
def test_ns_injection(self): manip = NamespaceInjector() collection = self.db.test def incoming_adds_ns(son): son = manip.transform_incoming(son, collection) assert "_ns" in son return son["_ns"] == collection.name qcheck.check_unittest(self, incoming_adds_ns, qcheck.gen_mongo_dict(3)) def outgoing_is_identity(son): return son == manip.transform_outgoing(son, collection) qcheck.check_unittest(self, outgoing_is_identity, qcheck.gen_mongo_dict(3))
def test_manipulator_properties(self): db = self.client.foo self.assertEqual([], db.incoming_manipulators) self.assertEqual([], db.incoming_copying_manipulators) self.assertEqual([], db.outgoing_manipulators) self.assertEqual([], db.outgoing_copying_manipulators) db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.add_son_manipulator(ObjectIdShuffler()) self.assertEqual(1, len(db.incoming_manipulators)) self.assertEqual(db.incoming_manipulators, ['NamespaceInjector']) self.assertEqual(2, len(db.incoming_copying_manipulators)) for name in db.incoming_copying_manipulators: self.assertTrue(name in ('ObjectIdShuffler', 'AutoReference')) self.assertEqual([], db.outgoing_manipulators) self.assertEqual(['AutoReference'], db.outgoing_copying_manipulators)
def session(self): """ Returns MongoDB """ if not hasattr(self, "db"): self.db = self.connection[self.app.config['MONGODB_DATABASE']] # we need namespaces in any case self.db.add_son_manipulator(NamespaceInjector()) if self.app.config['MONGODB_AUTOREF']: self.db.add_son_manipulator(AutoReference(self.db)) if self.app.config['MONGODB_AUTOINCREMENT']: self.db.add_son_manipulator(AutoincrementId()) self.db.add_son_manipulator(SavedObject()) return self.db
def test_auto_ref_and_deref_list(self): # Legacy API. db = self.client.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.drop_collection("users") db.drop_collection("messages") message_1 = {"title": "foo"} db.messages.save(message_1) message_2 = {"title": "bar"} db.messages.save(message_2) user = {"messages": [message_1, message_2]} db.users.save(user) db.messages.update(message_1, {"title": "buzz"}) self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"]) self.assertEqual("bar", db.users.find_one()["messages"][1]["title"])
def test_marc(self): db = self.connection.pymongo_test db.add_son_manipulator(AutoReference(db)) db.add_son_manipulator(NamespaceInjector()) db.drop_collection("users") db.drop_collection("messages") message_1 = {"title": "foo"} db.messages.save(message_1) message_2 = {"title": "bar"} db.messages.save(message_2) user = {"name": "marc", "messages": [message_1, message_2]} db.users.save(user) message = db.messages.find_one() db.messages.update(message, {"title": "buzz"}) self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"]) self.assertEqual("bar", db.users.find_one()["messages"][1]["title"])
def initialize(name=None, seeds=None): """ Initialize the connection pool and top-level database for pulp. """ global _connection, _database try: if not name: name = config.config.get('database', 'name') if not seeds: seeds = config.config.get('database', 'seeds') _log.info("Attempting Database connection with seeds = %s" % (seeds)) _connection = pymongo.Connection(seeds) _database = getattr(_connection, name) _database.add_son_manipulator(NamespaceInjector()) _database.add_son_manipulator(AutoReference(_database)) _log.info( "Database connection established with: seeds = %s, name = %s" % (seeds, name)) except Exception: _log.critical('Database initialization failed') _connection = None _database = None raise
def testRemoveWithRef(self): """ Remove a collection object while it is referenced in a separate collection. Use of AutoReference causes the external reference to become None, reflecting the object was deleted. """ connection = Connection() db = connection._test_dbrefs db.add_son_manipulator(NamespaceInjector()) db.add_son_manipulator(AutoReference(db)) db.drop_collection("books") db.drop_collection("groups") books = db.books groups = db.groups # Create book-1 bk1 = {} bk1["name"] = "Test-A" bk1["created-time"] = time.time() books.save(bk1) # Create group-1 grp1 = {} grp1["name"] = "FirstGroup" grp1["books"] = [bk1] groups.save(grp1) # lookup 'Test-A', then delete it b = books.find_one({"name":"Test-A"}) books.remove(b) found = [f for f in books.find({"name": "Test-A"})] self.assertTrue(len(found) == 0) found = [f for f in groups.find({"name": "FirstGroup"})] self.assertTrue(len(found) == 1) self.assertTrue(found[0]["name"] == "FirstGroup") self.assertTrue(len(found[0]["books"]) == 1) self.assertTrue(found[0]["books"][0] == None)
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32): """ Initialize the connection pool and top-level database for pulp. :param max_timeout: the maximum number of seconds to wait between connection retries :type max_timeout: int """ global _CONNECTION, _DATABASE try: connection_kwargs = {} if name is None: name = config.config.get('database', 'name') if seeds is None: seeds = config.config.get('database', 'seeds') if seeds != '': first_seed = seeds.split(',')[0] seed = first_seed.strip().split(':') if len(seed) == 2: connection_kwargs.update({ 'host': seed[0], 'port': int(seed[1]) }) else: connection_kwargs.update({'host': seed[0]}) if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE connection_kwargs['max_pool_size'] = max_pool_size if replica_set is None: if config.config.has_option('database', 'replica_set'): replica_set = config.config.get('database', 'replica_set') if replica_set is not None: connection_kwargs['replicaset'] = replica_set # Process SSL settings if config.config.getboolean('database', 'ssl'): connection_kwargs['ssl'] = True ssl_keyfile = config.config.get('database', 'ssl_keyfile') ssl_certfile = config.config.get('database', 'ssl_certfile') if ssl_keyfile: connection_kwargs['ssl_keyfile'] = ssl_keyfile if ssl_certfile: connection_kwargs['ssl_certfile'] = ssl_certfile verify_ssl = config.config.getboolean('database', 'verify_ssl') connection_kwargs[ 'ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE connection_kwargs['ssl_ca_certs'] = config.config.get( 'database', 'ca_path') # If username & password have been specified in the database config, # attempt to authenticate to the database username = config.config.get('database', 'username') password = config.config.get('database', 'password') if username: _logger.debug( _('Attempting username and password authentication.')) connection_kwargs['username'] = username connection_kwargs['password'] = password elif password and not username: raise Exception( _("The server config specified a database password, but is " "missing a database username.")) shadow_connection_kwargs = copy.deepcopy(connection_kwargs) if connection_kwargs.get('password'): shadow_connection_kwargs['password'] = '******' _logger.debug(_('Connection Arguments: %s') % shadow_connection_kwargs) # Wait until the Mongo database is available mongo_retry_timeout_seconds_generator = itertools.chain( [1, 2, 4, 8, 16], itertools.repeat(32)) while True: try: _CONNECTION = mongoengine.connect(name, **connection_kwargs) except mongoengine.connection.ConnectionError as e: next_delay = min(mongo_retry_timeout_seconds_generator.next(), max_timeout) msg = _( "Could not connect to MongoDB at %(url)s:\n%(e)s\n... Waiting " "%(retry_timeout)s seconds and trying again.") _logger.error(msg % { 'retry_timeout': next_delay, 'url': seeds, 'e': str(e) }) else: break time.sleep(next_delay) try: _DATABASE = mongoengine.connection.get_db() except OperationFailure as error: if error.code == 18: msg = _('Authentication to MongoDB ' 'with username and password failed.') raise RuntimeError(msg) _DATABASE.add_son_manipulator(NamespaceInjector()) # Query the collection names to ensure that we are authenticated properly _logger.debug(_('Querying the database to validate the connection.')) _DATABASE.collection_names() db_version = _CONNECTION.server_info()['version'] _logger.info( _("Mongo database for connection is version %s") % db_version) db_version_tuple = tuple(db_version.split(".")) db_min_version_tuple = tuple(MONGO_MINIMUM_VERSION.split(".")) if db_version_tuple < db_min_version_tuple: raise RuntimeError( _("Pulp requires Mongo version %s, but DB is reporting version %s" ) % (MONGO_MINIMUM_VERSION, db_version)) except Exception, e: _logger.critical(_('Database initialization failed: %s') % str(e)) _CONNECTION = None _DATABASE = None raise
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32): """ Initialize the connection pool and top-level database for pulp. Calling this more than once will raise a RuntimeError. :param max_timeout: the maximum number of seconds to wait between connection retries :type max_timeout: int """ global _CONNECTION, _DATABASE # We do not allow a second call to initialize(), as mongoengine.connect() will cache the last # initialized connection for all calls. Thus, any process that attempts to call initialize() # again might alter which database all further queries are made against. if _CONNECTION or _DATABASE: return try: connection_kwargs = {} if name is None: name = pulp2_mongodb.get('name') if seeds is None: seeds = pulp2_mongodb.get('seeds') seeds_list = seeds.split(',') if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE connection_kwargs['maxPoolSize'] = max_pool_size if replica_set is None: if pulp2_mongodb.get('replica_set'): replica_set = pulp2_mongodb.get('replica_set') if replica_set is not None: connection_kwargs['replicaSet'] = replica_set # Process SSL settings if pulp2_mongodb.get('ssl'): connection_kwargs['ssl'] = True ssl_keyfile = pulp2_mongodb.get('ssl_keyfile') ssl_certfile = pulp2_mongodb.get('ssl_certfile') if ssl_keyfile: connection_kwargs['ssl_keyfile'] = ssl_keyfile if ssl_certfile: connection_kwargs['ssl_certfile'] = ssl_certfile verify_ssl = pulp2_mongodb.get('verify_ssl') connection_kwargs['ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE connection_kwargs['ssl_ca_certs'] = pulp2_mongodb.get('ca_path') # If username & password have been specified in the database config, # attempt to authenticate to the database username = pulp2_mongodb.get('username') password = pulp2_mongodb.get('password') if username: _logger.debug(_('Attempting username and password authentication.')) connection_kwargs['username'] = username connection_kwargs['password'] = password elif password and not username: raise ConfigurationError(_("The config specified a database password, but is " "missing a database username.")) # Wait until the Mongo database is available mongo_retry_timeout_seconds_generator = itertools.chain([1, 2, 4, 8, 16], itertools.repeat(32)) if seeds != '': if len(seeds_list) > 1 and not replica_set: raise ConfigurationError(_("Database 'replica_set' config must be specified " "when more than one seed is provided.")) while True: _CONNECTION = _connect_to_one_of_seeds(connection_kwargs, seeds_list, name) if _CONNECTION: db_version = semantic_version.Version(_CONNECTION.server_info()['version']) if db_version < MONGO_MINIMUM_VERSION: raise RuntimeError(_("Pulp requires Mongo version %s, but DB is reporting" "version %s") % (MONGO_MINIMUM_VERSION, db_version)) break else: next_delay = min(next(mongo_retry_timeout_seconds_generator), max_timeout) msg = _("Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting " "%(retry_timeout)s seconds and trying again.") _logger.error(msg % {'retry_timeout': next_delay, 'url': seeds}) time.sleep(next_delay) else: raise ConfigurationError(_("Database 'seeds' config must include at least one " "hostname:port value.")) try: _DATABASE = mongoengine.connection.get_db() except OperationFailure as error: if error.code == 18: msg = _('Authentication to MongoDB ' 'with username and password failed.') raise RuntimeError(msg) _DATABASE.add_son_manipulator(NamespaceInjector()) # Query the collection names to ensure that we are authenticated properly _logger.debug(_('Querying the database to validate the connection.')) _DATABASE.collection_names() except Exception as e: _logger.critical(_('Database initialization failed: %s') % str(e)) _CONNECTION = None _DATABASE = None raise
parser.add_option('-a', '--autoref', action='store_true', help='Run with SON AutoReference Manipulators', default=False) parser.add_option('-r', '--num_repos', action='store', help='How many repos to display, default is all', default=None) parser.add_option('-p', '--num_packages', action='store', help='How many packages to display', default=5) options, args = parser.parse_args() connection = Connection() db = connection.pulp_database if options.autoref: db.add_son_manipulator(NamespaceInjector()) db.add_son_manipulator(AutoReference(db)) repos = db.repos found = repos.find() found_slice = found if options.num_repos: found_slice = found[:int(options.num_repos)] for r in found_slice: print "\nRepo: ", r
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None): """ Initialize the connection pool and top-level database for pulp. """ global _CONNECTION, _DATABASE try: connection_kwargs = {} if name is None: name = config.config.get('database', 'name') if seeds is None: seeds = config.config.get('database', 'seeds') if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE connection_kwargs['max_pool_size'] = max_pool_size if replica_set is None: if config.config.has_option('database', 'replica_set'): replica_set = config.config.get('database', 'replica_set') if replica_set is not None: connection_kwargs['replicaset'] = replica_set _LOG.info("Attempting Database connection with seeds = %s" % seeds) _LOG.info('Connection Arguments: %s' % connection_kwargs) _CONNECTION = pymongo.MongoClient(seeds, **connection_kwargs) # Decorate the methods that actually send messages to the db over the # network. These are the methods that call start_request, and the # decorator causes them call an corresponding end_request _CONNECTION._send_message = _end_request_decorator(_CONNECTION._send_message) _CONNECTION._send_message_with_response = _end_request_decorator(_CONNECTION._send_message_with_response) _DATABASE = getattr(_CONNECTION, name) # If username & password have been specified in the database config, # attempt to authenticate to the database if config.config.has_option('database', 'username') and \ config.config.has_option('database', 'password'): username = config.config.get('database', 'username') password = config.config.get('database', 'password') _LOG.info('Database authentication enabled, attempting username/password' 'authentication.') _DATABASE.authenticate(username, password) elif ((config.config.has_option('database', 'username') and not config.config.has_option('database', 'password')) or (not config.config.has_option('database', 'username') and config.config.has_option('database', 'password'))): raise Exception("The server config specified username/password authentication but " "is missing either the username or the password") _DATABASE.add_son_manipulator(NamespaceInjector()) # Query the collection names to ensure that we are authenticated properly _LOG.debug("Querying the database to validate the connection.") _DATABASE.collection_names() _LOG.info("Database connection established with: seeds = %s, name = %s" % (seeds, name)) except Exception, e: _LOG.critical('Database initialization failed: %s' % str(e)) _CONNECTION = None _DATABASE = None raise
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32): """ Initialize the connection pool and top-level database for pulp. Calling this more than once will raise a RuntimeError. :param max_timeout: the maximum number of seconds to wait between connection retries :type max_timeout: int :raises RuntimeError: This Exception is raised if initialize is called more than once """ global _CONNECTION, _DATABASE # We do not allow a second call to initialize(), as mongoengine.connect() will cache the last # initialized connection for all calls. Thus, any process that attempts to call initialize() # again might alter which database all further queries are made against. By raising this # Exception, we can ensure that only one database connection is established per process which # will help us to ensure that the connection does not get overridden later. if _CONNECTION or _DATABASE: raise RuntimeError( "The database is already initialized. It is an error to call this " "function a second time.") try: connection_kwargs = {} if name is None: name = config.config.get('database', 'name') if seeds is None: seeds = config.config.get('database', 'seeds') seeds_list = seeds.split(',') if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE connection_kwargs['max_pool_size'] = max_pool_size if replica_set is None: if config.config.has_option('database', 'replica_set'): replica_set = config.config.get('database', 'replica_set') if replica_set is not None: connection_kwargs['replicaSet'] = replica_set write_concern = config.config.get('database', 'write_concern') if write_concern not in ['majority', 'all']: raise PulpCodedException(error_code=error_codes.PLP0043) elif write_concern == 'all': write_concern = len(seeds_list) # Process SSL settings if config.config.getboolean('database', 'ssl'): connection_kwargs['ssl'] = True ssl_keyfile = config.config.get('database', 'ssl_keyfile') ssl_certfile = config.config.get('database', 'ssl_certfile') if ssl_keyfile: connection_kwargs['ssl_keyfile'] = ssl_keyfile if ssl_certfile: connection_kwargs['ssl_certfile'] = ssl_certfile verify_ssl = config.config.getboolean('database', 'verify_ssl') connection_kwargs[ 'ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE connection_kwargs['ssl_ca_certs'] = config.config.get( 'database', 'ca_path') # If username & password have been specified in the database config, # attempt to authenticate to the database username = config.config.get('database', 'username') password = config.config.get('database', 'password') if username: _logger.debug( _('Attempting username and password authentication.')) connection_kwargs['username'] = username connection_kwargs['password'] = password elif password and not username: raise Exception( _("The server config specified a database password, but is " "missing a database username.")) # Wait until the Mongo database is available mongo_retry_timeout_seconds_generator = itertools.chain( [1, 2, 4, 8, 16], itertools.repeat(32)) if seeds != '': if len(seeds_list) > 1 and not replica_set: raise PulpCodedException(error_code=error_codes.PLP0041) while True: _CONNECTION = _connect_to_one_of_seeds(connection_kwargs, seeds_list, name) if _CONNECTION: db_version = semantic_version.Version( _CONNECTION.server_info()['version']) if db_version < MONGO_MINIMUM_VERSION: raise RuntimeError( _("Pulp requires Mongo version %s, but DB is reporting" "version %s") % (MONGO_MINIMUM_VERSION, db_version)) elif db_version >= MONGO_WRITE_CONCERN_VERSION or replica_set: # Write concern of 'majority' only works with a replica set or when using # MongoDB >= 2.6.0 _CONNECTION.write_concern['w'] = write_concern else: _CONNECTION.write_concern['w'] = 1 _logger.info( _("Write concern for Mongo connection: %s") % _CONNECTION.write_concern) break else: next_delay = min( mongo_retry_timeout_seconds_generator.next(), max_timeout) msg = _( "Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting " "%(retry_timeout)s seconds and trying again.") _logger.error(msg % { 'retry_timeout': next_delay, 'url': seeds }) time.sleep(next_delay) else: raise PulpCodedException(error_code=error_codes.PLP0040) try: _DATABASE = mongoengine.connection.get_db() except OperationFailure as error: if error.code == 18: msg = _('Authentication to MongoDB ' 'with username and password failed.') raise RuntimeError(msg) _DATABASE.add_son_manipulator(NamespaceInjector()) # Query the collection names to ensure that we are authenticated properly _logger.debug(_('Querying the database to validate the connection.')) _DATABASE.collection_names() except Exception, e: _logger.critical(_('Database initialization failed: %s') % str(e)) _CONNECTION = None _DATABASE = None raise
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None): """ Initialize the connection pool and top-level database for pulp. """ global _CONNECTION, _DATABASE try: connection_kwargs = {} if name is None: name = config.config.get('database', 'name') if seeds is None: seeds = config.config.get('database', 'seeds') if max_pool_size is None: # we may want to make this configurable, but then again, we may not max_pool_size = _DEFAULT_MAX_POOL_SIZE connection_kwargs['max_pool_size'] = max_pool_size if replica_set is None: if config.config.has_option('database', 'replica_set'): replica_set = config.config.get('database', 'replica_set') if replica_set is not None: connection_kwargs['replicaset'] = replica_set _log.debug("Attempting Database connection with seeds = %s" % seeds) _log.debug('Connection Arguments: %s' % connection_kwargs) # Wait until the Mongo database is available while True: try: _CONNECTION = pymongo.MongoClient(seeds, **connection_kwargs) except pymongo.errors.ConnectionFailure: next_delay = _MONGO_RETRY_TIMEOUT_SECONDS_GENERATOR.next() msg = _( "Could not connect to MongoDB at %(url)s ... Waiting %(retry_timeout)s seconds " "and trying again.") _log.error(msg % {'retry_timeout': next_delay, 'url': seeds}) else: break time.sleep(next_delay) # Decorate the methods that actually send messages to the db over the # network. These are the methods that call start_request, and the # decorator causes them call an corresponding end_request _CONNECTION._send_message = _end_request_decorator( _CONNECTION._send_message) _CONNECTION._send_message_with_response = _end_request_decorator( _CONNECTION._send_message_with_response) _DATABASE = getattr(_CONNECTION, name) # If username & password have been specified in the database config, # attempt to authenticate to the database username = config.config.get('database', 'username') password = config.config.get('database', 'password') if username and password: _log.debug( 'Database authentication enabled, attempting username/password' ' authentication.') _DATABASE.authenticate(username, password) elif (username and not password) or (password and not username): raise Exception( "The server config specified username/password authentication but " "is missing either the username or the password") _DATABASE.add_son_manipulator(NamespaceInjector()) # Query the collection names to ensure that we are authenticated properly _log.debug("Querying the database to validate the connection.") _DATABASE.collection_names() db_version = _CONNECTION.server_info()['version'] _log.info( _("Mongo database for connection is version %s") % db_version) db_version_tuple = tuple(db_version.split(".")) db_min_version_tuple = tuple(MONGO_MINIMUM_VERSION.split(".")) if db_version_tuple < db_min_version_tuple: raise RuntimeError( _("Pulp requires Mongo version %s, but DB is reporting version %s" ) % (MONGO_MINIMUM_VERSION, db_version)) _log.debug( "Database connection established with: seeds = %s, name = %s" % (seeds, name)) except Exception, e: _log.critical('Database initialization failed: %s' % str(e)) _CONNECTION = None _DATABASE = None raise
def _database(self, db_name): connection = self._connection() database = getattr(connection, db_name) database.add_son_manipulator(NamespaceInjector()) database.add_son_manipulator(AutoReference(database)) return database
def testChangesWithRef(self): """ Change a collection object while it is referenced in another collection, use AutoReference Manipulator so the change is reflected. """ connection = Connection() db = connection._test_dbrefs #manipulators are required to keep references in sync db.add_son_manipulator(NamespaceInjector()) db.add_son_manipulator(AutoReference(db)) db.drop_collection("books") db.drop_collection("groups") books = db.books groups = db.groups # Create book-1 bk1 = {} bk1["name"] = "Test-A" bk1["created-time"] = time.time() books.save(bk1) # Create group-1 grp1 = {} grp1["name"] = "FirstGroup" grp1["books"] = [bk1] groups.save(grp1) #Ensure that we have only 1 instance in each books/groups found = [f for f in books.find({"name": "Test-A"})] self.assertTrue(len(found) == 1) bk1_id = found[0]["_id"] found = [f for f in groups.find({"name": "FirstGroup"})] self.assertTrue(len(found) == 1) # Verify that we saved 'Test-A' under groups self.assertTrue(len(found[0]["books"]) == 1) self.assertTrue(found[0]["books"][0]["name"] == 'Test-A') self.assertTrue(found[0]["books"][0]["_id"] == bk1_id) # lookup 'Test-A', save it (no-modifications), ensure we still have only # 1 instance of 'Test-A' b = books.find_one({"name":"Test-A"}) books.save(b) found = [f for f in books.find({"name": "Test-A"})] self.assertTrue(len(found) == 1) # lookup 'Test-A' and modify it and save it, ensure we only have # 1 instance of 'Test-A' b = books.find_one({"name":"Test-A"}) b["newEntry"] = "newValue" b["modified_time"] = time.time() books.save(b) found = [f for f in books.find({"name": "Test-A"})] self.assertTrue(len(found) == 1) # Ensure _id didn't change after our modification self.assertTrue(found[0]["_id"] == bk1_id) found = [f for f in groups.find({"name": "FirstGroup"})] self.assertTrue(len(found) == 1) self.assertTrue(found[0]["name"] == "FirstGroup") self.assertTrue(found[0]["books"][0]["_id"] == bk1_id) self.assertTrue(found[0]["books"][0].has_key("newEntry")) self.assertTrue(found[0]["books"][0]["newEntry"] == "newValue")