def test_auto_ref_and_deref(self):
# Test same functionality as in PyMongo's test_database.py; the
# implementation for Motor for async is a little complex so we test
# that it works here, and we don't just rely on synchrotest
# to cover it.
db = self.db
# We test a special hack where add_son_manipulator corrects our mistake
# if we pass a MotorDatabase, instead of Database, to AutoReference.
db.add_son_manipulator(AutoReference(db))
db.add_son_manipulator(NamespaceInjector())
a = {"hello": "world"}
b = {"test": a}
c = {"another test": b}
yield db.a.delete_many({})
yield db.b.delete_many({})
yield db.c.delete_many({})
yield db.a.save(a)
yield db.b.save(b)
yield db.c.save(c)
a["hello"] = "mike"
yield db.a.save(a)
result_a = yield db.a.find_one()
result_b = yield db.b.find_one()
result_c = yield db.c.find_one()
self.assertEqual(a, result_a)
self.assertEqual(a, result_b["test"])
self.assertEqual(a, result_c["another test"]["test"])
self.assertEqual(b, result_b)
self.assertEqual(b, result_c["another test"])
self.assertEqual(c, result_c)
def test_auto_ref_and_deref(self):
# Legacy API.
db = self.client.pymongo_test
db.add_son_manipulator(AutoReference(db))
db.add_son_manipulator(NamespaceInjector())
db.test.a.remove({})
db.test.b.remove({})
db.test.c.remove({})
a = {"hello": u"world"}
db.test.a.save(a)
b = {"test": a}
db.test.b.save(b)
c = {"another test": b}
db.test.c.save(c)
a["hello"] = "mike"
db.test.a.save(a)
self.assertEqual(db.test.a.find_one(), a)
self.assertEqual(db.test.b.find_one()["test"], a)
self.assertEqual(db.test.c.find_one()["another test"]["test"], a)
self.assertEqual(db.test.b.find_one(), b)
self.assertEqual(db.test.c.find_one()["another test"], b)
self.assertEqual(db.test.c.find_one(), c)
def initialize(name=None, seeds=None, max_pool_size=None):
"""
Initialize the connection pool and top-level database for pulp.
"""
global _CONNECTION, _DATABASE
try:
if name is None:
name = config.config.get('database', 'name')
if seeds is None:
seeds = config.config.get('database', 'seeds')
if max_pool_size is None:
# we may want to make this configurable, but then again, we may not
max_pool_size = _DEFAULT_MAX_POOL_SIZE
_LOG.info("Attempting Database connection with seeds = %s" % (seeds))
_CONNECTION = pymongo.Connection(seeds, max_pool_size=max_pool_size)
_DATABASE = getattr(_CONNECTION, name)
_DATABASE.add_son_manipulator(NamespaceInjector())
_DATABASE.add_son_manipulator(AutoReference(_DATABASE))
_LOG.info(
"Database connection established with: seeds = %s, name = %s" %
(seeds, name))
except:
_LOG.critical('Database initialization failed')
_CONNECTION = None
_DATABASE = None
raise
def __init__(self):
host = MONGODB_SETTINGS['host']
port = MONGODB_SETTINGS['port']
max_pool = MONGODB_SETTINGS['max_pool']
self.connection = motor.MotorClient(host, port, max_pool).open_sync()
self.db = self.connection[MONGODB_SETTINGS["database"]]
self.db.add_son_manipulator(NamespaceInjector())
self.db.add_son_manipulator(AutoReference(self.db))
def connect(db_name='pulp_database'):
"""
Connect to a test database.
Optional argument, db_name, specifies the name of the database.
"""
global _connection, _db
_connection = pymongo.Connection()
_db = getattr(_connection, db_name)
_db.add_son_manipulator(NamespaceInjector())
def database(self):
"""
Returns a connection to the given database.
"""
connection = self.connection
database = getattr(connection, self.db_name)
database.add_son_manipulator(NamespaceInjector())
database.add_son_manipulator(AutoReference(database))
return database
def __init__(cls):
uri = 'mongodb://' + MONGODB_USERNAME + ':' + MONGODB_PWD + '@' + MONGODB_SERVER + ':' + MONGODB_PORT \
+ '/' + MONGODB_DB
client = MongoClient(uri)
cls.database = client.get_database(MONGODB_DB)
database = client.get_database(MONGODB_DB)
# 自动创建与解引用
database.add_son_manipulator(NamespaceInjector())
database.add_son_manipulator(AutoReference(database))
cls.blogItemCollection = database.get_collection(
MONGODB_BLOG_ITEM_COLLECTION)
cls.authorItemCollection = database.get_collection(
MONGODB_AUTHOR_ITEM_COLLECTION)
def test_ns_injection(self):
manip = NamespaceInjector()
collection = self.db.test
def incoming_adds_ns(son):
son = manip.transform_incoming(son, collection)
assert "_ns" in son
return son["_ns"] == collection.name
qcheck.check_unittest(self, incoming_adds_ns,
qcheck.gen_mongo_dict(3))
def outgoing_is_identity(son):
return son == manip.transform_outgoing(son, collection)
qcheck.check_unittest(self, outgoing_is_identity,
qcheck.gen_mongo_dict(3))
def test_manipulator_properties(self):
db = self.client.foo
self.assertEqual([], db.incoming_manipulators)
self.assertEqual([], db.incoming_copying_manipulators)
self.assertEqual([], db.outgoing_manipulators)
self.assertEqual([], db.outgoing_copying_manipulators)
db.add_son_manipulator(AutoReference(db))
db.add_son_manipulator(NamespaceInjector())
db.add_son_manipulator(ObjectIdShuffler())
self.assertEqual(1, len(db.incoming_manipulators))
self.assertEqual(db.incoming_manipulators, ['NamespaceInjector'])
self.assertEqual(2, len(db.incoming_copying_manipulators))
for name in db.incoming_copying_manipulators:
self.assertTrue(name in ('ObjectIdShuffler', 'AutoReference'))
self.assertEqual([], db.outgoing_manipulators)
self.assertEqual(['AutoReference'], db.outgoing_copying_manipulators)
def session(self):
""" Returns MongoDB
"""
if not hasattr(self, "db"):
self.db = self.connection[self.app.config['MONGODB_DATABASE']]
# we need namespaces in any case
self.db.add_son_manipulator(NamespaceInjector())
if self.app.config['MONGODB_AUTOREF']:
self.db.add_son_manipulator(AutoReference(self.db))
if self.app.config['MONGODB_AUTOINCREMENT']:
self.db.add_son_manipulator(AutoincrementId())
self.db.add_son_manipulator(SavedObject())
return self.db
def test_auto_ref_and_deref_list(self):
# Legacy API.
db = self.client.pymongo_test
db.add_son_manipulator(AutoReference(db))
db.add_son_manipulator(NamespaceInjector())
db.drop_collection("users")
db.drop_collection("messages")
message_1 = {"title": "foo"}
db.messages.save(message_1)
message_2 = {"title": "bar"}
db.messages.save(message_2)
user = {"messages": [message_1, message_2]}
db.users.save(user)
db.messages.update(message_1, {"title": "buzz"})
self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"])
self.assertEqual("bar", db.users.find_one()["messages"][1]["title"])
def test_marc(self):
db = self.connection.pymongo_test
db.add_son_manipulator(AutoReference(db))
db.add_son_manipulator(NamespaceInjector())
db.drop_collection("users")
db.drop_collection("messages")
message_1 = {"title": "foo"}
db.messages.save(message_1)
message_2 = {"title": "bar"}
db.messages.save(message_2)
user = {"name": "marc", "messages": [message_1, message_2]}
db.users.save(user)
message = db.messages.find_one()
db.messages.update(message, {"title": "buzz"})
self.assertEqual("buzz", db.users.find_one()["messages"][0]["title"])
self.assertEqual("bar", db.users.find_one()["messages"][1]["title"])
def initialize(name=None, seeds=None):
"""
Initialize the connection pool and top-level database for pulp.
"""
global _connection, _database
try:
if not name:
name = config.config.get('database', 'name')
if not seeds:
seeds = config.config.get('database', 'seeds')
_log.info("Attempting Database connection with seeds = %s" % (seeds))
_connection = pymongo.Connection(seeds)
_database = getattr(_connection, name)
_database.add_son_manipulator(NamespaceInjector())
_database.add_son_manipulator(AutoReference(_database))
_log.info(
"Database connection established with: seeds = %s, name = %s" %
(seeds, name))
except Exception:
_log.critical('Database initialization failed')
_connection = None
_database = None
raise
def testRemoveWithRef(self):
"""
Remove a collection object while it is referenced in a
separate collection. Use of AutoReference causes the external
reference to become None, reflecting the object was deleted.
"""
connection = Connection()
db = connection._test_dbrefs
db.add_son_manipulator(NamespaceInjector())
db.add_son_manipulator(AutoReference(db))
db.drop_collection("books")
db.drop_collection("groups")
books = db.books
groups = db.groups
# Create book-1
bk1 = {}
bk1["name"] = "Test-A"
bk1["created-time"] = time.time()
books.save(bk1)
# Create group-1
grp1 = {}
grp1["name"] = "FirstGroup"
grp1["books"] = [bk1]
groups.save(grp1)
# lookup 'Test-A', then delete it
b = books.find_one({"name":"Test-A"})
books.remove(b)
found = [f for f in books.find({"name": "Test-A"})]
self.assertTrue(len(found) == 0)
found = [f for f in groups.find({"name": "FirstGroup"})]
self.assertTrue(len(found) == 1)
self.assertTrue(found[0]["name"] == "FirstGroup")
self.assertTrue(len(found[0]["books"]) == 1)
self.assertTrue(found[0]["books"][0] == None)
def initialize(name=None,
seeds=None,
max_pool_size=None,
replica_set=None,
max_timeout=32):
"""
Initialize the connection pool and top-level database for pulp.
:param max_timeout: the maximum number of seconds to wait between
connection retries
:type max_timeout: int
"""
global _CONNECTION, _DATABASE
try:
connection_kwargs = {}
if name is None:
name = config.config.get('database', 'name')
if seeds is None:
seeds = config.config.get('database', 'seeds')
if seeds != '':
first_seed = seeds.split(',')[0]
seed = first_seed.strip().split(':')
if len(seed) == 2:
connection_kwargs.update({
'host': seed[0],
'port': int(seed[1])
})
else:
connection_kwargs.update({'host': seed[0]})
if max_pool_size is None:
# we may want to make this configurable, but then again, we may not
max_pool_size = _DEFAULT_MAX_POOL_SIZE
connection_kwargs['max_pool_size'] = max_pool_size
if replica_set is None:
if config.config.has_option('database', 'replica_set'):
replica_set = config.config.get('database', 'replica_set')
if replica_set is not None:
connection_kwargs['replicaset'] = replica_set
# Process SSL settings
if config.config.getboolean('database', 'ssl'):
connection_kwargs['ssl'] = True
ssl_keyfile = config.config.get('database', 'ssl_keyfile')
ssl_certfile = config.config.get('database', 'ssl_certfile')
if ssl_keyfile:
connection_kwargs['ssl_keyfile'] = ssl_keyfile
if ssl_certfile:
connection_kwargs['ssl_certfile'] = ssl_certfile
verify_ssl = config.config.getboolean('database', 'verify_ssl')
connection_kwargs[
'ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE
connection_kwargs['ssl_ca_certs'] = config.config.get(
'database', 'ca_path')
# If username & password have been specified in the database config,
# attempt to authenticate to the database
username = config.config.get('database', 'username')
password = config.config.get('database', 'password')
if username:
_logger.debug(
_('Attempting username and password authentication.'))
connection_kwargs['username'] = username
connection_kwargs['password'] = password
elif password and not username:
raise Exception(
_("The server config specified a database password, but is "
"missing a database username."))
shadow_connection_kwargs = copy.deepcopy(connection_kwargs)
if connection_kwargs.get('password'):
shadow_connection_kwargs['password'] = '******'
_logger.debug(_('Connection Arguments: %s') % shadow_connection_kwargs)
# Wait until the Mongo database is available
mongo_retry_timeout_seconds_generator = itertools.chain(
[1, 2, 4, 8, 16], itertools.repeat(32))
while True:
try:
_CONNECTION = mongoengine.connect(name, **connection_kwargs)
except mongoengine.connection.ConnectionError as e:
next_delay = min(mongo_retry_timeout_seconds_generator.next(),
max_timeout)
msg = _(
"Could not connect to MongoDB at %(url)s:\n%(e)s\n... Waiting "
"%(retry_timeout)s seconds and trying again.")
_logger.error(msg % {
'retry_timeout': next_delay,
'url': seeds,
'e': str(e)
})
else:
break
time.sleep(next_delay)
try:
_DATABASE = mongoengine.connection.get_db()
except OperationFailure as error:
if error.code == 18:
msg = _('Authentication to MongoDB '
'with username and password failed.')
raise RuntimeError(msg)
_DATABASE.add_son_manipulator(NamespaceInjector())
# Query the collection names to ensure that we are authenticated properly
_logger.debug(_('Querying the database to validate the connection.'))
_DATABASE.collection_names()
db_version = _CONNECTION.server_info()['version']
_logger.info(
_("Mongo database for connection is version %s") % db_version)
db_version_tuple = tuple(db_version.split("."))
db_min_version_tuple = tuple(MONGO_MINIMUM_VERSION.split("."))
if db_version_tuple < db_min_version_tuple:
raise RuntimeError(
_("Pulp requires Mongo version %s, but DB is reporting version %s"
) % (MONGO_MINIMUM_VERSION, db_version))
except Exception, e:
_logger.critical(_('Database initialization failed: %s') % str(e))
_CONNECTION = None
_DATABASE = None
raise
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None, max_timeout=32):
"""
Initialize the connection pool and top-level database for pulp. Calling this more than once will
raise a RuntimeError.
:param max_timeout: the maximum number of seconds to wait between
connection retries
:type max_timeout: int
"""
global _CONNECTION, _DATABASE
# We do not allow a second call to initialize(), as mongoengine.connect() will cache the last
# initialized connection for all calls. Thus, any process that attempts to call initialize()
# again might alter which database all further queries are made against.
if _CONNECTION or _DATABASE:
return
try:
connection_kwargs = {}
if name is None:
name = pulp2_mongodb.get('name')
if seeds is None:
seeds = pulp2_mongodb.get('seeds')
seeds_list = seeds.split(',')
if max_pool_size is None:
# we may want to make this configurable, but then again, we may not
max_pool_size = _DEFAULT_MAX_POOL_SIZE
connection_kwargs['maxPoolSize'] = max_pool_size
if replica_set is None:
if pulp2_mongodb.get('replica_set'):
replica_set = pulp2_mongodb.get('replica_set')
if replica_set is not None:
connection_kwargs['replicaSet'] = replica_set
# Process SSL settings
if pulp2_mongodb.get('ssl'):
connection_kwargs['ssl'] = True
ssl_keyfile = pulp2_mongodb.get('ssl_keyfile')
ssl_certfile = pulp2_mongodb.get('ssl_certfile')
if ssl_keyfile:
connection_kwargs['ssl_keyfile'] = ssl_keyfile
if ssl_certfile:
connection_kwargs['ssl_certfile'] = ssl_certfile
verify_ssl = pulp2_mongodb.get('verify_ssl')
connection_kwargs['ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE
connection_kwargs['ssl_ca_certs'] = pulp2_mongodb.get('ca_path')
# If username & password have been specified in the database config,
# attempt to authenticate to the database
username = pulp2_mongodb.get('username')
password = pulp2_mongodb.get('password')
if username:
_logger.debug(_('Attempting username and password authentication.'))
connection_kwargs['username'] = username
connection_kwargs['password'] = password
elif password and not username:
raise ConfigurationError(_("The config specified a database password, but is "
"missing a database username."))
# Wait until the Mongo database is available
mongo_retry_timeout_seconds_generator = itertools.chain([1, 2, 4, 8, 16],
itertools.repeat(32))
if seeds != '':
if len(seeds_list) > 1 and not replica_set:
raise ConfigurationError(_("Database 'replica_set' config must be specified "
"when more than one seed is provided."))
while True:
_CONNECTION = _connect_to_one_of_seeds(connection_kwargs, seeds_list, name)
if _CONNECTION:
db_version = semantic_version.Version(_CONNECTION.server_info()['version'])
if db_version < MONGO_MINIMUM_VERSION:
raise RuntimeError(_("Pulp requires Mongo version %s, but DB is reporting"
"version %s") % (MONGO_MINIMUM_VERSION,
db_version))
break
else:
next_delay = min(next(mongo_retry_timeout_seconds_generator), max_timeout)
msg = _("Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting "
"%(retry_timeout)s seconds and trying again.")
_logger.error(msg % {'retry_timeout': next_delay, 'url': seeds})
time.sleep(next_delay)
else:
raise ConfigurationError(_("Database 'seeds' config must include at least one "
"hostname:port value."))
try:
_DATABASE = mongoengine.connection.get_db()
except OperationFailure as error:
if error.code == 18:
msg = _('Authentication to MongoDB '
'with username and password failed.')
raise RuntimeError(msg)
_DATABASE.add_son_manipulator(NamespaceInjector())
# Query the collection names to ensure that we are authenticated properly
_logger.debug(_('Querying the database to validate the connection.'))
_DATABASE.collection_names()
except Exception as e:
_logger.critical(_('Database initialization failed: %s') % str(e))
_CONNECTION = None
_DATABASE = None
raise
parser.add_option('-a',
'--autoref',
action='store_true',
help='Run with SON AutoReference Manipulators',
default=False)
parser.add_option('-r',
'--num_repos',
action='store',
help='How many repos to display, default is all',
default=None)
parser.add_option('-p',
'--num_packages',
action='store',
help='How many packages to display',
default=5)
options, args = parser.parse_args()
connection = Connection()
db = connection.pulp_database
if options.autoref:
db.add_son_manipulator(NamespaceInjector())
db.add_son_manipulator(AutoReference(db))
repos = db.repos
found = repos.find()
found_slice = found
if options.num_repos:
found_slice = found[:int(options.num_repos)]
for r in found_slice:
print "\nRepo: ", r
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None):
"""
Initialize the connection pool and top-level database for pulp.
"""
global _CONNECTION, _DATABASE
try:
connection_kwargs = {}
if name is None:
name = config.config.get('database', 'name')
if seeds is None:
seeds = config.config.get('database', 'seeds')
if max_pool_size is None:
# we may want to make this configurable, but then again, we may not
max_pool_size = _DEFAULT_MAX_POOL_SIZE
connection_kwargs['max_pool_size'] = max_pool_size
if replica_set is None:
if config.config.has_option('database', 'replica_set'):
replica_set = config.config.get('database', 'replica_set')
if replica_set is not None:
connection_kwargs['replicaset'] = replica_set
_LOG.info("Attempting Database connection with seeds = %s" % seeds)
_LOG.info('Connection Arguments: %s' % connection_kwargs)
_CONNECTION = pymongo.MongoClient(seeds, **connection_kwargs)
# Decorate the methods that actually send messages to the db over the
# network. These are the methods that call start_request, and the
# decorator causes them call an corresponding end_request
_CONNECTION._send_message = _end_request_decorator(_CONNECTION._send_message)
_CONNECTION._send_message_with_response = _end_request_decorator(_CONNECTION._send_message_with_response)
_DATABASE = getattr(_CONNECTION, name)
# If username & password have been specified in the database config,
# attempt to authenticate to the database
if config.config.has_option('database', 'username') and \
config.config.has_option('database', 'password'):
username = config.config.get('database', 'username')
password = config.config.get('database', 'password')
_LOG.info('Database authentication enabled, attempting username/password'
'authentication.')
_DATABASE.authenticate(username, password)
elif ((config.config.has_option('database', 'username') and
not config.config.has_option('database', 'password')) or
(not config.config.has_option('database', 'username') and
config.config.has_option('database', 'password'))):
raise Exception("The server config specified username/password authentication but "
"is missing either the username or the password")
_DATABASE.add_son_manipulator(NamespaceInjector())
# Query the collection names to ensure that we are authenticated properly
_LOG.debug("Querying the database to validate the connection.")
_DATABASE.collection_names()
_LOG.info("Database connection established with: seeds = %s, name = %s" % (seeds, name))
except Exception, e:
_LOG.critical('Database initialization failed: %s' % str(e))
_CONNECTION = None
_DATABASE = None
raise
def initialize(name=None,
seeds=None,
max_pool_size=None,
replica_set=None,
max_timeout=32):
"""
Initialize the connection pool and top-level database for pulp. Calling this more than once will
raise a RuntimeError.
:param max_timeout: the maximum number of seconds to wait between
connection retries
:type max_timeout: int
:raises RuntimeError: This Exception is raised if initialize is called more than once
"""
global _CONNECTION, _DATABASE
# We do not allow a second call to initialize(), as mongoengine.connect() will cache the last
# initialized connection for all calls. Thus, any process that attempts to call initialize()
# again might alter which database all further queries are made against. By raising this
# Exception, we can ensure that only one database connection is established per process which
# will help us to ensure that the connection does not get overridden later.
if _CONNECTION or _DATABASE:
raise RuntimeError(
"The database is already initialized. It is an error to call this "
"function a second time.")
try:
connection_kwargs = {}
if name is None:
name = config.config.get('database', 'name')
if seeds is None:
seeds = config.config.get('database', 'seeds')
seeds_list = seeds.split(',')
if max_pool_size is None:
# we may want to make this configurable, but then again, we may not
max_pool_size = _DEFAULT_MAX_POOL_SIZE
connection_kwargs['max_pool_size'] = max_pool_size
if replica_set is None:
if config.config.has_option('database', 'replica_set'):
replica_set = config.config.get('database', 'replica_set')
if replica_set is not None:
connection_kwargs['replicaSet'] = replica_set
write_concern = config.config.get('database', 'write_concern')
if write_concern not in ['majority', 'all']:
raise PulpCodedException(error_code=error_codes.PLP0043)
elif write_concern == 'all':
write_concern = len(seeds_list)
# Process SSL settings
if config.config.getboolean('database', 'ssl'):
connection_kwargs['ssl'] = True
ssl_keyfile = config.config.get('database', 'ssl_keyfile')
ssl_certfile = config.config.get('database', 'ssl_certfile')
if ssl_keyfile:
connection_kwargs['ssl_keyfile'] = ssl_keyfile
if ssl_certfile:
connection_kwargs['ssl_certfile'] = ssl_certfile
verify_ssl = config.config.getboolean('database', 'verify_ssl')
connection_kwargs[
'ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE
connection_kwargs['ssl_ca_certs'] = config.config.get(
'database', 'ca_path')
# If username & password have been specified in the database config,
# attempt to authenticate to the database
username = config.config.get('database', 'username')
password = config.config.get('database', 'password')
if username:
_logger.debug(
_('Attempting username and password authentication.'))
connection_kwargs['username'] = username
connection_kwargs['password'] = password
elif password and not username:
raise Exception(
_("The server config specified a database password, but is "
"missing a database username."))
# Wait until the Mongo database is available
mongo_retry_timeout_seconds_generator = itertools.chain(
[1, 2, 4, 8, 16], itertools.repeat(32))
if seeds != '':
if len(seeds_list) > 1 and not replica_set:
raise PulpCodedException(error_code=error_codes.PLP0041)
while True:
_CONNECTION = _connect_to_one_of_seeds(connection_kwargs,
seeds_list, name)
if _CONNECTION:
db_version = semantic_version.Version(
_CONNECTION.server_info()['version'])
if db_version < MONGO_MINIMUM_VERSION:
raise RuntimeError(
_("Pulp requires Mongo version %s, but DB is reporting"
"version %s") %
(MONGO_MINIMUM_VERSION, db_version))
elif db_version >= MONGO_WRITE_CONCERN_VERSION or replica_set:
# Write concern of 'majority' only works with a replica set or when using
# MongoDB >= 2.6.0
_CONNECTION.write_concern['w'] = write_concern
else:
_CONNECTION.write_concern['w'] = 1
_logger.info(
_("Write concern for Mongo connection: %s") %
_CONNECTION.write_concern)
break
else:
next_delay = min(
mongo_retry_timeout_seconds_generator.next(),
max_timeout)
msg = _(
"Could not connect to any of MongoDB seeds at %(url)s:\n... Waiting "
"%(retry_timeout)s seconds and trying again.")
_logger.error(msg % {
'retry_timeout': next_delay,
'url': seeds
})
time.sleep(next_delay)
else:
raise PulpCodedException(error_code=error_codes.PLP0040)
try:
_DATABASE = mongoengine.connection.get_db()
except OperationFailure as error:
if error.code == 18:
msg = _('Authentication to MongoDB '
'with username and password failed.')
raise RuntimeError(msg)
_DATABASE.add_son_manipulator(NamespaceInjector())
# Query the collection names to ensure that we are authenticated properly
_logger.debug(_('Querying the database to validate the connection.'))
_DATABASE.collection_names()
except Exception, e:
_logger.critical(_('Database initialization failed: %s') % str(e))
_CONNECTION = None
_DATABASE = None
raise
def initialize(name=None, seeds=None, max_pool_size=None, replica_set=None):
"""
Initialize the connection pool and top-level database for pulp.
"""
global _CONNECTION, _DATABASE
try:
connection_kwargs = {}
if name is None:
name = config.config.get('database', 'name')
if seeds is None:
seeds = config.config.get('database', 'seeds')
if max_pool_size is None:
# we may want to make this configurable, but then again, we may not
max_pool_size = _DEFAULT_MAX_POOL_SIZE
connection_kwargs['max_pool_size'] = max_pool_size
if replica_set is None:
if config.config.has_option('database', 'replica_set'):
replica_set = config.config.get('database', 'replica_set')
if replica_set is not None:
connection_kwargs['replicaset'] = replica_set
_log.debug("Attempting Database connection with seeds = %s" % seeds)
_log.debug('Connection Arguments: %s' % connection_kwargs)
# Wait until the Mongo database is available
while True:
try:
_CONNECTION = pymongo.MongoClient(seeds, **connection_kwargs)
except pymongo.errors.ConnectionFailure:
next_delay = _MONGO_RETRY_TIMEOUT_SECONDS_GENERATOR.next()
msg = _(
"Could not connect to MongoDB at %(url)s ... Waiting %(retry_timeout)s seconds "
"and trying again.")
_log.error(msg % {'retry_timeout': next_delay, 'url': seeds})
else:
break
time.sleep(next_delay)
# Decorate the methods that actually send messages to the db over the
# network. These are the methods that call start_request, and the
# decorator causes them call an corresponding end_request
_CONNECTION._send_message = _end_request_decorator(
_CONNECTION._send_message)
_CONNECTION._send_message_with_response = _end_request_decorator(
_CONNECTION._send_message_with_response)
_DATABASE = getattr(_CONNECTION, name)
# If username & password have been specified in the database config,
# attempt to authenticate to the database
username = config.config.get('database', 'username')
password = config.config.get('database', 'password')
if username and password:
_log.debug(
'Database authentication enabled, attempting username/password'
' authentication.')
_DATABASE.authenticate(username, password)
elif (username and not password) or (password and not username):
raise Exception(
"The server config specified username/password authentication but "
"is missing either the username or the password")
_DATABASE.add_son_manipulator(NamespaceInjector())
# Query the collection names to ensure that we are authenticated properly
_log.debug("Querying the database to validate the connection.")
_DATABASE.collection_names()
db_version = _CONNECTION.server_info()['version']
_log.info(
_("Mongo database for connection is version %s") % db_version)
db_version_tuple = tuple(db_version.split("."))
db_min_version_tuple = tuple(MONGO_MINIMUM_VERSION.split("."))
if db_version_tuple < db_min_version_tuple:
raise RuntimeError(
_("Pulp requires Mongo version %s, but DB is reporting version %s"
) % (MONGO_MINIMUM_VERSION, db_version))
_log.debug(
"Database connection established with: seeds = %s, name = %s" %
(seeds, name))
except Exception, e:
_log.critical('Database initialization failed: %s' % str(e))
_CONNECTION = None
_DATABASE = None
raise
def _database(self, db_name):
connection = self._connection()
database = getattr(connection, db_name)
database.add_son_manipulator(NamespaceInjector())
database.add_son_manipulator(AutoReference(database))
return database
def testChangesWithRef(self):
"""
Change a collection object while it is referenced in
another collection, use AutoReference Manipulator so
the change is reflected.
"""
connection = Connection()
db = connection._test_dbrefs
#manipulators are required to keep references in sync
db.add_son_manipulator(NamespaceInjector())
db.add_son_manipulator(AutoReference(db))
db.drop_collection("books")
db.drop_collection("groups")
books = db.books
groups = db.groups
# Create book-1
bk1 = {}
bk1["name"] = "Test-A"
bk1["created-time"] = time.time()
books.save(bk1)
# Create group-1
grp1 = {}
grp1["name"] = "FirstGroup"
grp1["books"] = [bk1]
groups.save(grp1)
#Ensure that we have only 1 instance in each books/groups
found = [f for f in books.find({"name": "Test-A"})]
self.assertTrue(len(found) == 1)
bk1_id = found[0]["_id"]
found = [f for f in groups.find({"name": "FirstGroup"})]
self.assertTrue(len(found) == 1)
# Verify that we saved 'Test-A' under groups
self.assertTrue(len(found[0]["books"]) == 1)
self.assertTrue(found[0]["books"][0]["name"] == 'Test-A')
self.assertTrue(found[0]["books"][0]["_id"] == bk1_id)
# lookup 'Test-A', save it (no-modifications), ensure we still have only
# 1 instance of 'Test-A'
b = books.find_one({"name":"Test-A"})
books.save(b)
found = [f for f in books.find({"name": "Test-A"})]
self.assertTrue(len(found) == 1)
# lookup 'Test-A' and modify it and save it, ensure we only have
# 1 instance of 'Test-A'
b = books.find_one({"name":"Test-A"})
b["newEntry"] = "newValue"
b["modified_time"] = time.time()
books.save(b)
found = [f for f in books.find({"name": "Test-A"})]
self.assertTrue(len(found) == 1)
# Ensure _id didn't change after our modification
self.assertTrue(found[0]["_id"] == bk1_id)
found = [f for f in groups.find({"name": "FirstGroup"})]
self.assertTrue(len(found) == 1)
self.assertTrue(found[0]["name"] == "FirstGroup")
self.assertTrue(found[0]["books"][0]["_id"] == bk1_id)
self.assertTrue(found[0]["books"][0].has_key("newEntry"))
self.assertTrue(found[0]["books"][0]["newEntry"] == "newValue")
