class TestMonitorRemovesRecoveringMember(unittest.TestCase):
# Members in STARTUP2 or RECOVERING states are shown in the primary's
# isMaster response, but aren't secondaries and shouldn't be read from.
# Verify that if a secondary goes into RECOVERING mode, the Monitor removes
# it from the set of readers.
def setUp(self):
members = [{}, {'priority': 0}, {'priority': 0}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{},
{'priority': 0},
{'arbiterOnly': True},
{'priority': 0, 'hidden': True},
{'priority': 0, 'slaveDelay': 5}
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestMonitorRemovesRecoveringMember(unittest.TestCase):
# Members in STARTUP2 or RECOVERING states are shown in the primary's
# isMaster response, but aren't secondaries and shouldn't be read from.
# Verify that if a secondary goes into RECOVERING mode, the Monitor removes
# it from the set of readers.
def setUp(self):
members = [{}, {'priority': 0}, {'priority': 0}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries,
mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary),
mode)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{}, {
'priority': 0
}, {
'arbiterOnly': True
}, {
'priority': 0,
'hidden': True
}, {
'priority': 0,
'slaveDelay': 5
}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def test_sasl_plain(self):
client = MongoClient(SASL_HOST, SASL_PORT)
self.assertTrue(
client.ldap.authenticate(SASL_USER, SASL_PASS, SASL_DB, 'PLAIN'))
client.ldap.test.find_one()
uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;'
'authSource=%s' % (quote_plus(SASL_USER), quote_plus(SASL_PASS),
SASL_HOST, SASL_PORT, SASL_DB))
client = MongoClient(uri)
client.ldap.test.find_one()
set_name = client.admin.command('ismaster').get('setName')
if set_name:
client = MongoReplicaSetClient(SASL_HOST,
port=SASL_PORT,
replicaSet=set_name)
self.assertTrue(
client.ldap.authenticate(SASL_USER, SASL_PASS, SASL_DB,
'PLAIN'))
client.ldap.test.find_one()
uri = ('mongodb://%s:%s@%s:%d/?authMechanism=PLAIN;'
'authSource=%s;replicaSet=%s' %
(quote_plus(SASL_USER), quote_plus(SASL_PASS), SASL_HOST,
SASL_PORT, SASL_DB, str(set_name)))
client = MongoReplicaSetClient(uri)
client.ldap.test.find_one()
def init(self):
logger.info('[Mongodb-Notification-Broker] Initialization of '
'mongodb_notification_broker module')
self._set_mongodb_url()
logger.debug('[Mongodb-Notification-Broker] Mongodb connect url: %s'
% self.mongodb_url)
# In case notification broker process down occasionally, the self.conn
# object must be dropped cleanly in Broker daemon.
self.do_stop()
try:
if not self.high_availability:
self.conn = MongoClient(self.mongodb_url)
else:
self.conn = MongoReplicaSetClient(self.mongodb_url)
except ConnectionFailure:
logger.warn('[Mongodb-Notification-Broker] Can not make connection '
' with MongoDB')
raise
except (InvalidURI, ConfigurationError):
logger.warn('[Mongodb-Notification-Broker] Mongodb connect url '
'error')
logger.warn('[Mongodb-Notification-Broker] Mongodb connect url: %s'
% self.mongodb_url)
raise
self._get_collections()
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def test_scram_sha1(self):
client = MongoClient(host, port)
self.assertTrue(
client.pymongo_test.authenticate('user',
'pass',
mechanism='SCRAM-SHA-1'))
client.pymongo_test.command('dbstats')
client = MongoClient('mongodb://*****:*****@%s:%d/pymongo_test'
'?authMechanism=SCRAM-SHA-1' % (host, port))
client.pymongo_test.command('dbstats')
if self.set_name:
client = MongoReplicaSetClient(
'mongodb://*****:*****@%s:%d/pymongo_test?authMechanism=SCRAM-SHA-1'
'&replicaSet=%s' % (host, port, self.set_name))
client = MongoReplicaSetClient(uri)
client.pymongo_test.command('dbstats')
client.read_preference = ReadPreference.SECONDARY
client.pymongo_test.command('dbstats')
def test_init_disconnected_with_auth(self):
c = self._get_client()
auth_context.client.admin.add_user("admin", "pass")
c.admin.authenticate("admin", "pass")
try:
c.pymongo_test.add_user("user",
"pass",
roles=['readWrite', 'userAdmin'])
# Auth with lazy connection.
host = one(self.hosts)
uri = "mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" % (
host[0], host[1], self.name)
authenticated_client = MongoReplicaSetClient(uri, _connect=False)
authenticated_client.pymongo_test.test.find_one()
# Wrong password.
bad_uri = (
"mongodb://*****:*****@%s:%d/pymongo_test?replicaSet=%s" %
(host[0], host[1], self.name))
bad_client = MongoReplicaSetClient(bad_uri, _connect=False)
self.assertRaises(OperationFailure,
bad_client.pymongo_test.test.find_one)
finally:
# Clean up.
remove_all_users(c.pymongo_test)
c.admin.remove_user('admin')
def test_gssapi_simple(self):
client = MongoClient(GSSAPI_HOST, GSSAPI_PORT)
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
# Just test that we can run a simple command.
self.assertTrue(client.database_names())
uri = ('mongodb://%s@%s:%d/?authMechanism='
'GSSAPI' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT))
client = MongoClient(uri)
self.assertTrue(client.database_names())
set_name = client.admin.command('ismaster').get('setName')
if set_name:
client = MongoReplicaSetClient(GSSAPI_HOST,
port=GSSAPI_PORT,
replicaSet=set_name)
self.assertTrue(
client.test.authenticate(PRINCIPAL, mechanism='GSSAPI'))
self.assertTrue(client.database_names())
uri = ('mongodb://%s@%s:%d/?authMechanism=GSSAPI;replicaSet'
'=%s' % (quote_plus(PRINCIPAL), GSSAPI_HOST, GSSAPI_PORT,
str(set_name)))
client = MongoReplicaSetClient(uri)
self.assertTrue(client.database_names())
class Connection():
def __init__(self):
self.connection = None
def connect(self,hosts="localhost", replicaSet="", readPreference=ReadPreference.SECONDARY_PREFERRED):
# if the replicaset is empty, we use a normal connection, if not, we use a MongoReplicaSetClient
# in wich case we set the read preference
# by default, SECONDARY_PREFERRED since we don't big have consistancy issues. Fail safe when the primary/secondary goes down.
if replicaSet == "":
print("Using mongodb host " + hosts + " for read/writes.")
self.connection = MongoConnection(hosts)
else:
print("Using mongodb hosts " + hosts + " with replica set " + replicaSet + " and read pref " + str(readPreference))
self.connection = MongoReplicaSetClient(hosts, replicaSet=replicaSet, read_preference=readPreference)
print("Replica set:")
print("PRIMARY: " + str(self.connection.primary))
print("SECONDARIES: " + str(self.connection.secondaries))
print("ARBITERS: " + str(self.connection.arbiters))
def close(self):
self.connection.close()
self.connection = None
# \return the requested db setting the readPreference if set.
def getDatabase(self, dbname, readPreference=None):
db = self.connection[dbname]
if readPreference != None:
db.read_preference = readPreference
return db
def getDB(name):
if name=='mongo':
conn=MongoReplicaSetClient('dbserver-1:27017,dbserver-2:27017,dbserver-3:27017', replicaSet='ats_rs')
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
return conn.brauth
elif name=='redis':
return redis.Redis(connection_pool = redis.ConnectionPool(host='192.168.7.210', port=6379, db=6))
else:
return None
def __init__(
self, standalones, members, mongoses, ismaster_hosts=None,
*args, **kwargs
):
MockClientBase.__init__(
self, standalones, members, mongoses, ismaster_hosts)
kwargs['_pool_class'] = my_partial(MockPool, self)
MongoReplicaSetClient.__init__(self, *args, **kwargs)
def __init__(self, settings, **kw):
if MongoClient is None:
raise NotConfigured('%s is missing pymongo or gridfs module.' %
self.__class__.__name__)
if (2, 4, 0) > mongo_version:
version = '.'.join('%s' % v for v in mongo_version)
raise NotConfigured(
'%s requires pymongo version >= 2.4 but got %s' %
(self.__class__.__name__, version))
super(MongodbCacheStorage, self).__init__(settings)
self.sharded = settings.getbool('HTTPCACHE_SHARDED', False)
kwargs = get_database(settings)
kwargs.update(kw)
db = kwargs.pop('db')
user = kwargs.pop('user', None)
password = kwargs.pop('password', None)
if 'replicaSet' in kwargs:
client = MongoReplicaSetClient(**kwargs)
else:
client = MongoClient(**kwargs)
# do not override a database passed in a 'mongodb://' URI string
try:
self.db = client.get_default_database()
except ConfigurationError:
self.db = client[db]
except TypeError:
# get_default_database() only since pymongo 2.6, but
# pymongo>2.5.2 only works with mongodb > ~2.4.3
# fall back to parsing uri string in this edge-case :(
if 'mongodb://' in kwargs.get('host'):
try:
from urlparse import urlparse
loc = urlparse(uri).path.strip('/')
if not loc:
self.db = client[db]
except (ImportError, Exception):
client.connection.close()
raise NotConfigured('%s could not reliably detect if \
there was a database passed in URI string. Please install \
urlparse to fix this, or use host:port arguments instead.'
% self.__class__.__name__)
else:
self.db = client[db]
if user is not None and password is not None:
self.db.authenticate(user, password)
logger.debug(
"Backend %(storage)s connected to %(host)s:%(port)s, using database '%(db)s'"
% {
'storage': self.__class__.__name__,
'host': client.host,
'port': client.port,
'db': db
})
self.fs = {}
class TestReplicaSetRequest(unittest.TestCase):
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
primary_pool = self.c._MongoReplicaSetClient__members[primary].pool
secondary_pool = self.c._MongoReplicaSetClient__members[secondary].pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
patience_seconds = 60
for _ in range(patience_seconds):
sleep(1)
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
else:
self.fail("Problem with test: No new primary after %s seconds" %
patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestReplicaSetRequest(unittest.TestCase):
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0], replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
primary_pool = self.c._MongoReplicaSetClient__members[primary].pool
secondary_pool = self.c._MongoReplicaSetClient__members[secondary].pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
patience_seconds = 60
for _ in range(patience_seconds):
sleep(1)
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
else:
self.fail("Problem with test: No new primary after %s seconds"
% patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def getDB(name):
if name == 'mongo':
conn = MongoReplicaSetClient(
'dbserver-1:27017,dbserver-2:27017,dbserver-3:27017',
replicaSet='ats_rs')
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
return conn.brauth
elif name == 'redis':
return redis.Redis(connection_pool=redis.ConnectionPool(
host='192.168.7.210', port=6379, db=6))
else:
return None
def _mongo(self, instances, force=False):
if not hasattr(AwsMongoBackup, 'mongo') or force:
mongo_rs_str = ','.join([x.public_dns_name for x in instances])
self.logger.debug("connecting to mongo URI %s" % mongo_rs_str)
self.mongo = MongoReplicaSetClient(mongo_rs_str,
replicaSet=self.replicaset)
if self.mongo.alive():
return self.mongo
else:
self.mongo = None
return None
def __init__(self,
standalones,
members,
mongoses,
ismaster_hosts=None,
*args,
**kwargs):
MockClientBase.__init__(self, standalones, members, mongoses,
ismaster_hosts)
kwargs['_pool_class'] = my_partial(MockPool, self)
MongoReplicaSetClient.__init__(self, *args, **kwargs)
def connect(self,hosts="localhost", replicaSet="", readPreference=ReadPreference.SECONDARY_PREFERRED):
# if the replicaset is empty, we use a normal connection, if not, we use a MongoReplicaSetClient
# in wich case we set the read preference
# by default, SECONDARY_PREFERRED since we don't big have consistancy issues. Fail safe when the primary/secondary goes down.
if replicaSet == "":
print("Using mongodb host " + hosts + " for read/writes.")
self.connection = MongoConnection(hosts)
else:
print("Using mongodb hosts " + hosts + " with replica set " + replicaSet + " and read pref " + str(readPreference))
self.connection = MongoReplicaSetClient(hosts, replicaSet=replicaSet, read_preference=readPreference)
print("Replica set:")
print("PRIMARY: " + str(self.connection.primary))
print("SECONDARIES: " + str(self.connection.secondaries))
print("ARBITERS: " + str(self.connection.arbiters))
def __init__(self, settings, **kw):
if MongoClient is None:
raise NotConfigured('%s is missing pymongo or gridfs module.' %
self.__class__.__name__)
if (2,4,0) > mongo_version:
version = '.'.join('%s'% v for v in mongo_version)
raise NotConfigured('%s requires pymongo version >= 2.4 but got %s' %
(self.__class__.__name__, version))
self.expire = settings.getint('HTTPCACHE_EXPIRATION_SECS')
self.sharded = settings.getbool('HTTPCACHE_SHARDED', False)
kwargs = get_database(settings)
kwargs.update(kw)
db = kwargs.pop('db')
user = kwargs.pop('user', None)
password = kwargs.pop('password', None)
if 'replicaSet' in kwargs:
client = MongoReplicaSetClient(**kwargs)
else:
client = MongoClient(**kwargs)
# do not override a database passed in a 'mongodb://' URI string
try:
self.db = client.get_default_database()
except ConfigurationError:
self.db = client[db]
except TypeError:
# get_default_database() only since pymongo 2.6, but
# pymongo>2.5.2 only works with mongodb > ~2.4.3
# fall back to parsing uri string in this edge-case :(
if 'mongodb://' in kwargs.get('host'):
try:
from urlparse import urlparse
loc = urlparse(uri).path.strip('/')
if not loc:
self.db = client[db]
except (ImportError, Exception):
client.connection.close()
raise NotConfigured('%s could not reliably detect if \
there was a database passed in URI string. Please install \
urlparse to fix this, or use host:port arguments instead.' %
self.__class__.__name__)
else:
self.db = client[db]
if user is not None and password is not None:
self.db.authenticate(user, password)
log.msg('%s connected to %s:%s, using database \'%s\'' %
(self.__class__.__name__, client.host, client.port, db),
level=log.DEBUG)
self.fs = {}
def test_passive_and_hidden(self):
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def _establish_connection(cls, *args, **kwargs):
for i in range(MAX_ATTEMPTS - 1):
try:
if 'replicaSet' in kwargs:
return MongoReplicaSetClient(*args, **kwargs)
else:
return MongoClient(*args, **kwargs)
except ConnectionFailure as e:
logging.error(
'Fail to connect to %s, %s [%s]', args, kwargs, e)
sleep(CONNECT_SLEEP_TIME)
if 'replicaSet' in kwargs:
return MongoReplicaSetClient(*args, **kwargs)
else:
return MongoClient(*args, **kwargs)
def test_stepdown_triggers_refresh(self):
c_find_one = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(one(c_find_one.secondaries),
_partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
sleep(1)
# Trigger a refresh
self.assertRaises(AutoReconnect, c_find_one.test.test.find_one)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
sleep(1)
# We've detected the stepdown
self.assertTrue(not c_find_one.primary
or primary != _partition_node(c_find_one.primary))
def getClient(conf, dbname):
'''
conf : json conf objet
conf=load_conf(configfile)
db = getClient(conf)['ceph']
collection =db['cluster']
cursor=collection.find_one()
Return a connexion to database specified in conf file
take care with authentication
'''
mongodb_host = conf.get("mongodb_host", "127.0.0.1")
mongodb_port = conf.get("mongodb_port", "27017")
mongodb_URL = "mongodb://" + mongodb_host + ":" + str(mongodb_port)
#mongodb replication
is_mongo_replicat = conf.get("is_mongo_replicat", 0)
mongodb_set = "'" + conf.get("mongodb_set", "") + "'"
mongodb_replicaSet = conf.get("mongodb_replicaSet", None)
mongodb_read_preference = conf.get("mongodb_read_preference", None)
if is_mongo_replicat == 1:
client = MongoReplicaSetClient(
eval(mongodb_set),
replicaSet=mongodb_replicaSet,
read_preference=eval(mongodb_read_preference))
else:
#if not replicated
client = MongoClient(mongodb_URL)
# mongo db authentication
is_mongo_authenticate = conf.get("is_mongo_authenticate", 0)
mongodb_user = conf.get("mongodb_user", "ceph")
mongodb_passwd = conf.get("mongodb_passwd", "empty")
if is_mongo_authenticate == 1:
client[dbname].authenticate(mongodb_user, mongodb_passwd)
return client[dbname]
def test_read_with_failover(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
def iter_cursor(cursor):
for _ in cursor:
pass
return True
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
db.test.insert([{'foo': i} for i in xrange(10)], w=w)
self.assertEqual(10, db.test.count())
db.read_preference = SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
cursor.next()
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
def mongoDBHelper(dbName, readOnly=False):
global connection
if len(app_config.MONGODB_CONFIG) == 0:
connection = None
return
try:
connection
except NameError:
if app_config.MONGODB_CONFIG['RS'] != "":
connection = MongoReplicaSetClient(
app_config.MONGODB_CONFIG['URL'],
replicaSet=app_config.MONGODB_CONFIG['RS'])
else:
connection = MongoClient(app_config.MONGODB_CONFIG['URL'])
db = connection[dbName]
if 'AUTH' in app_config.MONGODB_CONFIG:
if 'true' == app_config.MONGODB_CONFIG['AUTH']:
db.authenticate(app_config.MONGODB_CONFIG['USER'],
app_config.MONGODB_CONFIG['PASSWORD'])
else:
pass
# if readOnly:
# db.read_preference = ReadPreference.SECONDARY_PREFERRED
# else:
# db.read_preference = ReadPreference.PRIMARY_PREFERRED
return db
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = [
MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets,
read_preference=SECONDARY) for _ in xrange(2)
]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
self.assertEqual(one(c.secondaries), _partition_node(secondary))
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
self.assertRaises(AutoReconnect, self.c_count.test.test.count)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
sleep(1)
for c in self.c_find_one, self.c_count:
self.assertFalse(c.secondaries)
self.assertEqual(_partition_node(primary), c.primary)
def get_mongo(self):
"""Return the mongo connection from the environment."""
if self.mongo_db:
return self.mongo_db
if len(self.uri_params['nodelist']) > 1 and MongoReplicaSetClient:
nodes = ','.join(
['%s:%d' % n for n in self.uri_params['nodelist']])
client = MongoReplicaSetClient(nodes, **self.uri_params['options'])
else:
client = MongoClient(self.uri_params['nodelist'][0][0],
self.uri_params['nodelist'][0][1],
**self.uri_params['options'])
db = client[self.uri_params['database']]
if self.uri_params['username']:
if not db.authenticate(self.uri_params['username'],
self.uri_params['password']):
raise PluginError('Cannot authenticate to MongoDB for '
'user: %s' % self.uri_params['username'])
if self.post_create:
db = self.post_create(db)
self.mongo_db = db
return self.mongo_db
def test_rs_client_attrs(self):
pymongo_rs_client_only = set(['pool_class']).union(pymongo_client_only)
sync_cx_rs = MongoReplicaSetClient(env.rs_uri)
self.assertEqual(
attrs(sync_cx_rs) - pymongo_rs_client_only,
attrs(self.rsc) - motor_client_only)
def getClient(conf):
'''
Return a connexion to database specified in conf file
take care with autentication
'''
mongodb_host = conf.get("mongodb_host", "127.0.0.1")
mongodb_port = conf.get("mongodb_port", "27017")
mongodb_URL = "mongodb://"+str(mongodb_host)+":"+str(mongodb_port)
#mongodb replication
is_mongo_replicat = conf.get("is_mongo_replicat", 0)
mongodb_set = "'"+conf.get("mongodb_set","")+"'"
mongodb_replicaSet =conf.get("mongodb_replicaSet",None)
mongodb_read_preference = conf.get("mongodb_read_preference",None)
cluster = conf.get("cluster", "ceph")
if is_mongo_replicat == 1:
client = MongoReplicaSetClient(eval(mongodb_set), replicaSet=mongodb_replicaSet, read_preference=eval(mongodb_read_preference))
else:
#if not replicated
client = MongoClient(mongodb_URL)
# mongo db authentication
is_mongo_authenticate = conf.get("is_mongo_authenticate", 0)
mongodb_user = conf.get("mongodb_user", "ceph")
mongodb_passwd = conf.get("mongodb_passwd", "empty")
if is_mongo_authenticate == 1:
client[cluster].authenticate(mongodb_user,mongodb_passwd)
return client
def startapp(app):
# do not display server and version information
# https://stackoverflow.com/a/55209796/2249798
# https://stackoverflow.com/a/54947461/2249798
cherrypy.__version__ = ""
cherrypy.config.update({"response.headers.server": ""})
cherrypy._cperror._HTTPErrorTemplate = \
cherrypy._cperror._HTTPErrorTemplate.replace(
"Powered by <a href=\"http://www.cherrypy.org\">"
+ "CherryPy %(version)s</a>\n",
"%(version)s"
)
if config["mongodb"].getboolean('replicaset'):
print("Connecting to MongoDB ReplicaSet: %s" %
config["mongodb"]["url"])
dbe = MongoReplicaSetClient(config["mongodb"]["url"],
w="majority",
maxPoolSize=16,
socketTimeoutMS=60000,
connectTimeoutMS=30000,
waitQueueTimeoutMS=60000,
waitQueueMultiple=64)
atexit.register(dbe.close)
else:
print("Connecting to MongoDB: %s" % config["mongodb"]["url"])
dbe = MongoClient(config["mongodb"]["url"],
maxPoolSize=16,
socketTimeoutMS=60000,
connectTimeoutMS=30000,
waitQueueTimeoutMS=60000,
waitQueueMultiple=64)
dbm = dbe[config["mongodb"]["dbname"]]
return cherrypy.Application(app(dbm), script_name=None, config=None)
def test_cert_ssl_validation_hostname_fail(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests eg:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
if not CERT_SSL:
raise SkipTest("No mongod available over SSL with certs")
client = MongoClient(host, port, ssl=True, ssl_certfile=CLIENT_PEM)
response = client.admin.command('ismaster')
try:
MongoClient(pair,
ssl=True,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM)
self.fail("Invalid hostname should have failed")
except CertificateError:
pass
if 'setName' in response:
try:
MongoReplicaSetClient(pair,
replicaSet=response['setName'],
w=len(response['hosts']),
ssl=True,
ssl_certfile=CLIENT_PEM,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=CA_PEM)
self.fail("Invalid hostname should have failed")
except CertificateError:
pass
def test_cert_ssl_implicitly_set(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests eg:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
#
# Also requires an /etc/hosts entry where "server" is resolvable
if not CERT_SSL:
raise SkipTest("No mongod available over SSL with certs")
client = MongoClient(host, port, ssl_certfile=CLIENT_PEM)
response = client.admin.command('ismaster')
if 'setName' in response:
client = MongoReplicaSetClient(pair,
replicaSet=response['setName'],
w=len(response['hosts']),
ssl_certfile=CLIENT_PEM)
db = client.pymongo_ssl_test
db.test.drop()
self.assertTrue(db.test.insert({'ssl': True}))
self.assertTrue(db.test.find_one()['ssl'])
client.drop_database('pymongo_ssl_test')
def test_gssapi_threaded(self):
# Use auto_start_request=True to make sure each thread
# uses a different socket.
client = MongoClient(GSSAPI_HOST, auto_start_request=True)
self.assertTrue(client.test.authenticate(PRINCIPAL,
mechanism='GSSAPI'))
threads = []
for _ in xrange(4):
threads.append(AutoAuthenticateThread(client.foo))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.assertTrue(thread.success)
set_name = client.admin.command('ismaster').get('setName')
if set_name:
preference = ReadPreference.SECONDARY
client = MongoReplicaSetClient(GSSAPI_HOST,
replicaSet=set_name,
read_preference=preference)
self.assertTrue(
client.test.authenticate(PRINCIPAL, mechanism='GSSAPI'))
self.assertTrue(client.foo.command('dbstats'))
threads = []
for _ in xrange(4):
threads.append(AutoAuthenticateThread(client.foo))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.assertTrue(thread.success)
def QueryFunc(addr, insertnum):
print "query process"
contentsrc = "objids_0113.txt"
f = open(contentsrc, "r")
print "h0"
mm = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
print "hi"
fsize = os.path.getsize(contentsrc)
numtopicids = fsize / 24
addrnew = addr + ":29002"
print "h1"
client = MongoReplicaSetClient(addrnew, replicaSet="rs0")
db = client.topicdb
collection = db.topiccol
count = 0
totinsertnum = insertnum * 3
print "h2"
while count < totinsertnum:
topicidstart = random.randint(0, numtopicids - 1) * 24
topicidsend = topicidstart + 24
rand_topicid = mm[topicidstart:topicidsend]
try:
#collection.find_one({"_id":ObjectId(rand_topicid)}, read_preference=ReadPreference.SECONDARY_PREFERRED)
collection.find_one(
{"_id": ObjectId(rand_topicid)},
read_preference=ReadPreference.SECONDARY_PREFERRED)
#print "q id:", rand_topicid
#collection.find_one({"_id":ObjectId(rand_topicid)})
except pymongo.errors.AutoReconnect:
print "auto reconnect"
time.sleep(2)
continue
count += 1
mm.close()
def InsertFunc(addr, insertnum):
contentsrc = "cs.txt"
f = open(contentsrc, "r")
mm = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
fsize = os.path.getsize(contentsrc)
addrnew = addr + ":29002"
client = MongoReplicaSetClient(addrnew, replicaSet="rs0")
db = client.topicdb
collection = db.topiccol
count = 0
while count < insertnum:
post = GenerateTopicPost(mm, fsize)
try:
collection.insert(post)
except InvalidStringData:
continue
except pymongo.errors.AutoReconnect:
print "insert auto reconnect", time.ctime(time.time())
time.sleep(5)
print "sleep over", time.ctime(time.time())
continue
print "continue:", count
except Exception, e:
print e
count += 1
def test_writes_with_failover(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
def try_write():
for _ in xrange(30):
try:
db.test.insert({'bar': 'baz'})
return True
except AutoReconnect:
sleep(1)
return False
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
self.assertTrue(try_write())
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_secondary_failure(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def readers_changed():
for _ in xrange(20):
if c.secondaries != secondaries:
return True
sleep(1)
return False
killed = ha_tools.kill_secondary()
sleep(2 * MONITOR_INTERVAL)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
class TestReplicaSetAuth(unittest.TestCase):
def setUp(self):
members = [
{},
{'priority': 0},
{'priority': 0},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = MongoReplicaSetClient(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
self.c.admin.add_user('admin', 'adminpass')
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def test_auth_during_failover(self):
self.assertTrue(self.db.authenticate('user', 'userpass'))
self.assertTrue(self.db.foo.insert({'foo': 'bar'},
safe=True, w=3, wtimeout=1000))
self.db.logout()
self.assertRaises(OperationFailure, self.db.foo.find_one)
primary = '%s:%d' % self.c.primary
ha_tools.kill_members([primary], 2)
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# Make sure we can still authenticate
self.assertTrue(self.db.authenticate('user', 'userpass'))
# And still query.
self.db.read_preference = PRIMARY_PREFERRED
self.assertEqual('bar', self.db.foo.find_one()['foo'])
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = MongoClient(primary, use_greenlets=use_greenlets)
secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets)
rsc = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def _mongo(self, instances, force=False):
if not hasattr(AwsMongoBackup, 'mongo') or force:
mongo_rs_str = ','.join([x.public_dns_name for x in instances])
self.logger.debug("connecting to mongo URI %s" % mongo_rs_str)
self.mongo = MongoReplicaSetClient(
mongo_rs_str,
replicaSet=self.replicaset
)
if self.mongo.alive():
return self.mongo
else:
self.mongo = None
return None
def __getitem__(self, alias):
if alias in self._connections:
return self._connections[alias]
try:
conn = self.databases[alias]
except KeyError:
raise ConnectionDoesNotExist("The connection %s doesn't exist" % alias)
if type(conn['location']) == type([]):
conn['location'] = ','.join(conn['location'])
if 'replicaSet' in conn:
conn = MongoReplicaSetClient(conn['location'], replicaSet=conn['replicaSet'], safe=True, w=2, j=True)
conn.read_preferences = ReadPreference.PRIMARY_PREFERRED
conn.write_concerns = {'w': 2, 'j':True} # para sure
else:
# conn = MongoClient(conn['location'], safe=True, w=2, j=True)
conn = MongoClient(conn['location'], safe=True)
self._connections[alias] = ConnectionWrapper(conn, self.databases[alias]['name'], config=self.databases)
return self._connections[alias]
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def test_passive_and_hidden(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def init_mongodb_client(conf_item):
section = conf_item
servers = section.get('servers')
def build_node_desc(node_conf):
return '%s:%s' % (node_conf['host'], node_conf['port'])
if section.get('replica', False):
from pymongo import MongoReplicaSetClient
from pymongo import ReadPreference
client = MongoReplicaSetClient(','.join(map(build_node_desc, servers)), replicaSet=section.get('replName'))
pref = section.get('readPref', 'PRIMARY')
client.read_preference = getattr(ReadPreference, pref)
else:
from pymongo import MongoClient
s = servers[0]
client = MongoClient(s['host'], s['port'])
setattr(client, 'auth_list', set([]))
return client
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def setUp(self):
members = [
{},
{'priority': 0},
{'priority': 0},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = MongoReplicaSetClient(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
self.c.admin.add_user('admin', 'adminpass')
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
class AwsMongoBackup(object):
def __init__(self,
replicaset=None,
filters=None,
instance_ids=None,
ssh_opts=None,
dryrun=False,
region=None,
logger=None):
self.creation_time = datetime.utcnow().strftime("%m-%d-%Y %H:%M:%S")
if logger is not None:
self.logger = logger
if region is None:
region = 'us-east-1'
if AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY:
self.ec2 = ec2_connect_to_region(
region,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
else:
self.ec2 = ec2_connect_to_region(
region
)
if replicaset is None:
raise RuntimeError('replicaset must be provided.')
self.replicaset = replicaset
if filters:
self.instances = self._instances_via_filters(filters=filters)
elif instance_ids:
self.instances = self._instances_via_ids(instance_ids=instance_ids)
else:
raise RuntimeError('Either an API filter or a list of instance'
'IDs must be provided.')
self.logger.debug("found instances %s" % self.instances)
self.mongo = self._mongo(instances=self.instances)
self.logger.debug("connected to mongo %s" % self.mongo)
self.ssh_opts = ssh_opts
self.logger.debug("set ssh opts to %s" % self.ssh_opts)
self.dryrun = dryrun
def _instances_via_filters(self, filters=None):
if filters is None:
raise ValueError('filters must be a dict of valid EC2 API filters')
reservations = self.ec2.get_all_instances(
filters=filters
)
instances = []
for reservation in reservations:
instances.extend(reservation.instances)
return instances
def _instances_via_ids(self, instance_ids=None):
if instance_ids is None or type(instance_ids) is not 'list':
raise ValueError('instances must be provided in a list')
raise NotImplementedError("I'll come back to this later.")
def _mongo(self, instances, force=False):
if not hasattr(AwsMongoBackup, 'mongo') or force:
mongo_rs_str = ','.join([x.public_dns_name for x in instances])
self.logger.debug("connecting to mongo URI %s" % mongo_rs_str)
self.mongo = MongoReplicaSetClient(
mongo_rs_str,
replicaSet=self.replicaset
)
if self.mongo.alive():
return self.mongo
else:
self.mongo = None
return None
def _ssh(self, hostname, ssh_opts):
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(hostname=hostname, **ssh_opts)
self.logger.debug("connected via ssh to %s" % hostname)
return self.ssh
def test_replicaset(self):
test_result = True
err_str = ''
optime_dates = []
rs_states = {}
hidden_members = []
rs_status = self.mongo.admin.command('replSetGetStatus')
rs_member_hosts = [z[0] for z in self.mongo.hosts]
for rs_member in rs_status['members']:
if rs_member['name'].split(':')[0] not in rs_member_hosts:
hidden_members.append((
rs_member['name'].split(':')[0],
int(rs_member['name'].split(':')[1])
))
try:
rs_states[rs_member['state']] += 1
except KeyError:
rs_states[rs_member['state']] = 1
if rs_member['state'] not in [1, 2, 7]:
# primary, secondary, arbiter
err_str = "RS member {rs_member} has a state of {state}, "\
"please check RS integrity and try again."\
.format(
rs_member=rs_member['name'],
state=rs_member['stateStr']
)
test_result = False
return (test_result, err_str)
self.logger.debug("member %s passed state" % rs_member['name'])
if rs_member.get('health', 1) != 1:
err_str = "RS member {rs_member} is marked as unhealthy, "\
"please check RS integrity and try again."\
.format(rs_member=rs_member['name'])
test_result = False
return (test_result, err_str)
self.logger.debug("member %s passed health" % rs_member['name'])
if rs_member.get('pingMs', 0) > 10:
err_str = "ping time for RS member {rs_member} is larger than"\
"10ms. Please check network connectivity and try again."\
.format(rs_member=rs_member['name'])
test_result = False
return (test_result, err_str)
self.logger.debug("member %s passed pingMs" % rs_member['name'])
optime_dates.append(rs_member['optimeDate'])
self.hidden_members = hidden_members
if (max(optime_dates) - min(optime_dates)).total_seconds() > 5:
err_str = "optimeDates is over 5 seconds, there is too much "\
"replication lag to continue."
test_result = False
return (test_result, err_str)
self.logger.debug("passed replication lag test")
if len(self.mongo.secondaries) + len(hidden_members) < 2:
err_str = "There needs to be at least two secondaries or a hidden"\
" member available to do backups. Please check RS integrity "\
"and try again."
test_result = False
return (test_result, err_str)
self.logger.debug("mongo secondaries test passed")
if rs_states[1] != 1:
err_str = "There needs to be one and exactly one mongo primary to"\
" do backups. Please check RS integrity and try again."
test_result = False
return (test_result, err_str)
self.logger.debug("passed primary mongo test")
return (test_result, err_str)
def choose_member(self):
if self.hidden_members:
return self.hidden_members.pop()
else:
return self.mongo.secondaries.pop()
def backup(self):
# Test that the replica set is in a good state to perform backups
test_result, err_str = self.test_replicaset()
if test_result is False:
raise RuntimeError(err_str)
# Choose a member from which to back up
backup_member = self.choose_member()
self._ssh(backup_member[0], self.ssh_opts)
# Get the instance ID
stdin, stdout, stderr = self.ssh.exec_command(
'/usr/bin/curl http://169.254.169.254/latest/meta-data/instance-id'
)
instance_id = stdout.readline().rstrip()
self.logger.debug("Working on instance %s" % instance_id)
reservation = self.ec2.get_all_instances(instance_ids=[instance_id, ])
instance = reservation[0].instances[0]
self.logger.debug("got boto ec2 instance %s" % instance)
# Connect to the backup target directly
backup_member_mongo = MongoClient(
host=backup_member[0],
port=backup_member[1]
)
self.logger.debug("connected to mongo target %s" % backup_member_mongo)
freeze_rs = True
if backup_member_mongo.admin.command('isMaster').get('hidden', False):
# This member is hidden so we can safely take backups without
# doing any other maintenance work
freeze_rs = False
# Find what volume database data is on
cfg = backup_member_mongo.admin.command('getCmdLineOpts')
cfg_data_volume = cfg['parsed']['dbpath']
self.logger.debug("found parsed dbpath of %s" % cfg_data_volume)
stdin, stdout, stderr = self.ssh.exec_command(
'/usr/bin/sudo /bin/df {cfg_data_volume} | '
'/bin/grep -v "Filesystem"'
.format(cfg_data_volume=cfg_data_volume)
)
mount_info = stdout.readline().rstrip()
mount_info = mount_info.split(' ')[0]
self.logger.debug("working on mount %s" % mount_info)
# Find the matching EBS volume for this mount point
volumes = self.ec2.get_all_volumes(
filters={'attachment.instance-id': instance_id}
)
data_volume = None
for volume in volumes:
# There's a strange thing that happens, /dev/sdh1 can magically
# become /dev/xdh1 at boot time on instances. Check for both.
volume_mount_point = volume.attach_data.device
if volume_mount_point == mount_info or \
volume_mount_point.replace('sd', 'xvd') == mount_info:
data_volume = volume
if data_volume is None:
raise RuntimeError("Couldn't find EBS data volume!")
self.logger.debug("found data volume %s" % data_volume)
# Remove the member from the replicaset (mark as hidden)
if freeze_rs:
# Can probably use replSetMaintenance here but not available
# in my testing version
if self.dryrun:
self.logger.debug("Would have frozen replicaset")
else:
self.logger.debug('Freezing replicaset')
backup_member_mongo.admin.command({'replSetFreeze': 86400})
else:
self.logger.debug(
"skipping replicaset freeze, %s is a hidden member"
% backup_member[0]
)
# Fsynclock mongo
if self.dryrun:
self.logger.debug(
"Would have fsynclocked {backup_member}"
.format(backup_member=backup_member)
)
else:
self.logger.debug(
"fsync/locking {backup_member}"
.format(backup_member=backup_member)
)
backup_member_mongo.fsync(lock=True)
if self.dryrun:
self.logger.debug(
"Would have created snapshot of volume {volume}"
.format(volume=data_volume)
)
self.current_snapshot = None
else:
self.logger.debug("creating snapshot of %s" % data_volume)
snapshot = data_volume.create_snapshot(
description="mongobackup {date} {replicaset}".format(
date=self.creation_time,
replicaset=self.replicaset)
)
self.current_snapshot = snapshot.id
tags = {
'replicaset': self.replicaset,
'sourcehost': backup_member[0],
'creation_time': self.creation_time
}
self.logger.debug(
"adding tags %s to snapshot %s" % (tags, snapshot)
)
self.ec2.create_tags(
resource_ids=[snapshot.id, ],
tags=tags
)
# Unlock mongo
if self.dryrun:
self.logger.debug("Would have unlocked mongo")
else:
if freeze_rs:
self.logger.debug('unfreezing replicaset')
backup_member_mongo.admin.command({'replSetFreeze': 0})
self.logger.debug(
"unlocking {backup_member}"
.format(backup_member=backup_member)
)
backup_member_mongo.unlock()
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0], replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def __new__(mcs, name, bases, attrs):
new_class = super(ModelBase,
mcs).__new__(mcs, name, bases, attrs)
parents = [b for b in bases if isinstance(b, ModelBase)]
if not parents:
# If this isn't a subclass of Model, don't do anything special.
return new_class
# Processing Model metadata.
try:
meta = getattr(new_class, 'Meta')
except AttributeError:
meta = None
else:
# Won't need the original metadata container anymore.
delattr(new_class, 'Meta')
options = _Options(meta)
options.collection = options.collection or to_underscore(name)
if options.interface:
new_class._meta = None
new_class.database = None
new_class.collection = DummyCollection
return new_class
if not (options.host and options.port and options.database):
raise Exception(
'Model %r improperly configured: %s %s %s' % (
name, options.host, options.port, options.database))
# Checking connection / client pool for an existing connection / client.
pool_key = options.host,options.port
if options.replica_set_name:
logging.debug("Using replica_set_name=%s as database pool key." % options.replica_set_name)
pool_key = options.replica_set_name
if pool_key in mcs._connections:
client = mcs._connections[pool_key]
logging.debug("Got database client from pool for pool_key=%s" % (pool_key,))
else:
logging.debug("Creating new database client for pool_key=%s" % (pool_key,))
if options.replica_set_name:
logging.debug("Setting up a replica set client...")
client = MongoReplicaSetClient(options.replica_set_uri, replicaSet=options.replica_set_name)
client.read_preference = ReadPreference.SECONDARY_PREFERRED
else:
logging.debug("Setting up a normal client...")
client = MongoClient(options.host, options.port)
mcs._connections[pool_key] = client
new_class._meta = options
new_class.connection = client
new_class.database = client[options.database]
if options.username and options.password:
new_class.database.authenticate(options.username, options.password)
new_class.collection = options.collection_class(
new_class.database, options.collection, document_class=new_class)
if options.auto_index:
new_class.auto_index() # Generating required indices.
return new_class
def __init__(self, *args, **kwargs):
# Specifying that it should run both the inits
MongoKitConnection.__init__(self, *args, **kwargs)
PymongoReplicaSetConnection.__init__(self, *args, **kwargs)
def getMongoDB(self):
conn=MongoReplicaSetClient(Config['ReplicaSetServer'], replicaSet=Config['ReplicaSetName'])
conn.read_preference = ReadPreference.SECONDARY_PREFERRED
return conn.bugreporter
class TestReadPreference(unittest.TestCase):
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def set_ping_time(self, host, ping_time_seconds):
Member._host_to_ping_time[host] = ping_time_seconds
def clear_ping_times(self):
Member._host_to_ping_time.clear()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll(
[primary, secondary, other_secondary],
NEAREST, secondary_acceptable_latency_ms=1000*1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(MongoClient(
unpartition_node(primary), use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED
).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(MongoClient(
unpartition_node(primary), use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED
).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_pinning(self):
# To make the code terser, copy modes into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=True)
# Verify that changing the mode unpins the member. We'll try it for
# every relevant change of mode.
for mode0, mode1 in itertools.permutations(
(PRIMARY, SECONDARY, SECONDARY_PREFERRED, NEAREST), 2
):
# Try reading and then changing modes and reading again, see if we
# read from a different host
for _ in range(1000):
# pin to this host
host = utils.read_from_which_host(c, mode0)
# unpin?
new_host = utils.read_from_which_host(c, mode1)
if host != new_host:
# Reading with a different mode unpinned, hooray!
break
else:
self.fail(
"Changing from mode %s to mode %s never unpinned" % (
modes[mode0], modes[mode1]))
# Now verify changing the tag_sets unpins the member.
tags0 = [{'a': 'a'}, {}]
tags1 = [{'a': 'x'}, {}]
for _ in range(1000):
host = utils.read_from_which_host(c, NEAREST, tags0)
new_host = utils.read_from_which_host(c, NEAREST, tags1)
if host != new_host:
break
else:
self.fail(
"Changing from tags %s to tags %s never unpinned" % (
tags0, tags1))
# Finally, verify changing the secondary_acceptable_latency_ms unpins
# the member.
for _ in range(1000):
host = utils.read_from_which_host(c, SECONDARY, None, 15)
new_host = utils.read_from_which_host(c, SECONDARY, None, 20)
if host != new_host:
break
else:
self.fail(
"Changing secondary_acceptable_latency_ms from 15 to 20"
" never unpinned")
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
self.clear_ping_times()
members = ""
database = "xxx"
admin_user = "******"
admin_passwd = "xxxxxxxx"
replicaset = "rs0"
mongod_port = 27017
mongos_port = 27018
servers = [ "docker-1:27017","docker-2:27017","docker-3:27017" ]
uri_mongod = "mongodb://"+admin_user+":"+admin_passwd+"@"+"localhost"+":"+str(mongod_port) + "/admin"
uri_mongos = "localhost" +":"+str(mongos_port)
#Connection to mongos and mongod
try:
mongos = MongoClient(uri_mongos,read_preference=ReadPreference.PRIMARY_PREFERRED)
mongod = MongoReplicaSetClient(uri_mongod ,replicaSet=replicaset ,read_preference=ReadPreference.PRIMARY_PREFERRED)
except pymongo.errors.ConnectionFailure ,e:
sys.stderr.write("Connection %s failed: " % e)
sys.exit(1)
try:
#Enable the sharding for a database
mongos.admin.command( "enableSharding" ,database)
#add nodes to the shard
mongos.admin.command('addShard' , servers[0] , servers[1] , servers[2])
#initiate Sharding
config = {_id: replicaset, members: [
{_id: 0, host: servers[0]},
{_id: 1, host: servers[1]},
class MongodbBroker(BaseModule):
def __init__(self, mod_conf):
BaseModule.__init__(self, mod_conf)
self._parse_conf(mod_conf)
self.queue = Queue.Queue(self.queue_size)
self.conn = None
# service notification log broks.
# ref: service.raise_notification_log_entry
self.service_notification = ('contact',
'host',
'service_description',
'state',
'command',
'output'
)
# host notification log broks.
# ref: host.raise_notification_log_entry
self.host_notification = ('contact',
'host',
'state',
'command',
'output')
self.timestamp_regex = re.compile('.*\[(?P<timestamp>\d+)\].*')
def _parse_conf(self, mod_conf):
self.high_availability = to_bool(getattr(mod_conf,
'high_availability', 'false'))
if not self.high_availability:
self.stand_alone = getattr(mod_conf, 'stand_alone', '')
if not self.stand_alone:
logger.error('[Mongodb-Notification-Broker] Mongodb is '
'configured with high availability be false but '
'stand_alone is not configured')
raise Exception('[Mongodb-Notification-Broker] Configuration '
'Error')
else:
replica_set_str = getattr(mod_conf, 'replica_set', '')
self._set_replica_set(replica_set_str)
self.database = getattr(mod_conf,
'database', 'shinken_broker_notification')
self.username = getattr(mod_conf,
'username', 'shinken_broker_notification')
self.password = getattr(mod_conf,
'password', 'shinken_broker_notification')
self.url_options = getattr(mod_conf, 'url_options', '')
try:
self.retry_per_log = int(getattr(mod_conf, 'retry_per_log'))
except:
self.retry_per_log = 5
try:
self.queue_size = int(getattr(mod_conf, 'queue_size'))
except:
self.queue_size = 10000
def _set_replica_set(self, replica_set_str):
raw_members = replica_set_str.split(',')
members = []
for member in raw_members:
members.append(member.strip())
self.replica_set = members
def _set_mongodb_url(self):
scheme = 'mongodb://'
db_and_options = '/%s?%s' % (self.database, self.url_options)
credential = ':'.join((self.username, '%s@' % self.password))
if not self.high_availability:
address = self.stand_alone
mongodb_url = ''.join((scheme, credential, address, db_and_options))
else:
address = ','.join(self.replica_set)
mongodb_url = ''.join((scheme, credential, address, db_and_options))
self.mongodb_url = mongodb_url
# Called by Broker to do init work
def init(self):
logger.info('[Mongodb-Notification-Broker] Initialization of '
'mongodb_notification_broker module')
self._set_mongodb_url()
logger.debug('[Mongodb-Notification-Broker] Mongodb connect url: %s'
% self.mongodb_url)
# In case notification broker process down occasionally, the self.conn
# object must be dropped cleanly in Broker daemon.
self.do_stop()
try:
if not self.high_availability:
self.conn = MongoClient(self.mongodb_url)
else:
self.conn = MongoReplicaSetClient(self.mongodb_url)
except ConnectionFailure:
logger.warn('[Mongodb-Notification-Broker] Can not make connection '
' with MongoDB')
raise
except (InvalidURI, ConfigurationError):
logger.warn('[Mongodb-Notification-Broker] Mongodb connect url '
'error')
logger.warn('[Mongodb-Notification-Broker] Mongodb connect url: %s'
% self.mongodb_url)
raise
self._get_collections()
def _get_collections(self):
db = self.conn[self.database]
self.hosts = db['hosts']
self.services = db['services']
self.notifications = db['notifications']
# Override the same function in basemodule.py for clean up
def do_stop(self):
if self.conn:
self.conn.close()
self.conn = None
# If we are confronted with AutoReconnect Exception, then we should always
# retry until the operation succeeds. However, if other exception is thrown,
# we should ignore the operation and go to next operation.
def _process_db_operation(self, operation, *param):
reconnect_start = time.time()
result = None
while True:
try:
result = operation(*param)
except AutoReconnect:
logger.warn('[Mongodb-Notification-Broker] Update error. '
'Reconnected last %d seconds' % (time.time() - reconnect_start))
# avoid to invoke too many write operations
time.sleep(self.retry_per_log)
except Exception:
logger.warn('[Mongodb-Notification-Broker] Update error. '
'operation %s, param %s' % (operation, param))
logger.warn('[Mongodb-Notification-Broker] %s' % traceback.format_exc())
break
else:
logger.debug('[Mongodb-Notification-Broker] Update success. '
'Operation %s, param %s' % (operation, param))
break
return result
# main function to update mongodb database
def _save(self, ref, ref_identity, notification):
self._process_db_operation(self.notifications.insert, notification)
if ref == 'service':
_id = ','.join((ref_identity.get('host'),
ref_identity.get('service_description')))
cursor = self._process_db_operation(self.services.find,
{'_id': _id})
elif ref == 'host':
_id = ref_identity.get('host')
cursor = self._process_db_operation(self.hosts.find, {'_id': _id})
# if service or host find error, 'cursor' will be None.
# then we can not make sure that whether specific host or service
# exists. In order to not make data be corrupted, we should stop here.
if cursor:
if not cursor.count():
# if notification insert error, then '_id' will not be in it and we
# then should ignore the notification.
ref_identity.setdefault('notification_ids',
[notification.get('_id')] if '_id' in notification else [])
ref_identity.setdefault('_id', _id)
if ref == 'service':
self._process_db_operation(self.services.insert, ref_identity)
elif ref == 'host':
self._process_db_operation(self.hosts.insert, ref_identity)
else:
document = cursor[0]
notification_ids = document.get('notification_ids')
# if notification insert error, then '_id' will not be in it and we
# then should ignore the notification
if '_id' in notification:
notification_ids.append(notification.get('_id'))
if ref == 'service':
self._process_db_operation(self.services.update,
{'_id': _id},
{'$set': {'notification_ids': notification_ids}})
elif ref == 'host':
self._process_db_operation(self.hosts.update,
{'_id': _id},
{'$set': {'notification_ids': notification_ids}})
else:
logger.warn('[Mongodb-Notification-Broker] Update notification '
'success, link with host or service error.')
logger.debug('[Mongodb-Notification-Broker] Update notification ends.')
# restore 'log' type notification to self.queue
def _do_loop_turn(self):
while not self.interrupted:
broks = self.to_q.get()
for brok in broks:
brok.prepare()
self._manage_brok(brok)
def _update_db(self):
while True:
# if self.queue is empty, get operation will be blocked.
brok = self.queue.get()
logger.debug('[Mongodb-Notification-Broker] '
'Update notification begins.')
msg = brok.data['log']
parts = msg.split(':', 1)
if 'SERVICE' in parts[0]:
service_identiry, notification = self._process_notification_brok('service',
self.service_notification,
parts[0], parts[1])
self._save('service', service_identiry, notification)
elif 'HOST' in parts[0]:
host_identity, notification = self._process_notification_brok('host',
self.host_notification,
parts[0], parts[1])
self._save('host', host_identity, notification)
def _process_notification_brok(self, ref, keys, header, notification_info):
elts = notification_info.split(';', len(keys))
timestamp = ''
match = self.timestamp_regex.match(header)
if match:
timestamp = match.group('timestamp')
info_map = dict(zip(keys, elts))
if ref == 'service':
ref_identity = {'host': info_map.get('host'),
'service_description': info_map.get('service_description')
}
elif ref == 'host':
ref_identity = {'host': info_map.get('host')}
notification = {'contact': info_map.get('contact'),
'command': info_map.get('command'),
'output': info_map.get('output'),
'timestamp': timestamp
}
return ref_identity, notification
def _manage_brok(self, brok):
if brok.type == 'log' and 'NOTIFICATION' in brok.data['log']:
try:
self.queue.put_nowait(brok)
except Queue.Full:
logger.warn('[Mongodb-Notification-Broker] Queue full. '
'Ignore broks.')
# invoked by basemodule._main
def main(self):
logger.info('[Mongodb-Notification-Broker] Start main function.')
worker = Thread(target=self._update_db)
worker.setDaemon(True)
worker.start()
self._do_loop_turn()
