def member_add(self, member_id=None, params=None):
"""add new member into existing configuration"""
member_id = member_id or str(uuid4())
if 'members' in params:
# is replica set
rs_params = params.copy()
# Turn 'rs_id' -> 'id', to be consistent with 'server_id' below.
rs_params['id'] = rs_params.pop('rs_id', None)
rs_params.update({'sslParams': self.sslParams})
rs_params['version'] = params.pop('version', self._version)
rs_params['members'] = map(self._strip_auth, rs_params['members'])
rs_id = ReplicaSets().create(rs_params)
members = ReplicaSets().members(rs_id)
cfgs = rs_id + r"/" + ','.join([item['host'] for item in members])
result = self._add(cfgs, member_id)
if result.get('ok', 0) == 1:
self._shards[result['shardAdded']] = {'isReplicaSet': True, '_id': rs_id}
# return self._shards[result['shardAdded']].copy()
return self.member_info(member_id)
else:
# is single server
params.update({'autostart': True, 'sslParams': self.sslParams})
params = params.copy()
params['procParams'] = self._strip_auth(
params.get('procParams', {}))
params.setdefault('version', self._version)
logger.debug("servers create params: {params}".format(**locals()))
server_id = Servers().create('mongod', **params)
result = self._add(Servers().hostname(server_id), member_id)
if result.get('ok', 0) == 1:
self._shards[result['shardAdded']] = {'isServer': True, '_id': server_id}
return self.member_info(member_id)
def router_add(self, params):
"""add new router (mongos) into existing configuration"""
if self.uses_rs_configdb:
# Replica set configdb.
rs_id = self._configsvrs[0]
config_members = ReplicaSets().members(rs_id)
configdb = '%s/%s' % (rs_id, ','.join(m['host']
for m in config_members))
else:
configdb = ','.join(Servers().hostname(item)
for item in self._configsvrs)
server_id = params.pop('server_id', None)
version = params.pop('version', self._version)
params.update({'configdb': configdb})
if self.enable_ipv6:
common.enable_ipv6_single(params)
# Remove flags that turn auth on.
params = self._strip_auth(params)
self._routers.append(Servers().create('mongos',
params,
sslParams=self.sslParams,
autostart=True,
version=version,
server_id=server_id))
return {
'id': self._routers[-1],
'hostname': Servers().hostname(self._routers[-1])
}
def host_info(host_id):
logger.debug("host_info({host_id})".format(**locals()))
if host_id not in Servers():
return send_result(404)
result = Servers().info(host_id)
result['links'] = all_server_links(host_id, rel_to='get-server-info')
result['links'].append(server_link('get-servers'))
return send_result(200, result)
def router_connections(self):
"""Return a list of MongoClients, one for each mongos."""
clients = []
for server in self._routers:
if Servers().is_alive(server):
client = self.create_connection(Servers().hostname(server))
clients.append(client)
return clients
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
self.repl_cfg = {
'auth_key': 'secret',
'login': '******',
'password': '******',
'members': [{}, {}]
}
self.repl = ReplicaSet(self.repl_cfg)
def test_primary_stepdown(self):
# This tests Server, but only makes sense in the context of a replica set.
repl_id = self.rs.create(
{'id': 'test-rs-stepdown',
'members': [{}, {}, {"rsParams": {"priority": 1.4}}]})
primary = self.rs.primary(repl_id)
primary_server = Servers()._storage[primary['server_id']]
# No Exception.
primary_server.stepdown()
self.assertNotEqual(primary['mongodb_uri'],
self.rs.primary(repl_id)['mongodb_uri'])
def test_member_create(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
result = self.repl.member_create({}, 13)
self.assertTrue('host' in result)
self.assertTrue('_id' in result)
h_id = Servers().host_to_server_id(result['host'])
h_info = Servers().info(h_id)
self.assertIn(result['host'], h_info['mongodb_uri'])
self.assertTrue(h_info['procInfo']['alive'])
Servers().remove(h_id)
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {
'auth_key': 'secret',
'login': '******',
'password': '******',
'members': [{}, {}]
}
self.repl = ReplicaSet(self.repl_cfg)
def router_add(self, params):
"""add new router (mongos) into existing configuration"""
cfgs = ','.join([Servers().hostname(item) for item in self._configsvrs])
server_id = params.pop('server_id', None)
params.update({'configdb': cfgs})
# Remove flags that turn auth on.
params = self._strip_auth(params)
self._routers.append(Servers().create(
'mongos', params, sslParams=self.sslParams, autostart=True,
version=self._version, server_id=server_id))
return {'id': self._routers[-1], 'hostname': Servers().hostname(self._routers[-1])}
def host_command(host_id):
logger.debug("host_command({host_id})".format(**locals()))
if host_id not in Servers():
return send_result(404)
command = get_json(request.body).get('action')
if command is None:
raise RequestError('Expected body with an {"action": ...}.')
result = {
'command_result': Servers().command(host_id, command),
'links': all_server_links(host_id, rel_to='server-command')
}
result['links'].append(server_link('get-servers'))
return send_result(200, result)
def test_reset(self):
all_hosts = []
# Start a ShardedCluster with 1 router and 1 config server.
self.sh = ShardedCluster({})
# Add 1 Server shard and 1 ReplicaSet shard.
server_id = self.sh.member_add(params={})['_id']
all_hosts.append(Servers().hostname(server_id))
repl_id = self.sh.member_add(params={'members': [{}, {}, {}]})['_id']
# Shut down the standalone.
Servers().command(server_id, 'stop')
# Shut down each member of the replica set.
server_ids = [m['server_id'] for m in ReplicaSets().members(repl_id)]
for s_id in server_ids:
Servers().command(s_id, 'stop')
all_hosts.append(Servers().hostname(s_id))
# Shut down config server and router.
config_id = self.sh.configsvrs[0]['id']
print("config_id=%r" % config_id)
all_hosts.append(Servers().hostname(config_id))
router_id = self.sh.routers[0]['id']
print("router_id=%r" % router_id)
all_hosts.append(Servers().hostname(router_id))
Servers().command(config_id, 'stop')
Servers().command(router_id, 'stop')
# Reset the ShardedCluster.
self.sh.reset()
# Everything is up.
for host in all_hosts:
# No ConnectionFailure/AutoReconnect.
pymongo.MongoClient(host)
def _host_create(params):
host_id = params.get('id')
host_id = Servers().create(params['name'], params.get('procParams', {}),
params.get('sslParams', {}),
params.get('auth_key', ''),
params.get('login', ''),
params.get('password', ''),
params.get('timeout', 300),
params.get('autostart', True), host_id,
params.get('version', ''))
result = Servers().info(host_id)
server_id = result['id']
result['links'] = all_server_links(server_id)
return result
class ReplicaSetAuthTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
def tearDown(self):
if len(self.repl) > 0:
self.repl.cleanup()
def test_auth_connection(self):
self.assertTrue(isinstance(self.repl.connection().admin.collection_names(), list))
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_auth_admin(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
self.assertTrue(isinstance(c.admin.collection_names(), list))
self.assertTrue(c.admin.logout() is None)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_auth_collection(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
db = c.test_auth
db.add_user('user', 'userpass', roles=['readWrite'])
c.admin.logout()
self.assertTrue(db.authenticate('user', 'userpass'))
self.assertTrue(db.foo.insert({'foo': 'bar'}, w=2, wtimeout=1000))
self.assertTrue(isinstance(db.foo.find_one(), dict))
db.logout()
self.assertRaises(pymongo.errors.OperationFailure, db.foo.find_one)
def test_auth_arbiter_member_info(self):
self.repl = ReplicaSet({'members': [
{}, {'rsParams': {'arbiterOnly': True}}]})
info = self.repl.member_info(1)
for key in ('procInfo', 'mongodb_uri', 'statuses', 'rsInfo'):
self.assertIn(key, info)
rs_info = info['rsInfo']
for key in ('primary', 'secondary', 'arbiterOnly'):
self.assertIn(key, rs_info)
self.assertFalse(rs_info['primary'])
self.assertFalse(rs_info['secondary'])
self.assertTrue(rs_info['arbiterOnly'])
def router_add(self, params):
"""add new router (mongos) into existing configuration"""
cfgs = ','.join(
[Servers().hostname(item) for item in self._configsvrs])
params.update({'configdb': cfgs})
self._routers.append(Servers().create('mongos',
params,
sslParams=self.sslParams,
autostart=True,
auth_key=self.auth_key,
version=self._version))
return {
'id': self._routers[-1],
'hostname': Servers().hostname(self._routers[-1])
}
def test_primary_stepdown(self):
# This tests Server,
# but only makes sense in the context of a replica set.
repl_id = self.rs.create({
'id': 'test-rs-stepdown',
'members': [{}, {}, {}]
})
primary = self.rs.primary(repl_id)
primary_server = Servers()._storage[primary['server_id']]
time.sleep(20)
# No Exception.
primary_server.stepdown()
self.assertNotEqual(primary['mongodb_uri'],
self.rs.primary(repl_id)['mongodb_uri'])
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
self.repl_cfg = {
'members': [{}, {}, {
'rsParams': {
'priority': 0,
'hidden': True
}
}, {
'rsParams': {
'arbiterOnly': True
}
}]
}
def reset(self):
"""Ensure all shards, configs, and routers are running and available."""
# Ensure all shards by calling "reset" on each.
for shard_id in self._shards:
if self._shards[shard_id].get('isReplicaSet'):
singleton = ReplicaSets()
elif self._shards[shard_id].get('isServer'):
singleton = Servers()
singleton.command(self._shards[shard_id]['_id'], 'reset')
# Ensure all config servers by calling "reset" on each.
for config_id in self._configsvrs:
self.configdb_singleton.command(config_id, 'reset')
# Ensure all routers by calling "reset" on each.
for router_id in self._routers:
Servers().command(router_id, 'reset')
return self.info()
def test_member_info_auth_uri(self):
member_params = {
'procParams': {
'clusterAuthMode': 'x509',
'setParameter': {
'authenticationMechanisms': 'MONGODB-X509'
}
}
}
self.repl_cfg = {
'login': TEST_SUBJECT,
'authSource': '$external',
'members': [member_params, member_params],
'sslParams': {
'sslCAFile': certificate('ca.pem'),
'sslPEMKeyFile': certificate('server.pem'),
'sslMode': 'requireSSL',
'sslClusterFile': certificate('cluster_cert.pem'),
'sslAllowInvalidCertificates': True
}
}
self.repl = ReplicaSet(self.repl_cfg)
for i in range(len(self.repl)):
member = self.repl.member_info(i)
self.assertIn('mongodb_auth_uri', member)
uri = member['mongodb_auth_uri']
host = Servers().hostname(member['server_id'])
self.assertIn(host, uri)
self.assertIn(TEST_SUBJECT, uri)
self.assertIn('authSource=$external', uri)
self.assertIn('authMechanism=MONGODB-X509', uri)
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {
'members': [{}, {}, {
'rsParams': {
'priority': 0,
'hidden': True
}
}, {
'rsParams': {
'arbiterOnly': True
}
}]
}
def member_add(self, member_id=None, params=None):
"""add new member into existing configuration"""
member_id = member_id or str(uuid4())
if 'members' in params:
# is replica set
rs_params = params.copy()
rs_params.update({'auth_key': self.auth_key})
rs_params.update({'sslParams': self.sslParams})
if self.login and self.password:
rs_params.update({
'login': self.login,
'password': self.password
})
if self._version:
rs_params['version'] = self._version
rs_id = ReplicaSets().create(rs_params)
members = ReplicaSets().members(rs_id)
cfgs = rs_id + r"/" + ','.join([item['host'] for item in members])
result = self._add(cfgs, member_id)
if result.get('ok', 0) == 1:
self._shards[result['shardAdded']] = {
'isReplicaSet': True,
'_id': rs_id
}
# return self._shards[result['shardAdded']].copy()
return self.member_info(member_id)
else:
# is single server
params.update({
'autostart': True,
'auth_key': self.auth_key,
'sslParams': self.sslParams
})
params['procParams'] = params.get('procParams', {})
if self._version:
params['version'] = self._version
logger.debug("servers create params: {params}".format(**locals()))
server_id = Servers().create('mongod', **params)
result = self._add(Servers().hostname(server_id), member_id)
if result.get('ok', 0) == 1:
self._shards[result['shardAdded']] = {
'isServer': True,
'_id': server_id
}
# return self._shards[result['shardAdded']]
return self.member_info(member_id)
def configsvrs(self):
"""return list of config servers"""
if self.uses_rs_configdb:
rs_id = self._configsvrs[0]
mongodb_uri = ReplicaSets().info(rs_id)['mongodb_uri']
return [{'id': rs_id, 'mongodb_uri': mongodb_uri}]
return [{'id': h_id, 'hostname': Servers().hostname(h_id)}
for h_id in self._configsvrs]
def test_reset(self):
self.repl_cfg = {'members': [{}, {}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
server_ids = [m['server_id'] for m in self.repl.members()]
all_hosts = map(Servers().hostname, server_ids)
# Shut down all members of the ReplicaSet.
for server_id in server_ids:
Servers().command(server_id, 'stop')
# Reset the ReplicaSet. --- We should be able to connect to all members.
self.repl.reset()
for host in all_hosts:
# No ConnectionFailure/AutoReconnect.
connected(pymongo.MongoClient(host))
def test_member_freeze(self):
# This tests Server, but only makes sense in the context of a replica set.
repl_id = self.rs.create({
'members': [{
"rsParams": {
"priority": 19
}
}, {
"rsParams": {
"priority": 5
}
}, {}]
})
next_primary_info = self.rs.member_info(repl_id, 2)
next_primary = next_primary_info['mongodb_uri']
secondary_info = self.rs.member_info(repl_id, 1)
secondary_server = Servers()._storage[secondary_info['server_id']]
primary_info = self.rs.member_info(repl_id, 0)
primary_server = Servers()._storage[primary_info['server_id']]
assert_eventually(lambda: primary_server.connection.is_primary)
def freeze_and_stop():
self.assertTrue(secondary_server.freeze(10))
try:
# Call replSetStepDown before killing the primary's process.
# This raises OperationFailure if no secondaries are capable
# of taking over.
primary_server.connection.admin.command('replSetStepDown', 10)
except pymongo.errors.AutoReconnect:
# Have to stop the server as well so it doesn't get reelected.
primary_server.stop()
return True
except pymongo.errors.OperationFailure:
# No secondaries within 10 seconds of my optime...
return False
assert_eventually(freeze_and_stop, "Primary didn't step down.")
assert_eventually(
lambda: (self.rs.primary(repl_id)['mongodb_uri'] == next_primary),
"Secondary did not freeze.",
max_tries=120)
assert_eventually(
lambda: (self.rs.primary(repl_id)['mongodb_uri'] == self.rs.
member_info(repl_id, 1)['mongodb_uri']),
"Higher priority secondary never promoted.")
def cleanup(self):
"""cleanup configuration: stop and remove all servers"""
for _id, shard in self._shards.items():
if shard.get('isServer', False):
Servers().remove(shard['_id'])
if shard.get('isReplicaSet', False):
ReplicaSets().remove(shard['_id'])
for mongos in self._routers:
Servers().remove(mongos)
for config_id in self._configsvrs:
self.configdb_singleton.remove(config_id)
self._configsvrs = []
self._routers = []
self._shards = {}
def test_member_info_auth_uri(self):
for i in range(len(self.repl)):
member = self.repl.member_info(i)
self.assertIn('mongodb_auth_uri', member)
uri = member['mongodb_auth_uri']
host = Servers().hostname(member['server_id'])
self.assertIn(host, uri)
self.assertIn('admin:admin', uri)
self.assertIn('authSource=admin', uri)
def test_router(self):
config = {}
self.sh = ShardedCluster(config)
self.assertTrue(Servers().info(self.sh.router['id'])['statuses']['mongos'])
self.sh.cleanup()
config = {'routers': [{}, {}, {}]}
self.sh = ShardedCluster(config)
routers = self.sh.routers
hostname = routers[1]['hostname']
_id = routers[1]['id']
# stop routers 0 and 2
Servers().command(routers[0]['id'], 'stop')
Servers().command(routers[2]['id'], 'stop')
router = self.sh.router
self.assertEqual(router['id'], _id)
self.assertEqual(router['hostname'], hostname)
self.sh.cleanup()
def _remove(self, shard_name):
"""remove member from configuration"""
result = self.router_command("removeShard", shard_name, is_eval=False)
if result['ok'] == 1 and result['state'] == 'completed':
shard = self._shards.pop(shard_name)
if shard.get('isServer', False):
Servers().remove(shard['_id'])
if shard.get('isReplicaSet', False):
ReplicaSets().remove(shard['_id'])
return result
def __init_configsvr(self, params):
"""create and start config servers"""
self._configsvrs = []
for cfg in params:
# Remove flags that turn on auth.
cfg = self._strip_auth(cfg)
server_id = cfg.pop('server_id', None)
cfg.update({'configsvr': True})
self._configsvrs.append(Servers().create(
'mongod', cfg, sslParams=self.sslParams, autostart=True,
version=self._version, server_id=server_id))
def __init_configsvr(self, params):
"""create and start config servers"""
self._configsvrs = []
for cfg in params:
cfg.update({'configsvr': True})
self._configsvrs.append(Servers().create('mongod',
cfg,
sslParams=self.sslParams,
autostart=True,
auth_key=self.auth_key,
version=self._version))
def host_list():
logger.debug("host_list()")
servers = []
for server_id in Servers():
server_info = {'id': server_id}
server_info['links'] = all_server_links(
server_id, rel_to='get-servers')
servers.append(server_info)
response = {'links': [
base_link('service'),
base_link('get-releases'),
server_link('get-servers', self_rel=True),
server_link('add-server'),
replica_set_link('get-replica-sets'),
sharded_cluster_link('get-sharded-clusters')
]}
response['servers'] = servers
return send_result(200, response)
def test_info(self):
config = {
'configsvrs': [{}, {}, {}],
'routers': [{}, {}, {}],
'shards': [{}, {}]
}
sh_id = self.sh.create(config)
info = self.sh.info(sh_id)
self.assertTrue(isinstance(info, dict))
for item in ("shards", "configsvrs", "routers", "mongodb_uri",
"orchestration"):
self.assertTrue(item in info)
self.assertEqual(len(info['shards']), 2)
self.assertEqual(len(info['configsvrs']), 3)
self.assertEqual(len(info['routers']), 3)
mongodb_uri = info['mongodb_uri']
for router in info['routers']:
self.assertIn(Servers().hostname(router['id']), mongodb_uri)
self.assertTrue(mongodb_uri.find('mongodb://') == 0)
self.assertEqual(info['orchestration'], 'sharded_clusters')
def setUp(self):
PortPool().change_range()
self.path = tempfile.mktemp(prefix="test-storage")
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', ""))
def test_primary(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primary = self.repl.primary()
server_id = Servers().host_to_server_id(primary)
self.assertTrue(Servers().info(server_id)['statuses']['primary'])
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}, {'rsParams': {'arbiterOnly': True}}]}
def setUp(self):
PortPool().change_range()
self.path = tempfile.mktemp(prefix="test-storage")
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
class ReplicaSetAuthTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
self.repl_cfg = {'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
def tearDown(self):
if len(self.repl) > 0:
self.repl.cleanup()
def test_auth_connection(self):
self.assertTrue(isinstance(self.repl.connection().admin.collection_names(), list))
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_auth_admin(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
self.assertTrue(isinstance(c.admin.collection_names(), list))
self.assertTrue(c.admin.logout() is None)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_auth_collection(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
db = c.test_auth
db.add_user('user', 'userpass', roles=['readWrite'])
c.admin.logout()
self.assertTrue(db.authenticate('user', 'userpass'))
self.assertTrue(db.foo.insert({'foo': 'bar'}, w=2, wtimeout=10000))
self.assertTrue(isinstance(db.foo.find_one(), dict))
db.logout()
self.assertRaises(pymongo.errors.OperationFailure, db.foo.find_one)
def test_auth_arbiter_member_info(self):
self.repl.cleanup()
self.repl = ReplicaSet({'members': [
{}, {'rsParams': {'arbiterOnly': True}}]})
info = self.repl.member_info(1)
for key in ('procInfo', 'mongodb_uri', 'statuses', 'rsInfo'):
self.assertIn(key, info)
rs_info = info['rsInfo']
for key in ('primary', 'secondary', 'arbiterOnly'):
self.assertIn(key, rs_info)
self.assertFalse(rs_info['primary'])
self.assertFalse(rs_info['secondary'])
self.assertTrue(rs_info['arbiterOnly'])
def test_mongodb_auth_uri(self):
self.assertIn('mongodb_auth_uri', self.repl.info())
rs_auth_uri = self.repl.info()['mongodb_auth_uri']
hosts = ','.join(m['host'] for m in self.repl.members())
self.assertIn(hosts, rs_auth_uri)
self.assertIn('admin:admin', rs_auth_uri)
self.assertIn('authSource=admin', rs_auth_uri)
replset_param = 'replicaSet=' + self.repl.repl_id
self.assertIn(replset_param, rs_auth_uri)
def test_member_info_auth_uri(self):
for i in range(len(self.repl)):
member = self.repl.member_info(i)
self.assertIn('mongodb_auth_uri', member)
uri = member['mongodb_auth_uri']
host = Servers().hostname(member['server_id'])
self.assertIn(host, uri)
self.assertIn('admin:admin', uri)
self.assertIn('authSource=admin', uri)
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
self.repl_cfg = {'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
TEST_RELEASES = (
{'default-release': os.environ.get('MONGOBIN', '')},
'default-release')
# Set up the default mongo binaries to use from MONGOBIN.
set_releases(*TEST_RELEASES)
# Turn off journal for tests.
Server.mongod_default['nojournal'] = True
SSL_ENABLED = False
SERVER_VERSION = (2, 6)
__server_id = Servers().create(name='mongod', procParams={})
try:
# Server version
info = Servers().info(__server_id)['serverInfo']
version_str = re.search('((\d+\.)+\d+)', info['version']).group(0)
SERVER_VERSION = tuple(map(int, version_str.split('.')))
# Do we have SSL support?
SSL_ENABLED = bool(info.get('OpenSSLVersion'))
finally:
Servers().cleanup()
if sys.version_info[:2] == (2, 6):
import unittest2 as unittest
from unittest2 import SkipTest
else:
import unittest
from unittest import SkipTest
command = get_json(request.body).get('action')
if command is None:
raise RequestError('Expected body with an {"action": ...}.')
result = {
'command_result': Servers().command(host_id, command),
'links': all_server_links(host_id, rel_to='server-command')
}
result['links'].append(server_link('get-servers'))
return send_result(200, result)
ROUTES = {
Route('/', method='GET'): base_uri,
Route('/releases', method='GET'): releases_list,
Route('/servers', method='POST'): host_create,
Route('/servers', method='GET'): host_list,
Route('/servers/<host_id>', method='GET'): host_info,
Route('/servers/<host_id>', method='PUT'): host_create_by_id,
Route('/servers/<host_id>', method='DELETE'): host_del,
Route('/servers/<host_id>', method='POST'): host_command
}
setup_versioned_routes(ROUTES, version='v1')
# Assume v1 if no version is specified.
setup_versioned_routes(ROUTES)
if __name__ == '__main__':
hs = Servers()
hs.set_settings()
run(host='localhost', port=8889, debug=True, reloader=False)
class ServersTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.path = tempfile.mktemp(prefix="test-storage")
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', ""))
def remove_path(self, path):
onerror = lambda func, filepath, exc_info: (os.chmod(filepath, stat.S_IWUSR), func(filepath))
if os.path.isfile(path):
try:
os.remove(path)
except OSError:
time.sleep(2)
onerror(os.remove, path, None)
def tearDown(self):
self.servers.cleanup()
self.remove_path(self.path)
def test_singleton(self):
self.assertEqual(id(self.servers), id(Servers()))
def test_set_settings(self):
default_release = 'old-release'
releases = {default_release: os.path.join(os.getcwd(), 'bin')}
self.servers.set_settings(releases, default_release)
self.assertEqual(releases, self.servers.releases)
self.assertEqual(default_release, self.servers.default_release)
def test_bool(self):
self.assertEqual(False, bool(self.servers))
self.servers.create('mongod', {}, autostart=False)
self.assertTrue(True, bool(self.servers))
def test_operations(self):
server_id = self.servers.create('mongod', {}, autostart=False)
self.assertTrue(len(self.servers) == 1)
self.assertTrue(server_id in self.servers)
server_id2 = 'server-id2'
server2 = Server(os.path.join(os.environ.get('MONGOBIN', ''), 'mongod'), {})
server2.start(30)
server2_pid = server2.info()['procInfo']['pid']
self.servers[server_id2] = server2
self.assertTrue(self.servers[server_id2]['procInfo']['pid'] == server2_pid)
self.assertTrue(server_id2 in self.servers)
for h_id in self.servers:
self.assertTrue(h_id in (server_id, server_id2))
operator.delitem(self.servers, server_id2)
self.assertFalse(server_id2 in self.servers)
server2.stop()
server2.cleanup()
def test_cleanup(self):
self.servers.create('mongod', {}, autostart=False)
self.servers.create('mongod', {}, autostart=True)
self.assertTrue(len(self.servers) == 2)
self.servers.cleanup()
self.assertTrue(len(self.servers) == 0)
def test_new_server(self):
self.assertTrue(len(self.servers) == 0)
server_id = self.servers.create('mongod', {}, autostart=False)
info = self.servers.info(server_id)
self.assertTrue(len(self.servers) == 1)
self.assertNotIn('pid', info['procInfo'])
server_id2 = self.servers.create('mongod', {}, autostart=True)
info = self.servers.info(server_id2)
self.assertTrue(info['procInfo']['pid'] > 0)
self.assertRaises(OSError, self.servers.create, 'fake_process_', {})
def test_new_server_with_auth(self):
server_id = self.servers.create('mongod', {}, login='******', password='******', autostart=True)
hostname = self.servers.hostname(server_id)
c = pymongo.MongoClient(hostname)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
self.assertTrue(c.admin.authenticate('adminko', 'password'))
self.assertTrue(isinstance(c.admin.collection_names(), list))
self.assertTrue(c.admin.logout() is None)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_hdel(self):
self.assertEqual(len(self.servers), 0)
h_id = self.servers.create('mongod', {}, autostart=True)
self.assertEqual(len(self.servers), 1)
h_info = self.servers.info(h_id)['procInfo']
self.assertTrue(os.path.exists(h_info['params']['dbpath']))
self.assertTrue(os.path.exists(h_info['optfile']))
self.servers.remove(h_id)
self.assertEqual(len(self.servers), 0) # check length
# check cleanup
self.assertFalse(os.path.exists(h_info['params']['dbpath']))
self.assertFalse(os.path.exists(h_info['optfile']))
def test_hcommand(self):
h_id = self.servers.create('mongod', {}, autostart=False)
self.assertTrue(self.servers.command(h_id, 'start'))
self.assertTrue(self.servers.command(h_id, 'stop'))
self.assertTrue(self.servers.command(h_id, 'start'))
self.assertTrue(self.servers.command(h_id, 'restart'))
self.assertRaises(ValueError, self.servers.command, h_id, 'fake')
def test_hinfo(self):
h_id = self.servers.create('mongod', {}, autostart=False)
info = self.servers.info(h_id)
self.assertEqual(info['id'], h_id)
self.assertNotIn('pid', info['procInfo'])
self.assertEqual(info['statuses'], {})
self.assertEqual(info['serverInfo'], {})
def test_id_by_hostname(self):
h_id = self.servers.create('mongod', {}, autostart=True)
h_uri = self.servers.info(h_id)['uri']
h2_id = self.servers.create('mongod', {}, autostart=True)
h2_uri = self.servers.info(h2_id)['uri']
self.assertTrue(self.servers.id_by_hostname(h_uri) == h_id)
self.assertTrue(self.servers.id_by_hostname(h2_uri) == h2_id)
def test_hostname(self):
h_id = self.servers.create('mongod', {}, autostart=True)
h_uri = self.servers.info(h_id)['uri']
self.assertEqual(self.servers.hostname(h_id), h_uri)
def test_is_alive(self):
h_id = self.servers.create('mongod', {}, autostart=True)
self.assertEqual(self.servers.is_alive(h_id), True)
self.servers.command(h_id, 'stop')
self.assertEqual(self.servers.is_alive(h_id), False)
def test_db_command(self):
h_id = self.servers.create('mongod', {}, autostart=False)
self.assertRaises(pymongo.errors.PyMongoError, self.servers.db_command, h_id, 'serverStatus', None, False)
self.servers.command(h_id, 'start', 10)
self.assertEqual(self.servers.db_command(h_id, 'serverStatus', arg=None, is_eval=False).get('ok', -1), 1)
self.assertEqual(self.servers.db_command(h_id, 'db.getName()', arg=None, is_eval=True), 'admin')
def test_id_specified(self):
id = 'xyzzy'
h_id = self.servers.create('mongod', {}, autostart=False, server_id=id)
self.assertEqual(id, h_id)
class ReplicaSetTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(*TEST_RELEASES)
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}, {'rsParams': {'arbiterOnly': True}}]}
# self.repl = ReplicaSet(self.repl_cfg)
def tearDown(self):
if hasattr(self, 'repl'):
self.repl.cleanup()
def test_len(self):
self.repl = ReplicaSet(self.repl_cfg)
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']))
self.repl.member_del(3)
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']) - 1)
self.repl.repl_member_add({'rsParams': {'arbiterOnly': True}})
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']))
def test_cleanup(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']))
self.repl.cleanup()
self.assertTrue(len(self.repl) == 0)
def test_member_id_to_host(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
members = self.repl.config['members']
for member in members:
host = self.repl.member_id_to_host(member['_id'])
self.assertEqual(member['host'], host)
def test_host2id(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
members = self.repl.config['members']
for member in members:
self.assertEqual(member['_id'],
self.repl.host2id(member['host']))
def test_update_server_map(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
origin = self.repl.server_map.copy()
self.repl.update_server_map(self.repl.config)
self.assertEqual(self.repl.server_map, origin)
def test_repl_update(self):
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}]}
self.repl = ReplicaSet(self.repl_cfg)
config = self.repl.config
config['members'][1]['priority'] = 0
config['members'][1]['hidden'] = True
self.assertTrue(self.repl.repl_update(config))
self.assertTrue(self.repl.config['members'][1]['hidden'])
def test_info(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
cfg = self.repl.config
info = self.repl.info()
self.assertEqual(info['auth_key'], self.repl.auth_key)
self.assertEqual(info['id'], self.repl.repl_id)
self.assertEqual(len(info['members']), len(cfg['members']))
members1 = sorted(cfg['members'], key=lambda item: item['_id'])
members2 = sorted(info['members'], key=lambda item: item['_id'])
for i in range(len(members1)):
self.assertEqual(members1[i]['_id'], members2[i]['_id'])
self.assertEqual(members1[i]['host'], members2[i]['host'])
def test_repl_member_add(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
member_id = self.repl.repl_member_add({"rsParams": {"priority": 0, "hidden": True}})
self.assertTrue(member_id >= 0)
member = [item for item in self.repl.config['members'] if item['_id'] == member_id][0]
self.assertTrue(member['hidden'])
def test_run_command(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
self.assertEqual(self.repl.run_command("rs.status()", is_eval=True)['ok'], 1)
result = self.repl.run_command('serverStatus', arg=None, is_eval=False, member_id=0)['repl']
for key in ('me', 'ismaster', 'setName', 'primary', 'hosts'):
self.assertTrue(key in result)
self.assertEqual(self.repl.run_command(command="replSetGetStatus", is_eval=False)['ok'], 1)
def test_config(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
config = self.repl.config
self.assertTrue('_id' in config)
self.assertTrue('members' in config)
def test_member_create(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
result = self.repl.member_create({}, 13)
self.assertTrue('host' in result)
self.assertTrue('_id' in result)
h_id = Servers().host_to_server_id(result['host'])
h_info = Servers().info(h_id)
self.assertIn(result['host'], h_info['mongodb_uri'])
self.assertTrue(h_info['procInfo']['alive'])
Servers().remove(h_id)
def test_member_del(self):
self.repl_cfg = {'members': [{}, {}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
m_count = len(self.repl.config['members'])
self.assertTrue(self.repl.member_del(2))
self.assertEqual(len(self.repl.config['members']), m_count - 1)
def test_member_del_no_reconfig(self):
self.repl_cfg = {'members': [{}, {}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
m_count = len(self.repl.config['members'])
self.assertTrue(self.repl.member_del(2, reconfig=False))
self.assertEqual(len(self.repl.config['members']), m_count)
self.repl.server_map.pop(2)
def test_member_update(self):
self.repl = ReplicaSet(self.repl_cfg)
member = [item for item in self.repl.config['members'] if item['_id'] == 2][0]
self.assertTrue(member.get('hidden', False))
self.assertTrue(self.repl.member_update(2, {"rsParams": {"priority": 1, "hidden": False}}))
member = [item for item in self.repl.config['members'] if item['_id'] == 2][0]
self.assertFalse(member.get('hidden', False))
def test_member_info(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
member = [item for item in self.repl.config['members'] if item['_id'] == 1][0]
result = self.repl.member_info(1)
self.assertTrue(result['procInfo']['alive'])
self.assertIn(member['host'], result['mongodb_uri'])
self.assertTrue(len(result['rsInfo']) > 0)
def test_member_command(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
_id = 1
self.assertTrue(self.repl.member_info(_id)['procInfo']['alive'])
self.repl.member_command(_id, 'stop')
self.assertFalse(self.repl.member_info(_id)['procInfo']['alive'])
self.repl.member_command(_id, 'start')
self.assertTrue(self.repl.member_info(_id)['procInfo']['alive'])
self.repl.member_command(_id, 'restart')
self.assertTrue(self.repl.member_info(_id)['procInfo']['alive'])
def test_members(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
members1 = sorted(self.repl.config['members'], key=lambda item: item['_id'])
members2 = sorted(self.repl.members(), key=lambda item: item['_id'])
self.assertEqual(len(members1), len(members2))
for i in range(len(members1)):
self.assertEqual(members1[i]['host'], members2[i]['host'])
self.assertEqual(members1[i]['_id'], members2[i]['_id'])
def test_primary(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primary = self.repl.primary()
server_id = Servers().host_to_server_id(primary)
self.assertTrue(Servers().info(server_id)['statuses']['primary'])
def test_get_members_in_state(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primaries = self.repl.get_members_in_state(1)
self.assertEqual(len(primaries), 1)
self.assertEqual(primaries[0], self.repl.primary())
def test_connection(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
_id = 1
hostname = self.repl.member_id_to_host(_id)
self.assertTrue(self.repl.connection(timeout=5))
self.assertTrue(self.repl.connection(hostname=hostname, timeout=5))
self.repl.member_command(_id, 'stop')
self.assertRaises(pymongo.errors.AutoReconnect, lambda: self.repl.connection(hostname=hostname, timeout=5))
def test_secondaries(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
secondaries = [item['host'] for item in self.repl.secondaries()]
self.assertEqual(secondaries, self.repl.get_members_in_state(2))
def test_arbiters(self):
self.repl = ReplicaSet(self.repl_cfg)
arbiters = [item['host'] for item in self.repl.arbiters()]
self.assertEqual(arbiters, self.repl.get_members_in_state(7))
def test_hidden(self):
self.repl = ReplicaSet(self.repl_cfg)
for _ in self.repl.hidden():
self.assertTrue(self.repl.run_command('serverStatus', arg=None, is_eval=False, member_id=2)['repl']['hidden'])
def test_passives(self):
self.repl = ReplicaSet(self.repl_cfg)
self.repl.repl_member_add({"rsParams": {"priority": 0}})
for member in self.repl.passives():
self.assertTrue(member['host'] in self.repl.run_command('isMaster', is_eval=False).get('passives'))
def test_servers(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
self.repl.repl_member_add({"rsParams": {"priority": 0}})
for member in self.repl.servers():
self.assertTrue(member['host'] in self.repl.run_command('isMaster', is_eval=False).get('hosts'))
def test_compare_servers_passives(self):
self.repl = ReplicaSet(self.repl_cfg)
self.repl.repl_member_add({"rsParams": {"priority": 0}})
self.repl.repl_member_add({})
servers = self.repl.servers()
passives = self.repl.passives()
for item in servers:
self.assertTrue(item not in passives)
for item in passives:
self.assertTrue(item not in servers)
def test_wait_while_reachable(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
servers = [member['host'] for member in self.repl.members()]
self.assertTrue(self.repl.wait_while_reachable(servers, timeout=10))
self.repl.member_command(1, 'stop')
self.assertFalse(self.repl.wait_while_reachable(servers, timeout=10))
def test_reset(self):
self.repl_cfg = {'members': [{}, {}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
server_ids = [m['server_id'] for m in self.repl.members()]
all_hosts = [Servers().hostname(server_id) for server_id in server_ids]
# Shut down all members of the ReplicaSet.
for server_id in server_ids:
Servers().command(server_id, 'stop')
# Reset the ReplicaSet. --- We should be able to connect to all members.
self.repl.reset()
for host in all_hosts:
# No ConnectionFailure/AutoReconnect.
connected(pymongo.MongoClient(host))
def test_rs_settings(self):
if SERVER_VERSION < (2, 4):
raise SkipTest(
"Need at least MongoDB >= 2.4 to test replica set settings.")
self.repl_cfg = {
'rsSettings': {'heartbeatTimeoutSecs': 20},
'members': [{}]
}
self.repl = ReplicaSet(self.repl_cfg)
conn = self.repl.connection()
if SERVER_VERSION >= (2, 8):
config = conn.admin.command('replSetGetConfig')['config']
else:
config = conn.local.system.replset.find_one()
self.assertEqual(config['settings']['heartbeatTimeoutSecs'], 20)
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}, {'rsParams': {'arbiterOnly': True}}]}
def setUp(self):
PortPool().change_range()
self.servers = Servers()
self.servers.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
