def member_add(self, member_id=None, params=None): """add new member into existing configuration""" member_id = member_id or str(uuid4()) if 'members' in params: # is replica set rs_params = params.copy() # Turn 'rs_id' -> 'id', to be consistent with 'server_id' below. rs_params['id'] = rs_params.pop('rs_id', None) rs_params.update({'sslParams': self.sslParams}) rs_params['version'] = params.pop('version', self._version) rs_params['members'] = map(self._strip_auth, rs_params['members']) rs_id = ReplicaSets().create(rs_params) members = ReplicaSets().members(rs_id) cfgs = rs_id + r"/" + ','.join([item['host'] for item in members]) result = self._add(cfgs, member_id) if result.get('ok', 0) == 1: self._shards[result['shardAdded']] = {'isReplicaSet': True, '_id': rs_id} # return self._shards[result['shardAdded']].copy() return self.member_info(member_id) else: # is single server params.update({'autostart': True, 'sslParams': self.sslParams}) params = params.copy() params['procParams'] = self._strip_auth( params.get('procParams', {})) params.setdefault('version', self._version) logger.debug("servers create params: {params}".format(**locals())) server_id = Servers().create('mongod', **params) result = self._add(Servers().hostname(server_id), member_id) if result.get('ok', 0) == 1: self._shards[result['shardAdded']] = {'isServer': True, '_id': server_id} return self.member_info(member_id)
def router_add(self, params): """add new router (mongos) into existing configuration""" if self.uses_rs_configdb: # Replica set configdb. rs_id = self._configsvrs[0] config_members = ReplicaSets().members(rs_id) configdb = '%s/%s' % (rs_id, ','.join(m['host'] for m in config_members)) else: configdb = ','.join(Servers().hostname(item) for item in self._configsvrs) server_id = params.pop('server_id', None) version = params.pop('version', self._version) params.update({'configdb': configdb}) if self.enable_ipv6: common.enable_ipv6_single(params) # Remove flags that turn auth on. params = self._strip_auth(params) self._routers.append(Servers().create('mongos', params, sslParams=self.sslParams, autostart=True, version=version, server_id=server_id)) return { 'id': self._routers[-1], 'hostname': Servers().hostname(self._routers[-1]) }
def router_connections(self): """Return a list of MongoClients, one for each mongos.""" clients = [] for server in self._routers: if Servers().is_alive(server): client = self.create_connection(Servers().hostname(server)) clients.append(client) return clients
def host_info(host_id): logger.debug("host_info({host_id})".format(**locals())) if host_id not in Servers(): return send_result(404) result = Servers().info(host_id) result['links'] = all_server_links(host_id, rel_to='get-server-info') result['links'].append(server_link('get-servers')) return send_result(200, result)
def test_member_create(self): self.repl_cfg = {'members': [{}, {}]} self.repl = ReplicaSet(self.repl_cfg) result = self.repl.member_create({}, 13) self.assertTrue('host' in result) self.assertTrue('_id' in result) h_id = Servers().host_to_server_id(result['host']) h_info = Servers().info(h_id) self.assertIn(result['host'], h_info['mongodb_uri']) self.assertTrue(h_info['procInfo']['alive']) Servers().remove(h_id)
def router_add(self, params): """add new router (mongos) into existing configuration""" cfgs = ','.join([Servers().hostname(item) for item in self._configsvrs]) server_id = params.pop('server_id', None) params.update({'configdb': cfgs}) # Remove flags that turn auth on. params = self._strip_auth(params) self._routers.append(Servers().create( 'mongos', params, sslParams=self.sslParams, autostart=True, version=self._version, server_id=server_id)) return {'id': self._routers[-1], 'hostname': Servers().hostname(self._routers[-1])}
def host_command(host_id): logger.debug("host_command({host_id})".format(**locals())) if host_id not in Servers(): return send_result(404) command = get_json(request.body).get('action') if command is None: raise RequestError('Expected body with an {"action": ...}.') result = { 'command_result': Servers().command(host_id, command), 'links': all_server_links(host_id, rel_to='server-command') } result['links'].append(server_link('get-servers')) return send_result(200, result)
def test_reset(self): all_hosts = [] # Start a ShardedCluster with 1 router and 1 config server. self.sh = ShardedCluster({}) # Add 1 Server shard and 1 ReplicaSet shard. server_id = self.sh.member_add(params={})['_id'] all_hosts.append(Servers().hostname(server_id)) repl_id = self.sh.member_add(params={'members': [{}, {}, {}]})['_id'] # Shut down the standalone. Servers().command(server_id, 'stop') # Shut down each member of the replica set. server_ids = [m['server_id'] for m in ReplicaSets().members(repl_id)] for s_id in server_ids: Servers().command(s_id, 'stop') all_hosts.append(Servers().hostname(s_id)) # Shut down config server and router. config_id = self.sh.configsvrs[0]['id'] print("config_id=%r" % config_id) all_hosts.append(Servers().hostname(config_id)) router_id = self.sh.routers[0]['id'] print("router_id=%r" % router_id) all_hosts.append(Servers().hostname(router_id)) Servers().command(config_id, 'stop') Servers().command(router_id, 'stop') # Reset the ShardedCluster. self.sh.reset() # Everything is up. for host in all_hosts: # No ConnectionFailure/AutoReconnect. pymongo.MongoClient(host)
def _host_create(params): host_id = params.get('id') host_id = Servers().create(params['name'], params.get('procParams', {}), params.get('sslParams', {}), params.get('auth_key', ''), params.get('login', ''), params.get('password', ''), params.get('timeout', 300), params.get('autostart', True), host_id, params.get('version', '')) result = Servers().info(host_id) server_id = result['id'] result['links'] = all_server_links(server_id) return result
def router_add(self, params): """add new router (mongos) into existing configuration""" cfgs = ','.join( [Servers().hostname(item) for item in self._configsvrs]) params.update({'configdb': cfgs}) self._routers.append(Servers().create('mongos', params, sslParams=self.sslParams, autostart=True, auth_key=self.auth_key, version=self._version)) return { 'id': self._routers[-1], 'hostname': Servers().hostname(self._routers[-1]) }
def reset(self): """Ensure all shards, configs, and routers are running and available.""" # Ensure all shards by calling "reset" on each. for shard_id in self._shards: if self._shards[shard_id].get('isReplicaSet'): singleton = ReplicaSets() elif self._shards[shard_id].get('isServer'): singleton = Servers() singleton.command(self._shards[shard_id]['_id'], 'reset') # Ensure all config servers by calling "reset" on each. for config_id in self._configsvrs: self.configdb_singleton.command(config_id, 'reset') # Ensure all routers by calling "reset" on each. for router_id in self._routers: Servers().command(router_id, 'reset') return self.info()
def test_member_info_auth_uri(self): member_params = { 'procParams': { 'clusterAuthMode': 'x509', 'setParameter': { 'authenticationMechanisms': 'MONGODB-X509' } } } self.repl_cfg = { 'login': TEST_SUBJECT, 'authSource': '$external', 'members': [member_params, member_params], 'sslParams': { 'sslCAFile': certificate('ca.pem'), 'sslPEMKeyFile': certificate('server.pem'), 'sslMode': 'requireSSL', 'sslClusterFile': certificate('cluster_cert.pem'), 'sslAllowInvalidCertificates': True } } self.repl = ReplicaSet(self.repl_cfg) for i in range(len(self.repl)): member = self.repl.member_info(i) self.assertIn('mongodb_auth_uri', member) uri = member['mongodb_auth_uri'] host = Servers().hostname(member['server_id']) self.assertIn(host, uri) self.assertIn(TEST_SUBJECT, uri) self.assertIn('authSource=$external', uri) self.assertIn('authMechanism=MONGODB-X509', uri)
def member_add(self, member_id=None, params=None): """add new member into existing configuration""" member_id = member_id or str(uuid4()) if 'members' in params: # is replica set rs_params = params.copy() rs_params.update({'auth_key': self.auth_key}) rs_params.update({'sslParams': self.sslParams}) if self.login and self.password: rs_params.update({ 'login': self.login, 'password': self.password }) if self._version: rs_params['version'] = self._version rs_id = ReplicaSets().create(rs_params) members = ReplicaSets().members(rs_id) cfgs = rs_id + r"/" + ','.join([item['host'] for item in members]) result = self._add(cfgs, member_id) if result.get('ok', 0) == 1: self._shards[result['shardAdded']] = { 'isReplicaSet': True, '_id': rs_id } # return self._shards[result['shardAdded']].copy() return self.member_info(member_id) else: # is single server params.update({ 'autostart': True, 'auth_key': self.auth_key, 'sslParams': self.sslParams }) params['procParams'] = params.get('procParams', {}) if self._version: params['version'] = self._version logger.debug("servers create params: {params}".format(**locals())) server_id = Servers().create('mongod', **params) result = self._add(Servers().hostname(server_id), member_id) if result.get('ok', 0) == 1: self._shards[result['shardAdded']] = { 'isServer': True, '_id': server_id } # return self._shards[result['shardAdded']] return self.member_info(member_id)
def configsvrs(self): """return list of config servers""" if self.uses_rs_configdb: rs_id = self._configsvrs[0] mongodb_uri = ReplicaSets().info(rs_id)['mongodb_uri'] return [{'id': rs_id, 'mongodb_uri': mongodb_uri}] return [{'id': h_id, 'hostname': Servers().hostname(h_id)} for h_id in self._configsvrs]
def test_reset(self): self.repl_cfg = {'members': [{}, {}, {}]} self.repl = ReplicaSet(self.repl_cfg) server_ids = [m['server_id'] for m in self.repl.members()] all_hosts = map(Servers().hostname, server_ids) # Shut down all members of the ReplicaSet. for server_id in server_ids: Servers().command(server_id, 'stop') # Reset the ReplicaSet. --- We should be able to connect to all members. self.repl.reset() for host in all_hosts: # No ConnectionFailure/AutoReconnect. connected(pymongo.MongoClient(host))
def cleanup(self): """cleanup configuration: stop and remove all servers""" for _id, shard in self._shards.items(): if shard.get('isServer', False): Servers().remove(shard['_id']) if shard.get('isReplicaSet', False): ReplicaSets().remove(shard['_id']) for mongos in self._routers: Servers().remove(mongos) for config_id in self._configsvrs: self.configdb_singleton.remove(config_id) self._configsvrs = [] self._routers = [] self._shards = {}
def test_member_freeze(self): # This tests Server, but only makes sense in the context of a replica set. repl_id = self.rs.create({ 'members': [{ "rsParams": { "priority": 19 } }, { "rsParams": { "priority": 5 } }, {}] }) next_primary_info = self.rs.member_info(repl_id, 2) next_primary = next_primary_info['mongodb_uri'] secondary_info = self.rs.member_info(repl_id, 1) secondary_server = Servers()._storage[secondary_info['server_id']] primary_info = self.rs.member_info(repl_id, 0) primary_server = Servers()._storage[primary_info['server_id']] assert_eventually(lambda: primary_server.connection.is_primary) def freeze_and_stop(): self.assertTrue(secondary_server.freeze(10)) try: # Call replSetStepDown before killing the primary's process. # This raises OperationFailure if no secondaries are capable # of taking over. primary_server.connection.admin.command('replSetStepDown', 10) except pymongo.errors.AutoReconnect: # Have to stop the server as well so it doesn't get reelected. primary_server.stop() return True except pymongo.errors.OperationFailure: # No secondaries within 10 seconds of my optime... return False assert_eventually(freeze_and_stop, "Primary didn't step down.") assert_eventually( lambda: (self.rs.primary(repl_id)['mongodb_uri'] == next_primary), "Secondary did not freeze.", max_tries=120) assert_eventually( lambda: (self.rs.primary(repl_id)['mongodb_uri'] == self.rs. member_info(repl_id, 1)['mongodb_uri']), "Higher priority secondary never promoted.")
def test_router(self): config = {} self.sh = ShardedCluster(config) self.assertTrue(Servers().info(self.sh.router['id'])['statuses']['mongos']) self.sh.cleanup() config = {'routers': [{}, {}, {}]} self.sh = ShardedCluster(config) routers = self.sh.routers hostname = routers[1]['hostname'] _id = routers[1]['id'] # stop routers 0 and 2 Servers().command(routers[0]['id'], 'stop') Servers().command(routers[2]['id'], 'stop') router = self.sh.router self.assertEqual(router['id'], _id) self.assertEqual(router['hostname'], hostname) self.sh.cleanup()
def test_member_info_auth_uri(self): for i in range(len(self.repl)): member = self.repl.member_info(i) self.assertIn('mongodb_auth_uri', member) uri = member['mongodb_auth_uri'] host = Servers().hostname(member['server_id']) self.assertIn(host, uri) self.assertIn('admin:admin', uri) self.assertIn('authSource=admin', uri)
def _remove(self, shard_name): """remove member from configuration""" result = self.router_command("removeShard", shard_name, is_eval=False) if result['ok'] == 1 and result['state'] == 'completed': shard = self._shards.pop(shard_name) if shard.get('isServer', False): Servers().remove(shard['_id']) if shard.get('isReplicaSet', False): ReplicaSets().remove(shard['_id']) return result
def __init_configsvr(self, params): """create and start config servers""" self._configsvrs = [] for cfg in params: # Remove flags that turn on auth. cfg = self._strip_auth(cfg) server_id = cfg.pop('server_id', None) cfg.update({'configsvr': True}) self._configsvrs.append(Servers().create( 'mongod', cfg, sslParams=self.sslParams, autostart=True, version=self._version, server_id=server_id))
def setUp(self): PortPool().change_range() self.servers = Servers() self.servers.set_settings(*TEST_RELEASES) self.repl_cfg = { 'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}] } self.repl = ReplicaSet(self.repl_cfg)
def __init_configsvr(self, params): """create and start config servers""" self._configsvrs = [] for cfg in params: cfg.update({'configsvr': True}) self._configsvrs.append(Servers().create('mongod', cfg, sslParams=self.sslParams, autostart=True, auth_key=self.auth_key, version=self._version))
def setUp(self): PortPool().change_range() self.servers = Servers() self.servers.set_settings(os.environ.get('MONGOBIN', None)) self.repl_cfg = { 'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}] } self.repl = ReplicaSet(self.repl_cfg)
def test_primary_stepdown(self): # This tests Server, # but only makes sense in the context of a replica set. repl_id = self.rs.create({ 'id': 'test-rs-stepdown', 'members': [{}, {}, {}] }) primary = self.rs.primary(repl_id) primary_server = Servers()._storage[primary['server_id']] time.sleep(20) # No Exception. primary_server.stepdown() self.assertNotEqual(primary['mongodb_uri'], self.rs.primary(repl_id)['mongodb_uri'])
def setUp(self): PortPool().change_range() self.servers = Servers() self.servers.set_settings(*TEST_RELEASES) self.repl_cfg = { 'members': [{}, {}, { 'rsParams': { 'priority': 0, 'hidden': True } }, { 'rsParams': { 'arbiterOnly': True } }] }
def setUp(self): PortPool().change_range() self.servers = Servers() self.servers.set_settings(os.environ.get('MONGOBIN', None)) self.repl_cfg = { 'members': [{}, {}, { 'rsParams': { 'priority': 0, 'hidden': True } }, { 'rsParams': { 'arbiterOnly': True } }] }
def host_list(): logger.debug("host_list()") servers = [] for server_id in Servers(): server_info = {'id': server_id} server_info['links'] = all_server_links( server_id, rel_to='get-servers') servers.append(server_info) response = {'links': [ base_link('service'), base_link('get-releases'), server_link('get-servers', self_rel=True), server_link('add-server'), replica_set_link('get-replica-sets'), sharded_cluster_link('get-sharded-clusters') ]} response['servers'] = servers return send_result(200, response)
def test_info(self): config = { 'configsvrs': [{}, {}, {}], 'routers': [{}, {}, {}], 'shards': [{}, {}] } sh_id = self.sh.create(config) info = self.sh.info(sh_id) self.assertTrue(isinstance(info, dict)) for item in ("shards", "configsvrs", "routers", "mongodb_uri", "orchestration"): self.assertTrue(item in info) self.assertEqual(len(info['shards']), 2) self.assertEqual(len(info['configsvrs']), 3) self.assertEqual(len(info['routers']), 3) mongodb_uri = info['mongodb_uri'] for router in info['routers']: self.assertIn(Servers().hostname(router['id']), mongodb_uri) self.assertTrue(mongodb_uri.find('mongodb://') == 0) self.assertEqual(info['orchestration'], 'sharded_clusters')
def test_primary(self): self.repl_cfg = {'members': [{}, {}]} self.repl = ReplicaSet(self.repl_cfg) primary = self.repl.primary() server_id = Servers().host_to_server_id(primary) self.assertTrue(Servers().info(server_id)['statuses']['primary'])