def test_primary_stepdown(self):
# This tests Host, but only makes sense in the context of a replica set.
repl_id = self.rs.create(
{'id': 'test-rs-stepdown',
'members': [{}, {}, {"rsParams": {"priority": 1.4}}]})
primary = self.rs.primary(repl_id)
primary_host = Hosts()._storage[primary['host_id']]
# No Exception.
primary_host.stepdown()
self.assertNotEqual(primary['uri'], self.rs.primary(repl_id)['uri'])
def putInQueue(event, context):
# For demo purposes, this function is invoked manually
# Also for demo purposes, we will use a static list here
# We need to figure out a way to put stuff in the queue regularly
target_list = [
"www.mozilla.org",
"infosec.mozilla.org",
]
hosts = Hosts(target_list)
target = Target(hosts.next())
SQS_CLIENT.send_message(QueueUrl=os.getenv('SQS_URL'),
MessageBody="manual|" + target.name)
def test_with_hosts(self):
hostname_array = [
"foo.example.com",
"bar.example.com",
"baz.example.com"
]
hosts = Hosts(hostname_array)
assert type(hosts.next()) is str
assert type(hosts.next()) is str
assert type(hosts.next()) is str
assert type(hosts.next()) is str
assert type(hosts.next()) is str
def queue_scheduled(self, event, context, hostname_list=[]):
if len(hostname_list) == 0:
hosts = Hosts()
hostname_list = hosts.getList()
for hostname in hostname_list:
self.sqs_client.send_message(QueueUrl=self.queueURL,
DelaySeconds=2,
MessageBody="tlsobservatory|" +
hostname + "|")
self.logger.info(
"Tasking TLS observatory scan of: {}".format(hostname))
self.logger.info("Host list has been added to the queue "
"for TLS observatory scan.")
def test_member_freeze(self):
# This tests Host, but only makes sense in the context of a replica set.
repl_id = self.rs.create(
{'members': [{"rsParams": {"priority": 19}},
{"rsParams": {"priority": 5}}, {}]})
next_primary_info = self.rs.member_info(repl_id, 2)
next_primary = next_primary_info['uri']
secondary_info = self.rs.member_info(repl_id, 1)
secondary_host = Hosts()._storage[secondary_info['host_id']]
primary_info = self.rs.member_info(repl_id, 0)
primary_host = Hosts()._storage[primary_info['host_id']]
self.assertTrue(secondary_host.freeze(10))
primary_host.stop()
self.assertEqual(self.rs.primary(repl_id)['uri'], next_primary)
time.sleep(12)
self.assertEqual(self.rs.primary(repl_id)['uri'],
self.rs.member_info(repl_id, 1)['uri'])
def test_with_hosts_for_host_fixing(self):
hostname_array = [
"foo.example.com",
"bar.example.com",
"baz.example.com",
"foo2.example.com",
"bar2.example.com",
"baz2.example.com"
]
hosts = Hosts(hostname_array)
# Make 6 selections from next()
selected_hosts = []
selected_hosts.append(hosts.next())
selected_hosts.append(hosts.next())
selected_hosts.append(hosts.next())
selected_hosts.append(hosts.next())
selected_hosts.append(hosts.next())
selected_hosts.append(hosts.next())
# check the len of a uniq'd selection to
# make sure we're not stuck on the same host
assert len(list(set(selected_hosts))) > 1
class ReplicaSetAuthTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.hosts = Hosts()
self.hosts.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
def tearDown(self):
if len(self.repl) > 0:
self.repl.cleanup()
def test_auth_connection(self):
self.assertTrue(isinstance(self.repl.connection().admin.collection_names(), list))
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_auth_admin(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
self.assertTrue(isinstance(c.admin.collection_names(), list))
self.assertTrue(c.admin.logout() is None)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_auth_collection(self):
c = pymongo.MongoReplicaSetClient(self.repl.primary(), replicaSet=self.repl.repl_id)
self.assertTrue(c.admin.authenticate('admin', 'admin'))
db = c.test_auth
db.add_user('user', 'userpass', roles=['readWrite'])
c.admin.logout()
self.assertTrue(db.authenticate('user', 'userpass'))
self.assertTrue(db.foo.insert({'foo': 'bar'}, safe=True, w=2, wtimeout=1000))
self.assertTrue(isinstance(db.foo.find_one(), dict))
db.logout()
self.assertRaises(pymongo.errors.OperationFailure, db.foo.find_one)
def setUp(self):
PortPool().change_range()
self.path = tempfile.mktemp(prefix="test-storage")
self.hosts = Hosts()
self.hosts.set_settings(os.environ.get('MONGOBIN', ""))
class HostsTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.path = tempfile.mktemp(prefix="test-storage")
self.hosts = Hosts()
self.hosts.set_settings(os.environ.get('MONGOBIN', ""))
def remove_path(self, path):
onerror = lambda func, filepath, exc_info: (os.chmod(filepath, stat.S_IWUSR), func(filepath))
if os.path.isfile(path):
try:
os.remove(path)
except OSError:
time.sleep(2)
onerror(os.remove, path, None)
def tearDown(self):
self.hosts.cleanup()
self.remove_path(self.path)
def test_singleton(self):
self.assertEqual(id(self.hosts), id(Hosts()))
def test_set_settings(self):
path = os.path.join(os.getcwd(), 'bin')
self.hosts.set_settings(path)
self.assertEqual(path, self.hosts.bin_path)
def test_bool(self):
self.assertEqual(False, bool(self.hosts))
self.hosts.create('mongod', {}, autostart=False)
self.assertTrue(True, bool(self.hosts))
def test_operations(self):
host_id = self.hosts.create('mongod', {}, autostart=False)
self.assertTrue(len(self.hosts) == 1)
self.assertTrue(host_id in self.hosts)
host_id2 = 'host-id2'
host2 = Host(os.path.join(os.environ.get('MONGOBIN', ''), 'mongod'), {})
host2.start(30)
host2_pid = host2.info()['procInfo']['pid']
self.hosts[host_id2] = host2
self.assertTrue(self.hosts[host_id2]['procInfo']['pid'] == host2_pid)
self.assertTrue(host_id2 in self.hosts)
for h_id in self.hosts:
self.assertTrue(h_id in (host_id, host_id2))
operator.delitem(self.hosts, host_id2)
self.assertFalse(host_id2 in self.hosts)
host2.stop()
host2.cleanup()
def test_cleanup(self):
self.hosts.create('mongod', {}, autostart=False)
self.hosts.create('mongod', {}, autostart=True)
self.assertTrue(len(self.hosts) == 2)
self.hosts.cleanup()
self.assertTrue(len(self.hosts) == 0)
def test_new_host(self):
self.assertTrue(len(self.hosts) == 0)
host_id = self.hosts.create('mongod', {}, autostart=False)
info = self.hosts.info(host_id)
self.assertTrue(len(self.hosts) == 1)
self.assertNotIn('pid', info['procInfo'])
host_id2 = self.hosts.create('mongod', {}, autostart=True)
info = self.hosts.info(host_id2)
self.assertTrue(info['procInfo']['pid'] > 0)
self.assertRaises(OSError, self.hosts.create, 'fake_process_', {})
def test_new_host_with_auth(self):
host_id = self.hosts.create('mongod', {}, login='******', password='******', autostart=True)
hostname = self.hosts.hostname(host_id)
c = pymongo.MongoClient(hostname)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
self.assertTrue(c.admin.authenticate('adminko', 'password'))
self.assertTrue(isinstance(c.admin.collection_names(), list))
self.assertTrue(c.admin.logout() is None)
self.assertRaises(pymongo.errors.OperationFailure, c.admin.collection_names)
def test_hdel(self):
self.assertEqual(len(self.hosts), 0)
h_id = self.hosts.create('mongod', {}, autostart=True)
self.assertEqual(len(self.hosts), 1)
h_info = self.hosts.info(h_id)['procInfo']
self.assertTrue(os.path.exists(h_info['params']['dbpath']))
self.assertTrue(os.path.exists(h_info['optfile']))
self.hosts.remove(h_id)
self.assertEqual(len(self.hosts), 0) # check length
# check cleanup
self.assertFalse(os.path.exists(h_info['params']['dbpath']))
self.assertFalse(os.path.exists(h_info['optfile']))
def test_hcommand(self):
h_id = self.hosts.create('mongod', {}, autostart=False)
self.assertTrue(self.hosts.command(h_id, 'start'))
self.assertTrue(self.hosts.command(h_id, 'stop'))
self.assertTrue(self.hosts.command(h_id, 'start'))
self.assertTrue(self.hosts.command(h_id, 'restart'))
self.assertRaises(ValueError, self.hosts.command, h_id, 'fake')
def test_hinfo(self):
h_id = self.hosts.create('mongod', {}, autostart=False)
info = self.hosts.info(h_id)
self.assertEqual(info['id'], h_id)
self.assertNotIn('pid', info['procInfo'])
self.assertEqual(info['statuses'], {})
self.assertEqual(info['serverInfo'], {})
def test_id_by_hostname(self):
h_id = self.hosts.create('mongod', {}, autostart=True)
h_uri = self.hosts.info(h_id)['uri']
h2_id = self.hosts.create('mongod', {}, autostart=True)
h2_uri = self.hosts.info(h2_id)['uri']
self.assertTrue(self.hosts.id_by_hostname(h_uri) == h_id)
self.assertTrue(self.hosts.id_by_hostname(h2_uri) == h2_id)
def test_hostname(self):
h_id = self.hosts.create('mongod', {}, autostart=True)
h_uri = self.hosts.info(h_id)['uri']
self.assertEqual(self.hosts.hostname(h_id), h_uri)
def test_is_alive(self):
h_id = self.hosts.create('mongod', {}, autostart=True)
self.assertEqual(self.hosts.is_alive(h_id), True)
self.hosts.command(h_id, 'stop')
self.assertEqual(self.hosts.is_alive(h_id), False)
def test_db_command(self):
h_id = self.hosts.create('mongod', {}, autostart=False)
self.assertRaises(pymongo.errors.PyMongoError, self.hosts.db_command, h_id, 'serverStatus', None, False)
self.hosts.command(h_id, 'start', 10)
self.assertEqual(self.hosts.db_command(h_id, 'serverStatus', arg=None, is_eval=False).get('ok', -1), 1)
self.assertEqual(self.hosts.db_command(h_id, 'db.getName()', arg=None, is_eval=True), 'admin')
def test_id_specified(self):
id = 'xyzzy'
h_id = self.hosts.create('mongod', {}, autostart=False, host_id=id)
self.assertEqual(id, h_id)
data['id'] = host_id
return _host_create(data)
@route('/servers/<host_id>', method='DELETE')
# TODO: return 400 code if process failed
@error_wrap
def host_del(host_id):
logger.debug("host_del({host_id})")
if host_id not in Hosts():
return send_result(404)
Hosts().remove(host_id)
return send_result(204)
@route('/servers/<host_id>', method='POST')
@error_wrap
def host_command(host_id):
logger.debug("host_command({host_id})".format(**locals()))
if host_id not in Hosts():
return send_result(404)
command = json.loads(request.body.read())['action']
Hosts().command(host_id, command)
return send_result(200)
if __name__ == '__main__':
hs = Hosts()
hs.set_settings()
run(host='localhost', port=8889, debug=True, reloader=False)
def test_with_no_hosts(self):
hosts = Hosts()
assert hosts.next() is None
def setUp(self):
PortPool().change_range()
self.hosts = Hosts()
self.hosts.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'auth_key': 'secret', 'login': '******', 'password': '******', 'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
def setUp(self):
PortPool().change_range()
self.hosts = Hosts()
self.hosts.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}, {'rsParams': {'arbiterOnly': True}}]}
class ReplicaSetTestCase(unittest.TestCase):
def setUp(self):
PortPool().change_range()
self.hosts = Hosts()
self.hosts.set_settings(os.environ.get('MONGOBIN', None))
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}, {'rsParams': {'arbiterOnly': True}}]}
# self.repl = ReplicaSet(self.repl_cfg)
def tearDown(self):
if hasattr(self, 'repl'):
self.repl.cleanup()
def test_len(self):
self.repl = ReplicaSet(self.repl_cfg)
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']))
self.repl.member_del(3)
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']) - 1)
self.repl.repl_member_add({'rsParams': {'arbiterOnly': True}})
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']))
def test_cleanup(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
self.assertTrue(len(self.repl) == len(self.repl_cfg['members']))
self.repl.cleanup()
self.assertTrue(len(self.repl) == 0)
def test_id2host(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
members = self.repl.config['members']
for member in members:
self.assertTrue(member['host'] == self.repl.id2host(member['_id']))
def test_host2id(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
members = self.repl.config['members']
for member in members:
self.assertTrue(member['_id'] == self.repl.host2id(member['host']))
def test_update_host_map(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
origin = self.repl.host_map.copy()
self.repl.update_host_map(self.repl.config)
self.assertEqual(self.repl.host_map, origin)
def test_repl_update(self):
self.repl_cfg = {'members': [{}, {}, {'rsParams': {'priority': 0, 'hidden': True}}]}
self.repl = ReplicaSet(self.repl_cfg)
config = self.repl.config
config['members'][1]['priority'] = 0
config['members'][1]['hidden'] = True
self.assertTrue(self.repl.repl_update(config))
self.assertTrue(self.repl.config['members'][1]['hidden'])
def test_info(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
cfg = self.repl.config
info = self.repl.info()
self.assertEqual(info['auth_key'], self.repl.auth_key)
self.assertEqual(info['id'], self.repl.repl_id)
self.assertEqual(len(info['members']), len(cfg['members']))
members1 = sorted(cfg['members'], key=lambda item: item['_id'])
members2 = sorted(info['members'], key=lambda item: item['_id'])
for i in range(len(members1)):
self.assertEqual(members1[i]['_id'], members2[i]['_id'])
self.assertEqual(members1[i]['host'], members2[i]['host'])
def test_repl_member_add(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
member_id = self.repl.repl_member_add({"rsParams": {"priority": 0, "hidden": True}})
self.assertTrue(member_id >= 0)
member = [item for item in self.repl.config['members'] if item['_id'] == member_id][0]
self.assertTrue(member['hidden'])
def test_run_command(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
self.assertEqual(self.repl.run_command("rs.status()", is_eval=True)['ok'], 1)
result = self.repl.run_command('serverStatus', arg=None, is_eval=False, member_id=0)['repl']
for key in ('me', 'ismaster', 'setName', 'primary', 'hosts'):
self.assertTrue(key in result)
self.assertEqual(self.repl.run_command(command="replSetGetStatus", is_eval=False)['ok'], 1)
def test_config(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
config = self.repl.config
self.assertTrue('_id' in config)
self.assertTrue('members' in config)
def test_member_create(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
result = self.repl.member_create({}, 13)
self.assertTrue('host' in result)
self.assertTrue('_id' in result)
h_id = Hosts().id_by_hostname(result['host'])
h_info = Hosts().info(h_id)
self.assertEqual(result['host'], h_info['uri'])
self.assertTrue(h_info['procInfo']['alive'])
Hosts().remove(h_id)
def test_member_del(self):
self.repl_cfg = {'members': [{}, {}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
m_count = len(self.repl.config['members'])
self.assertTrue(self.repl.member_del(2))
self.assertEqual(len(self.repl.config['members']), m_count - 1)
def test_member_del_no_reconfig(self):
self.repl_cfg = {'members': [{}, {}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
m_count = len(self.repl.config['members'])
self.assertTrue(self.repl.member_del(2, reconfig=False))
self.assertEqual(len(self.repl.config['members']), m_count)
self.repl.host_map.pop(2)
def test_member_update(self):
self.repl = ReplicaSet(self.repl_cfg)
member = [item for item in self.repl.config['members'] if item['_id'] == 2][0]
self.assertTrue(member.get('hidden', False))
self.assertTrue(self.repl.member_update(2, {"rsParams": {"priority": 1, "hidden": False}}))
member = [item for item in self.repl.config['members'] if item['_id'] == 2][0]
self.assertFalse(member.get('hidden', False))
def test_member_info(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
member = [item for item in self.repl.config['members'] if item['_id'] == 1][0]
result = self.repl.member_info(1)
self.assertTrue(result['procInfo']['alive'])
self.assertEqual(member['host'], result['uri'])
self.assertTrue(len(result['rsInfo']) > 0)
def test_member_command(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
_id = 1
self.assertTrue(self.repl.member_info(_id)['procInfo']['alive'])
self.repl.member_command(_id, 'stop')
self.assertFalse(self.repl.member_info(_id)['procInfo']['alive'])
self.repl.member_command(_id, 'start')
self.assertTrue(self.repl.member_info(_id)['procInfo']['alive'])
self.repl.member_command(_id, 'restart')
self.assertTrue(self.repl.member_info(_id)['procInfo']['alive'])
def test_member_freeze(self):
self.repl_cfg = {"members": [{"rsParams": {"priority": 19}}, {"rsParams": {"priority": 7}}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primary_next = self.repl.member_info(2)['uri']
self.assertTrue(self.repl.member_freeze(1, 30))
self.repl.member_command(0, 'stop')
self.assertEqual(self.repl.primary(), primary_next)
time.sleep(40)
self.assertEqual(self.repl.primary(), self.repl.member_info(1)['uri'])
def test_members(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
members1 = sorted(self.repl.config['members'], key=lambda item: item['_id'])
members2 = sorted(self.repl.members(), key=lambda item: item['_id'])
self.assertEqual(len(members1), len(members2))
for i in xrange(len(members1)):
self.assertEqual(members1[i]['host'], members2[i]['host'])
self.assertEqual(members1[i]['_id'], members2[i]['_id'])
def test_stepdown(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primary = self.repl.primary()
self.assertTrue(self.repl.stepdown())
self.assertNotEqual(primary, self.repl.primary())
def test_primary(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primary = self.repl.primary()
self.assertTrue(Hosts().info(Hosts().id_by_hostname(primary))['statuses']['primary'])
def test_get_members_in_state(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
primaries = self.repl.get_members_in_state(1)
self.assertEqual(len(primaries), 1)
self.assertEqual(primaries[0], self.repl.primary())
def test_connection(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
_id = 1
hostname = self.repl.id2host(_id)
self.assertTrue(self.repl.connection(timeout=5))
self.assertTrue(self.repl.connection(hostname=hostname, timeout=5))
self.repl.member_command(_id, 'stop')
self.assertRaises(pymongo.errors.AutoReconnect, lambda: self.repl.connection(hostname=hostname, timeout=5))
def test_secondaries(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
secondaries = [item['host'] for item in self.repl.secondaries()]
self.assertEqual(secondaries, self.repl.get_members_in_state(2))
def test_arbiters(self):
self.repl = ReplicaSet(self.repl_cfg)
arbiters = [item['host'] for item in self.repl.arbiters()]
self.assertEqual(arbiters, self.repl.get_members_in_state(7))
def test_hidden(self):
self.repl = ReplicaSet(self.repl_cfg)
for _ in self.repl.hidden():
self.assertTrue(self.repl.run_command('serverStatus', arg=None, is_eval=False, member_id=2)['repl']['hidden'])
def test_passives(self):
self.repl = ReplicaSet(self.repl_cfg)
self.repl.repl_member_add({"rsParams": {"priority": 0}})
for member in self.repl.passives():
self.assertTrue(member['host'] in self.repl.run_command('isMaster', is_eval=False).get('passives'))
def test_hosts(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
self.repl.repl_member_add({"rsParams": {"priority": 0}})
for member in self.repl.hosts():
self.assertTrue(member['host'] in self.repl.run_command('isMaster', is_eval=False).get('hosts'))
def test_compare_hosts_passives(self):
self.repl = ReplicaSet(self.repl_cfg)
self.repl.repl_member_add({"rsParams": {"priority": 0}})
self.repl.repl_member_add({})
hosts = self.repl.hosts()
passives = self.repl.passives()
for item in hosts:
self.assertTrue(item not in passives)
for item in passives:
self.assertTrue(item not in hosts)
def test_wait_while_reachable(self):
self.repl_cfg = {'members': [{}, {}]}
self.repl = ReplicaSet(self.repl_cfg)
hosts = [member['host'] for member in self.repl.members()]
self.assertTrue(self.repl.wait_while_reachable(hosts, timeout=10))
self.repl.member_command(1, 'stop')
self.assertFalse(self.repl.wait_while_reachable(hosts, timeout=10))