class TestPuppetMaster(Base):
def __init__(self, *args, **kwargs):
super(TestPuppetMaster, self).__init__(*args, **kwargs)
self.remote = SSHClient()
def setUp(self):
logging.info('Admin node ip: %s' % self.get_admin_node_ip())
self.ip = self.get_admin_node_ip()
def tearDown(self):
pass
def test_puppetmaster_alive(self):
wait(
lambda: tcp_ping(self.ip, 8140),
timeout=5
)
self.remote.connect_ssh(self.ip, 'root', 'r00tme')
ps_output = self.remote.execute('ps ax')['stdout']
pm_processes = filter(
lambda x: '/usr/sbin/puppetmasterd' in x,
ps_output
)
logging.debug("Found puppet master processes: %s" % pm_processes)
self.assertEquals(len(pm_processes), 4)
def test_network_config(self):
self._revert_nodes()
self._clean_clusters()
self._basic_provisioning('network_config', {'controller': ['slave1']})
slave = ci.environment.node['slave1']
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles)
ifaces_fail = False
for iface in node['network_data']:
try:
ifname = "%s.%s@%s" % (
iface['dev'], iface['vlan'], iface['dev']
)
ifname_short = "%s.%s" % (iface['dev'], iface['vlan'])
except KeyError:
ifname = iface['dev']
iface_data = ''.join(
ctrl_ssh.execute(
'/sbin/ip addr show dev %s' % ifname_short
)['stdout']
)
if iface_data.find(ifname) == -1:
logging.error("Interface %s is absent" % ifname_short)
ifaces_fail = True
else:
try:
if iface_data.find("inet %s" % iface['ip']) == -1:
logging.error(
"Interface %s does not have ip %s" % (
ifname_short, iface['ip']
)
)
ifaces_fail = True
except KeyError:
if iface_data.find("inet ") != -1:
logging.error(
"Interface %s does have ip. And it should not" %
ifname_short
)
ifaces_fail = True
try:
if iface_data.find("brd %s" % iface['brd']) == -1:
logging.error(
"Interface %s does not have broadcast %s" % (
ifname_short, iface['brd']
)
)
ifaces_fail = True
except KeyError:
pass
self.assertEquals(ifaces_fail, False)
def _check_cluster_status(self, ip, smiles_count, networks_count=1):
logging.info("Checking cluster status: ip=%s smiles=%s networks=%s",
ip, smiles_count, networks_count)
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
ctrl_ssh = SSHClient()
ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles)
ret = ctrl_ssh.execute('/usr/bin/nova-manage service list')
nova_status = (
(ret['exit_status'] == 0)
and (''.join(ret['stdout']).count(":-)") == smiles_count)
and (''.join(ret['stdout']).count("XXX") == 0)
)
if not nova_status:
logging.warn("Nova check fails:\n%s" % ret['stdout'])
ret = ctrl_ssh.execute('. /root/openrc; glance index')
cirros_status = (
(ret['exit_status'] == 0)
and (''.join(ret['stdout']).count("TestVM") == 1)
)
if not cirros_status:
logging.warn("Cirros check fails:\n%s" % ret['stdout'])
ret = ctrl_ssh.execute('/usr/bin/nova-manage network list')
nets_status = (
(ret['exit_status'] == 0)
and (len(ret['stdout']) == networks_count + 1)
)
if not nets_status:
logging.warn("Networks check fails:\n%s" % ret['stdout'])
return (
nova_status and
cirros_status and
nets_status and
self._status_logserver()
)
class TestNode(Base):
def __init__(self, *args, **kwargs):
super(TestNode, self).__init__(*args, **kwargs)
self.remote = SSHClient()
self.client = HTTPClient(url="http://%s:8000" %
self.get_admin_node_ip())
self.ssh_user = "******"
self.ssh_passwd = "r00tme"
self.admin_host = self.get_admin_node_ip()
self.remote.connect_ssh(self.admin_host, self.ssh_user,
self.ssh_passwd)
def setUp(self):
pass
def tearDown(self):
self._wait_for_threads()
try:
self._stop_logserver()
except AttributeError:
pass
def _start_logserver(self, handler=None):
self._logserver_status = False
if not handler:
"""
We define log message handler in such a way
assuming that if at least one message is received
logging works fine.
"""
def handler(message):
self._logserver_status = True
self.logserver = LogServer(address=self.get_host_node_ip(), port=5514)
self.logserver.set_handler(handler)
self.logserver.start()
def _stop_logserver(self):
self.logserver.stop()
self._logserver_status = False
def _status_logserver(self):
return self._logserver_status
def test_release_upload(self):
self._upload_sample_release()
def test_http_returns_200(self):
resp = self.client.get("/")
self.assertEquals(200, resp.getcode())
def test_create_empty_cluster(self):
self._create_cluster(name='empty')
@snapshot_errors
def test_node_deploy(self):
self._revert_nodes()
self._bootstrap_nodes(['slave1'])
@snapshot_errors
def test_updating_nodes_in_cluster(self):
self._revert_nodes()
cluster_id = self._create_cluster(name='empty')
nodes = self._bootstrap_nodes(['slave1'])
self._update_nodes_in_cluster(cluster_id, nodes)
@snapshot_errors
def test_one_node_provisioning(self):
self._revert_nodes()
self._clean_clusters()
self._basic_provisioning('provision', {'controller': ['slave1']})
@snapshot_errors
def test_simple_cluster_flat(self):
logging.info("Testing simple flat installation.")
self._revert_nodes()
cluster_name = 'simple_flat'
nodes = {'controller': ['slave1'], 'compute': ['slave2']}
cluster_id = self._basic_provisioning(cluster_name, nodes)
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 5), timeout=300)
logging.info("Verifying networks for simple flat installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
for vlan in vlans:
for n in (slave, slave2):
self._restore_vlan_in_ebtables(n.interfaces[0].target_dev,
vlan, False)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network simple flat', 60 * 2)
@snapshot_errors
def test_simple_cluster_vlan(self):
logging.info("Testing simple vlan installation.")
self._revert_nodes()
cluster_name = 'simple_vlan'
nodes = {'controller': ['slave1'], 'compute': ['slave2']}
self._create_cluster(name=cluster_name, net_manager="VlanManager")
cluster_id = self._basic_provisioning(cluster_name, nodes)
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 5, 8), timeout=300)
logging.info("Verifying networks for simple vlan installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
for vlan in vlans:
for n in (slave, slave2):
self._restore_vlan_in_ebtables(n.interfaces[0].target_dev,
vlan, False)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network simple vlan', 60 * 2)
@snapshot_errors
def test_ha_cluster_flat(self):
logging.info("Testing ha flat installation.")
self._revert_nodes()
cluster_name = 'ha_flat'
nodes = {
'controller': ['slave1', 'slave2', 'slave3'],
'compute': ['slave4', 'slave5']
}
cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60)
logging.info("Checking cluster status on slave1")
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 13), timeout=300)
logging.info("Verifying networks for ha flat installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
slave3 = ci.environment.node['slave3']
slave4 = ci.environment.node['slave4']
slave5 = ci.environment.node['slave5']
for vlan in vlans:
for n in (slave, slave2, slave3, slave4, slave5):
self._restore_vlan_in_ebtables(n.interfaces[0].target_dev,
vlan, False)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network ha flat', 60 * 2)
@snapshot_errors
def test_ha_cluster_vlan(self):
logging.info("Testing ha vlan installation.")
self._revert_nodes()
cluster_name = 'ha_vlan'
nodes = {
'controller': ['slave1', 'slave2', 'slave3'],
'compute': ['slave4', 'slave5']
}
self._create_cluster(name=cluster_name, net_manager="VlanManager")
cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60)
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 13, 8),
timeout=300)
logging.info("Verifying networks for ha vlan installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
slave3 = ci.environment.node['slave3']
slave4 = ci.environment.node['slave4']
slave5 = ci.environment.node['slave5']
for vlan in vlans:
for n in (slave, slave2, slave3, slave4, slave5):
self._restore_vlan_in_ebtables(n.interfaces[0].target_dev,
vlan, False)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network ha vlan', 60 * 2)
@snapshot_errors
def test_network_config(self):
self._revert_nodes()
self._clean_clusters()
self._basic_provisioning('network_config', {'controller': ['slave1']})
slave = ci.environment.node['slave1']
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles)
ifaces_fail = False
for iface in node['network_data']:
try:
ifname = "%s.%s@%s" % (iface['dev'], iface['vlan'],
iface['dev'])
ifname_short = "%s.%s" % (iface['dev'], iface['vlan'])
except KeyError:
ifname = iface['dev']
iface_data = ''.join(
ctrl_ssh.execute('/sbin/ip addr show dev %s' %
ifname_short)['stdout'])
if iface_data.find(ifname) == -1:
logging.error("Interface %s is absent" % ifname_short)
ifaces_fail = True
else:
try:
if iface_data.find("inet %s" % iface['ip']) == -1:
logging.error("Interface %s does not have ip %s" %
(ifname_short, iface['ip']))
ifaces_fail = True
except KeyError:
if iface_data.find("inet ") != -1:
logging.error(
"Interface %s does have ip. And it should not" %
ifname_short)
ifaces_fail = True
try:
if iface_data.find("brd %s" % iface['brd']) == -1:
logging.error(
"Interface %s does not have broadcast %s" %
(ifname_short, iface['brd']))
ifaces_fail = True
except KeyError:
pass
self.assertEquals(ifaces_fail, False)
@snapshot_errors
def test_node_deletion(self):
self._revert_nodes()
cluster_name = 'node_deletion'
node_name = 'slave1'
nodes = {'controller': [node_name]}
cluster_id = self._basic_provisioning(cluster_name, nodes)
slave = ci.environment.node[node_name]
node = self._get_slave_node_by_devops_node(slave)
self.client.put("/api/nodes/%s/" % node['id'],
{'pending_deletion': True})
task = self._launch_provisioning(cluster_id)
self._task_wait(task, 'Node deletion', 60)
timer = time.time()
timeout = 5 * 60
while True:
response = self.client.get("/api/nodes/")
nodes = json.loads(response.read())
for n in nodes:
if (n['mac'] == node['mac'] and n['status'] == 'discover'):
return
if (time.time() - timer) > timeout:
raise Exception("Bootstrap boot timeout expired!")
time.sleep(5)
@snapshot_errors
def test_network_verify_with_blocked_vlan(self):
self._revert_nodes()
cluster_name = 'net_verify'
cluster_id = self._create_cluster(name=cluster_name)
node_names = ['slave1', 'slave2']
nailgun_slave_nodes = self._bootstrap_nodes(node_names)
devops_nodes = [ci.environment.node[n] for n in node_names]
logging.info("Clear BROUTING table entries.")
vlans = self._get_cluster_vlans(cluster_id)
for vlan in vlans:
for node in devops_nodes:
for interface in node.interfaces:
self._restore_vlan_in_ebtables(interface.target_dev, vlan,
False)
self._update_nodes_in_cluster(cluster_id, nailgun_slave_nodes)
for node in devops_nodes:
for interface in node.interfaces:
self._block_vlan_in_ebtables(interface.target_dev, vlans[0])
task = self._run_network_verify(cluster_id)
task = self._task_wait(task,
'Verify network in cluster with blocked vlan',
60 * 2, True)
self.assertEquals(task['status'], 'error')
@snapshot_errors
def test_multinic_bootstrap_booting(self):
self._revert_nodes()
slave = filter(lambda n: n.name != 'admin' and len(n.interfaces) > 1,
ci.environment.nodes)[0]
nodename = slave.name
logging.info("Using node %r with %d interfaces", nodename,
len(slave.interfaces))
slave.stop()
admin = ci.environment.node['admin']
macs = [i.mac_address for i in slave.interfaces]
logging.info("Block all MACs: %s.", ', '.join([m for m in macs]))
for mac in macs:
self._block_mac_in_ebtables(mac)
self.addCleanup(self._restore_mac_in_ebtables, mac)
for mac in macs:
logging.info("Trying to boot node %r via interface with MAC %s...",
nodename, mac)
self._restore_mac_in_ebtables(mac)
slave.start()
nailgun_slave = self._bootstrap_nodes([nodename])[0]
self.assertEqual(mac.upper(), nailgun_slave['mac'].upper())
slave.stop()
admin.restore_snapshot('initial')
self._block_mac_in_ebtables(mac)
@staticmethod
def _block_mac_in_ebtables(mac):
try:
subprocess.check_output(
'sudo ebtables -t filter -A FORWARD -s %s -j DROP' % mac,
stderr=subprocess.STDOUT,
shell=True)
logging.debug("MAC %s blocked via ebtables.", mac)
except subprocess.CalledProcessError as e:
raise Exception("Can't block mac %s via ebtables: %s", mac,
e.output)
@staticmethod
def _restore_mac_in_ebtables(mac):
try:
subprocess.check_output(
'sudo ebtables -t filter -D FORWARD -s %s -j DROP' % mac,
stderr=subprocess.STDOUT,
shell=True)
logging.debug("MAC %s unblocked via ebtables.", mac)
except subprocess.CalledProcessError as e:
logging.warn("Can't restore mac %s via ebtables: %s", mac,
e.output)
def _block_vlan_in_ebtables(self, target_dev, vlan):
try:
subprocess.check_output(
'sudo ebtables -t broute -A BROUTING -i %s -p 8021Q'
' --vlan-id %s -j DROP' % (target_dev, vlan),
stderr=subprocess.STDOUT,
shell=True)
self.addCleanup(self._restore_vlan_in_ebtables, target_dev, vlan)
logging.debug("Vlan %s on interface %s blocked via ebtables.",
vlan, target_dev)
except subprocess.CalledProcessError as e:
raise Exception("Can't block vlan %s for interface %s"
" via ebtables: %s" % (vlan, target_dev, e.output))
def _get_cluster_vlans(self, cluster_id):
resp = self.client.get('/api/clusters/%d/network_configuration/' %
cluster_id)
self.assertEquals(200, resp.getcode())
cluster_vlans = []
networks = json.loads(resp.read())['networks']
for n in networks:
amount = n.get('amount', 1)
cluster_vlans.extend(
range(n['vlan_start'], n['vlan_start'] + amount))
self.assertNotEqual(cluster_vlans, [])
return cluster_vlans
@staticmethod
def _restore_vlan_in_ebtables(target_dev, vlan, log=True):
try:
subprocess.check_output(
'sudo ebtables -t broute -D BROUTING -i %s -p 8021Q'
' --vlan-id %s -j DROP' % (target_dev, vlan),
stderr=subprocess.STDOUT,
shell=True)
logging.debug("Vlan %s on interface %s unblocked via ebtables.",
vlan, target_dev)
except subprocess.CalledProcessError as e:
if log:
logging.warn("Can't restore vlan %s for interface %s"
" via ebtables: %s" %
(vlan, target_dev, e.output))
def _run_network_verify(self, cluster_id):
logging.info("Run network verify in cluster %d", cluster_id)
resp = self.client.get('/api/clusters/%d/network_configuration/' %
cluster_id)
self.assertEquals(200, resp.getcode())
network_configuration = json.loads(resp.read())
changes = self.client.put(
'/api/clusters/%d/network_configuration/verify/' % cluster_id,
network_configuration)
self.assertEquals(200, changes.getcode())
return json.loads(changes.read())
def _basic_provisioning(self,
cluster_name,
nodes_dict,
task_timeout=30 * 60):
self._start_logserver()
self._clean_clusters()
cluster_id = self._create_cluster(name=cluster_name)
# Here we updating cluster editable attributes
# In particular we set extra syslog server
response = self.client.get("/api/clusters/%s/attributes/" % cluster_id)
attrs = json.loads(response.read())
attrs["editable"]["syslog"]["syslog_server"]["value"] = \
self.get_host_node_ip()
attrs["editable"]["syslog"]["syslog_port"]["value"] = \
self.logserver.bound_port()
self.client.put("/api/clusters/%s/attributes/" % cluster_id, attrs)
node_names = []
for role in nodes_dict:
node_names += nodes_dict[role]
try:
if len(node_names) > 1:
if len(nodes_dict['controller']) == 1:
self.client.put("/api/clusters/%s/" % cluster_id,
{"mode": "multinode"})
if len(nodes_dict['controller']) > 1:
self.client.put("/api/clusters/%s/" % cluster_id,
{"mode": "ha"})
except KeyError:
pass
nodes = self._bootstrap_nodes(node_names)
for role in nodes_dict:
for n in nodes_dict[role]:
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
self.client.put("/api/nodes/%s/" % node['id'], {
"role": role,
"pending_addition": True
})
self._update_nodes_in_cluster(cluster_id, nodes)
task = self._launch_provisioning(cluster_id)
self._task_wait(task, 'Installation', task_timeout)
logging.info("Checking role files on slave nodes")
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
for role in nodes_dict:
for n in nodes_dict[role]:
logging.info("Checking /tmp/%s-file on %s" % (role, n))
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
for i in node['meta']['interfaces']:
ip = i.get('ip', None)
if ip:
logging.debug("Trying to connect to %s via ssh", ip)
try:
ctrl_ssh.connect_ssh(ip,
'root',
key_filename=keyfiles)
except Exception, e:
logging.debug("Unable to connect to %s: %s", ip,
str(e))
continue
ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role)
self.assertEquals(
ret['exit_status'], 0,
("File '/tmp/%s-file' not found" % role))
ctrl_ssh.disconnect()
break
else:
i_name = i.get('name') or i.get('mac') or str(i)
logging.debug("Interface doesn't have an IP: %r",
i_name)
self.assertNotEqual(
ip, None, "Unable to fing a valid IP"
" for node %s" % n)
return cluster_id
def _basic_provisioning(self,
cluster_name,
nodes_dict,
task_timeout=30 * 60):
self._start_logserver()
self._clean_clusters()
cluster_id = self._create_cluster(name=cluster_name)
# Here we updating cluster editable attributes
# In particular we set extra syslog server
response = self.client.get("/api/clusters/%s/attributes/" % cluster_id)
attrs = json.loads(response.read())
attrs["editable"]["syslog"]["syslog_server"]["value"] = \
self.get_host_node_ip()
attrs["editable"]["syslog"]["syslog_port"]["value"] = \
self.logserver.bound_port()
self.client.put("/api/clusters/%s/attributes/" % cluster_id, attrs)
node_names = []
for role in nodes_dict:
node_names += nodes_dict[role]
try:
if len(node_names) > 1:
if len(nodes_dict['controller']) == 1:
self.client.put("/api/clusters/%s/" % cluster_id,
{"mode": "multinode"})
if len(nodes_dict['controller']) > 1:
self.client.put("/api/clusters/%s/" % cluster_id,
{"mode": "ha"})
except KeyError:
pass
nodes = self._bootstrap_nodes(node_names)
for role in nodes_dict:
for n in nodes_dict[role]:
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
self.client.put("/api/nodes/%s/" % node['id'], {
"role": role,
"pending_addition": True
})
self._update_nodes_in_cluster(cluster_id, nodes)
task = self._launch_provisioning(cluster_id)
self._task_wait(task, 'Installation', task_timeout)
logging.info("Checking role files on slave nodes")
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
for role in nodes_dict:
for n in nodes_dict[role]:
logging.info("Checking /tmp/%s-file on %s" % (role, n))
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
for i in node['meta']['interfaces']:
ip = i.get('ip', None)
if ip:
logging.debug("Trying to connect to %s via ssh", ip)
try:
ctrl_ssh.connect_ssh(ip,
'root',
key_filename=keyfiles)
except Exception, e:
logging.debug("Unable to connect to %s: %s", ip,
str(e))
continue
ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role)
self.assertEquals(
ret['exit_status'], 0,
("File '/tmp/%s-file' not found" % role))
ctrl_ssh.disconnect()
break
else:
i_name = i.get('name') or i.get('mac') or str(i)
logging.debug("Interface doesn't have an IP: %r",
i_name)
self.assertNotEqual(
ip, None, "Unable to fing a valid IP"
" for node %s" % n)
class TestNode(Base):
def __init__(self, *args, **kwargs):
super(TestNode, self).__init__(*args, **kwargs)
self.remote = SSHClient()
self.client = HTTPClient(
url="http://%s:8000" % self.get_admin_node_ip()
)
self.ssh_user = "******"
self.ssh_passwd = "r00tme"
self.admin_host = self.get_admin_node_ip()
self.remote.connect_ssh(
self.admin_host,
self.ssh_user,
self.ssh_passwd
)
def setUp(self):
pass
def tearDown(self):
self._wait_for_threads()
try:
self._stop_logserver()
except AttributeError:
pass
def _start_logserver(self, handler=None):
self._logserver_status = False
if not handler:
"""
We define log message handler in such a way
assuming that if at least one message is received
logging works fine.
"""
def handler(message):
self._logserver_status = True
self.logserver = LogServer(
address=self.get_host_node_ip(),
port=5514
)
self.logserver.set_handler(handler)
self.logserver.start()
def _stop_logserver(self):
self.logserver.stop()
self._logserver_status = False
def _status_logserver(self):
return self._logserver_status
def test_release_upload(self):
self._upload_sample_release()
def test_http_returns_200(self):
resp = self.client.get("/")
self.assertEquals(200, resp.getcode())
def test_create_empty_cluster(self):
self._create_cluster(name='empty')
@snapshot_errors
def test_node_deploy(self):
self._revert_nodes()
self._bootstrap_nodes(['slave1'])
@snapshot_errors
def test_updating_nodes_in_cluster(self):
self._revert_nodes()
cluster_id = self._create_cluster(name='empty')
nodes = self._bootstrap_nodes(['slave1'])
self._update_nodes_in_cluster(cluster_id, nodes)
@snapshot_errors
def test_one_node_provisioning(self):
self._revert_nodes()
self._clean_clusters()
self._basic_provisioning('provision', {'controller': ['slave1']})
@snapshot_errors
def test_simple_cluster_flat(self):
logging.info("Testing simple flat installation.")
self._revert_nodes()
cluster_name = 'simple_flat'
nodes = {'controller': ['slave1'], 'compute': ['slave2']}
cluster_id = self._basic_provisioning(cluster_name, nodes)
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 5), timeout=300)
logging.info("Verifying networks for simple flat installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
for vlan in vlans:
for n in (slave, slave2):
self._restore_vlan_in_ebtables(
n.interfaces[0].target_dev,
vlan,
False
)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network simple flat', 60 * 2)
@snapshot_errors
def test_simple_cluster_vlan(self):
logging.info("Testing simple vlan installation.")
self._revert_nodes()
cluster_name = 'simple_vlan'
nodes = {'controller': ['slave1'], 'compute': ['slave2']}
self._create_cluster(name=cluster_name, net_manager="VlanManager")
cluster_id = self._basic_provisioning(cluster_name, nodes)
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 5, 8), timeout=300)
logging.info("Verifying networks for simple vlan installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
for vlan in vlans:
for n in (slave, slave2):
self._restore_vlan_in_ebtables(
n.interfaces[0].target_dev,
vlan,
False
)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network simple vlan', 60 * 2)
@snapshot_errors
def test_ha_cluster_flat(self):
logging.info("Testing ha flat installation.")
self._revert_nodes()
cluster_name = 'ha_flat'
nodes = {
'controller': ['slave1', 'slave2', 'slave3'],
'compute': ['slave4', 'slave5']
}
cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60)
logging.info("Checking cluster status on slave1")
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(lambda: self._check_cluster_status(node['ip'], 13), timeout=300)
logging.info("Verifying networks for ha flat installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
slave3 = ci.environment.node['slave3']
slave4 = ci.environment.node['slave4']
slave5 = ci.environment.node['slave5']
for vlan in vlans:
for n in (slave, slave2, slave3, slave4, slave5):
self._restore_vlan_in_ebtables(
n.interfaces[0].target_dev,
vlan,
False
)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network ha flat', 60 * 2)
@snapshot_errors
def test_ha_cluster_vlan(self):
logging.info("Testing ha vlan installation.")
self._revert_nodes()
cluster_name = 'ha_vlan'
nodes = {
'controller': ['slave1', 'slave2', 'slave3'],
'compute': ['slave4', 'slave5']
}
self._create_cluster(name=cluster_name, net_manager="VlanManager")
cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60)
slave = ci.environment.node['slave1']
node = self._get_slave_node_by_devops_node(slave)
wait(
lambda: self._check_cluster_status(node['ip'], 13, 8),
timeout=300
)
logging.info("Verifying networks for ha vlan installation.")
vlans = self._get_cluster_vlans(cluster_id)
slave2 = ci.environment.node['slave2']
slave3 = ci.environment.node['slave3']
slave4 = ci.environment.node['slave4']
slave5 = ci.environment.node['slave5']
for vlan in vlans:
for n in (slave, slave2, slave3, slave4, slave5):
self._restore_vlan_in_ebtables(
n.interfaces[0].target_dev,
vlan,
False
)
task = self._run_network_verify(cluster_id)
self._task_wait(task, 'Verify network ha vlan', 60 * 2)
@snapshot_errors
def test_network_config(self):
self._revert_nodes()
self._clean_clusters()
self._basic_provisioning('network_config', {'controller': ['slave1']})
slave = ci.environment.node['slave1']
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles)
ifaces_fail = False
for iface in node['network_data']:
try:
ifname = "%s.%s@%s" % (
iface['dev'], iface['vlan'], iface['dev']
)
ifname_short = "%s.%s" % (iface['dev'], iface['vlan'])
except KeyError:
ifname = iface['dev']
iface_data = ''.join(
ctrl_ssh.execute(
'/sbin/ip addr show dev %s' % ifname_short
)['stdout']
)
if iface_data.find(ifname) == -1:
logging.error("Interface %s is absent" % ifname_short)
ifaces_fail = True
else:
try:
if iface_data.find("inet %s" % iface['ip']) == -1:
logging.error(
"Interface %s does not have ip %s" % (
ifname_short, iface['ip']
)
)
ifaces_fail = True
except KeyError:
if iface_data.find("inet ") != -1:
logging.error(
"Interface %s does have ip. And it should not" %
ifname_short
)
ifaces_fail = True
try:
if iface_data.find("brd %s" % iface['brd']) == -1:
logging.error(
"Interface %s does not have broadcast %s" % (
ifname_short, iface['brd']
)
)
ifaces_fail = True
except KeyError:
pass
self.assertEquals(ifaces_fail, False)
@snapshot_errors
def test_node_deletion(self):
self._revert_nodes()
cluster_name = 'node_deletion'
node_name = 'slave1'
nodes = {'controller': [node_name]}
cluster_id = self._basic_provisioning(cluster_name, nodes)
slave = ci.environment.node[node_name]
node = self._get_slave_node_by_devops_node(slave)
self.client.put("/api/nodes/%s/" % node['id'],
{'pending_deletion': True})
task = self._launch_provisioning(cluster_id)
self._task_wait(task, 'Node deletion', 60)
timer = time.time()
timeout = 5 * 60
while True:
response = self.client.get("/api/nodes/")
nodes = json.loads(response.read())
for n in nodes:
if (n['mac'] == node['mac'] and n['status'] == 'discover'):
return
if (time.time() - timer) > timeout:
raise Exception("Bootstrap boot timeout expired!")
time.sleep(5)
@snapshot_errors
def test_network_verify_with_blocked_vlan(self):
self._revert_nodes()
cluster_name = 'net_verify'
cluster_id = self._create_cluster(name=cluster_name)
node_names = ['slave1', 'slave2']
nailgun_slave_nodes = self._bootstrap_nodes(node_names)
devops_nodes = [ci.environment.node[n] for n in node_names]
logging.info("Clear BROUTING table entries.")
vlans = self._get_cluster_vlans(cluster_id)
for vlan in vlans:
for node in devops_nodes:
for interface in node.interfaces:
self._restore_vlan_in_ebtables(interface.target_dev,
vlan, False)
self._update_nodes_in_cluster(cluster_id, nailgun_slave_nodes)
for node in devops_nodes:
for interface in node.interfaces:
self._block_vlan_in_ebtables(interface.target_dev, vlans[0])
task = self._run_network_verify(cluster_id)
task = self._task_wait(task,
'Verify network in cluster with blocked vlan',
60 * 2, True)
self.assertEquals(task['status'], 'error')
@snapshot_errors
def test_multinic_bootstrap_booting(self):
self._revert_nodes()
slave = filter(lambda n: n.name != 'admin' and len(n.interfaces) > 1,
ci.environment.nodes)[0]
nodename = slave.name
logging.info("Using node %r with %d interfaces", nodename,
len(slave.interfaces))
slave.stop()
admin = ci.environment.node['admin']
macs = [i.mac_address for i in slave.interfaces]
logging.info("Block all MACs: %s.",
', '.join([m for m in macs]))
for mac in macs:
self._block_mac_in_ebtables(mac)
self.addCleanup(self._restore_mac_in_ebtables, mac)
for mac in macs:
logging.info("Trying to boot node %r via interface with MAC %s...",
nodename, mac)
self._restore_mac_in_ebtables(mac)
slave.start()
nailgun_slave = self._bootstrap_nodes([nodename])[0]
self.assertEqual(mac.upper(), nailgun_slave['mac'].upper())
slave.stop()
admin.restore_snapshot('initial')
self._block_mac_in_ebtables(mac)
@staticmethod
def _block_mac_in_ebtables(mac):
try:
subprocess.check_output(
'sudo ebtables -t filter -A FORWARD -s %s -j DROP' % mac,
stderr=subprocess.STDOUT,
shell=True
)
logging.debug("MAC %s blocked via ebtables.", mac)
except subprocess.CalledProcessError as e:
raise Exception("Can't block mac %s via ebtables: %s",
mac, e.output)
@staticmethod
def _restore_mac_in_ebtables(mac):
try:
subprocess.check_output(
'sudo ebtables -t filter -D FORWARD -s %s -j DROP' % mac,
stderr=subprocess.STDOUT,
shell=True
)
logging.debug("MAC %s unblocked via ebtables.", mac)
except subprocess.CalledProcessError as e:
logging.warn("Can't restore mac %s via ebtables: %s",
mac, e.output)
def _block_vlan_in_ebtables(self, target_dev, vlan):
try:
subprocess.check_output(
'sudo ebtables -t broute -A BROUTING -i %s -p 8021Q'
' --vlan-id %s -j DROP' % (
target_dev, vlan
),
stderr=subprocess.STDOUT,
shell=True
)
self.addCleanup(self._restore_vlan_in_ebtables,
target_dev, vlan)
logging.debug("Vlan %s on interface %s blocked via ebtables.",
vlan, target_dev)
except subprocess.CalledProcessError as e:
raise Exception("Can't block vlan %s for interface %s"
" via ebtables: %s" %
(vlan, target_dev, e.output))
def _get_cluster_vlans(self, cluster_id):
resp = self.client.get(
'/api/clusters/%d/network_configuration/' % cluster_id)
self.assertEquals(200, resp.getcode())
cluster_vlans = []
networks = json.loads(resp.read())['networks']
for n in networks:
amount = n.get('amount', 1)
cluster_vlans.extend(range(n['vlan_start'],
n['vlan_start'] + amount))
self.assertNotEqual(cluster_vlans, [])
return cluster_vlans
@staticmethod
def _restore_vlan_in_ebtables(target_dev, vlan, log=True):
try:
subprocess.check_output(
'sudo ebtables -t broute -D BROUTING -i %s -p 8021Q'
' --vlan-id %s -j DROP' % (
target_dev, vlan
),
stderr=subprocess.STDOUT,
shell=True
)
logging.debug("Vlan %s on interface %s unblocked via ebtables.",
vlan, target_dev)
except subprocess.CalledProcessError as e:
if log:
logging.warn("Can't restore vlan %s for interface %s"
" via ebtables: %s" %
(vlan, target_dev, e.output))
def _run_network_verify(self, cluster_id):
logging.info(
"Run network verify in cluster %d",
cluster_id
)
resp = self.client.get(
'/api/clusters/%d/network_configuration/' % cluster_id)
self.assertEquals(200, resp.getcode())
network_configuration = json.loads(resp.read())
changes = self.client.put(
'/api/clusters/%d/network_configuration/verify/' % cluster_id,
network_configuration
)
self.assertEquals(200, changes.getcode())
return json.loads(changes.read())
def _basic_provisioning(self, cluster_name, nodes_dict,
task_timeout=30 * 60):
self._start_logserver()
self._clean_clusters()
cluster_id = self._create_cluster(name=cluster_name)
# Here we updating cluster editable attributes
# In particular we set extra syslog server
response = self.client.get(
"/api/clusters/%s/attributes/" % cluster_id
)
attrs = json.loads(response.read())
attrs["editable"]["syslog"]["syslog_server"]["value"] = \
self.get_host_node_ip()
attrs["editable"]["syslog"]["syslog_port"]["value"] = \
self.logserver.bound_port()
self.client.put(
"/api/clusters/%s/attributes/" % cluster_id,
attrs
)
node_names = []
for role in nodes_dict:
node_names += nodes_dict[role]
try:
if len(node_names) > 1:
if len(nodes_dict['controller']) == 1:
self.client.put(
"/api/clusters/%s/" % cluster_id,
{"mode": "multinode"}
)
if len(nodes_dict['controller']) > 1:
self.client.put(
"/api/clusters/%s/" % cluster_id,
{"mode": "ha"}
)
except KeyError:
pass
nodes = self._bootstrap_nodes(node_names)
for role in nodes_dict:
for n in nodes_dict[role]:
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
self.client.put(
"/api/nodes/%s/" % node['id'],
{"role": role, "pending_addition": True}
)
self._update_nodes_in_cluster(cluster_id, nodes)
task = self._launch_provisioning(cluster_id)
self._task_wait(task, 'Installation', task_timeout)
logging.info("Checking role files on slave nodes")
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
for role in nodes_dict:
for n in nodes_dict[role]:
logging.info("Checking /tmp/%s-file on %s" % (role, n))
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
for i in node['meta']['interfaces']:
ip = i.get('ip', None)
if ip:
logging.debug("Trying to connect to %s via ssh", ip)
try:
ctrl_ssh.connect_ssh(ip, 'root',
key_filename=keyfiles)
except Exception, e:
logging.debug("Unable to connect to %s: %s", ip,
str(e))
continue
ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role)
self.assertEquals(ret['exit_status'], 0,
("File '/tmp/%s-file' not found" %
role))
ctrl_ssh.disconnect()
break
else:
i_name = i.get('name') or i.get('mac') or str(i)
logging.debug("Interface doesn't have an IP: %r",
i_name)
self.assertNotEqual(ip, None, "Unable to fing a valid IP"
" for node %s" % n)
return cluster_id
def _basic_provisioning(self, cluster_name, nodes_dict,
task_timeout=30 * 60):
self._start_logserver()
self._clean_clusters()
cluster_id = self._create_cluster(name=cluster_name)
# Here we updating cluster editable attributes
# In particular we set extra syslog server
response = self.client.get(
"/api/clusters/%s/attributes/" % cluster_id
)
attrs = json.loads(response.read())
attrs["editable"]["syslog"]["syslog_server"]["value"] = \
self.get_host_node_ip()
attrs["editable"]["syslog"]["syslog_port"]["value"] = \
self.logserver.bound_port()
self.client.put(
"/api/clusters/%s/attributes/" % cluster_id,
attrs
)
node_names = []
for role in nodes_dict:
node_names += nodes_dict[role]
try:
if len(node_names) > 1:
if len(nodes_dict['controller']) == 1:
self.client.put(
"/api/clusters/%s/" % cluster_id,
{"mode": "multinode"}
)
if len(nodes_dict['controller']) > 1:
self.client.put(
"/api/clusters/%s/" % cluster_id,
{"mode": "ha"}
)
except KeyError:
pass
nodes = self._bootstrap_nodes(node_names)
for role in nodes_dict:
for n in nodes_dict[role]:
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
self.client.put(
"/api/nodes/%s/" % node['id'],
{"role": role, "pending_addition": True}
)
self._update_nodes_in_cluster(cluster_id, nodes)
task = self._launch_provisioning(cluster_id)
self._task_wait(task, 'Installation', task_timeout)
logging.info("Checking role files on slave nodes")
keyfiles = ci.environment.node['admin'].metadata['keyfiles']
for role in nodes_dict:
for n in nodes_dict[role]:
logging.info("Checking /tmp/%s-file on %s" % (role, n))
slave = ci.environment.node[n]
node = self._get_slave_node_by_devops_node(slave)
ctrl_ssh = SSHClient()
for i in node['meta']['interfaces']:
ip = i.get('ip', None)
if ip:
logging.debug("Trying to connect to %s via ssh", ip)
try:
ctrl_ssh.connect_ssh(ip, 'root',
key_filename=keyfiles)
except Exception, e:
logging.debug("Unable to connect to %s: %s", ip,
str(e))
continue
ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role)
self.assertEquals(ret['exit_status'], 0,
("File '/tmp/%s-file' not found" %
role))
ctrl_ssh.disconnect()
break
else:
i_name = i.get('name') or i.get('mac') or str(i)
logging.debug("Interface doesn't have an IP: %r",
i_name)
self.assertNotEqual(ip, None, "Unable to fing a valid IP"
" for node %s" % n)