class TestPuppetMaster(Base): def __init__(self, *args, **kwargs): super(TestPuppetMaster, self).__init__(*args, **kwargs) self.remote = SSHClient() def setUp(self): logging.info('Admin node ip: %s' % self.get_admin_node_ip()) self.ip = self.get_admin_node_ip() def tearDown(self): pass def test_puppetmaster_alive(self): wait( lambda: tcp_ping(self.ip, 8140), timeout=5 ) self.remote.connect_ssh(self.ip, 'root', 'r00tme') ps_output = self.remote.execute('ps ax')['stdout'] pm_processes = filter( lambda x: '/usr/sbin/puppetmasterd' in x, ps_output ) logging.debug("Found puppet master processes: %s" % pm_processes) self.assertEquals(len(pm_processes), 4)
def __init__(self, *args, **kwargs): super(TestNode, self).__init__(*args, **kwargs) self.remote = SSHClient() self.client = HTTPClient(url="http://%s:8000" % self.get_admin_node_ip()) self.ssh_user = "******" self.ssh_passwd = "r00tme" self.admin_host = self.get_admin_node_ip() self.remote.connect_ssh(self.admin_host, self.ssh_user, self.ssh_passwd)
def test_network_config(self): self._revert_nodes() self._clean_clusters() self._basic_provisioning('network_config', {'controller': ['slave1']}) slave = ci.environment.node['slave1'] keyfiles = ci.environment.node['admin'].metadata['keyfiles'] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles) ifaces_fail = False for iface in node['network_data']: try: ifname = "%s.%s@%s" % ( iface['dev'], iface['vlan'], iface['dev'] ) ifname_short = "%s.%s" % (iface['dev'], iface['vlan']) except KeyError: ifname = iface['dev'] iface_data = ''.join( ctrl_ssh.execute( '/sbin/ip addr show dev %s' % ifname_short )['stdout'] ) if iface_data.find(ifname) == -1: logging.error("Interface %s is absent" % ifname_short) ifaces_fail = True else: try: if iface_data.find("inet %s" % iface['ip']) == -1: logging.error( "Interface %s does not have ip %s" % ( ifname_short, iface['ip'] ) ) ifaces_fail = True except KeyError: if iface_data.find("inet ") != -1: logging.error( "Interface %s does have ip. And it should not" % ifname_short ) ifaces_fail = True try: if iface_data.find("brd %s" % iface['brd']) == -1: logging.error( "Interface %s does not have broadcast %s" % ( ifname_short, iface['brd'] ) ) ifaces_fail = True except KeyError: pass self.assertEquals(ifaces_fail, False)
def _check_cluster_status(self, ip, smiles_count, networks_count=1): logging.info("Checking cluster status: ip=%s smiles=%s networks=%s", ip, smiles_count, networks_count) keyfiles = ci.environment.node['admin'].metadata['keyfiles'] ctrl_ssh = SSHClient() ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles) ret = ctrl_ssh.execute('/usr/bin/nova-manage service list') nova_status = ((ret['exit_status'] == 0) and (''.join(ret['stdout']).count(":-)") == smiles_count) and (''.join(ret['stdout']).count("XXX") == 0)) if not nova_status: logging.warn("Nova check fails:\n%s" % ret['stdout']) ret = ctrl_ssh.execute('. /root/openrc; glance index') cirros_status = ((ret['exit_status'] == 0) and (''.join(ret['stdout']).count("TestVM") == 1)) if not cirros_status: logging.warn("Cirros check fails:\n%s" % ret['stdout']) ret = ctrl_ssh.execute('/usr/bin/nova-manage network list') nets_status = ((ret['exit_status'] == 0) and (len(ret['stdout']) == networks_count + 1)) if not nets_status: logging.warn("Networks check fails:\n%s" % ret['stdout']) return (nova_status and cirros_status and nets_status and self._status_logserver())
def test_network_config(self): self._revert_nodes() self._clean_clusters() self._basic_provisioning('network_config', {'controller': ['slave1']}) slave = ci.environment.node['slave1'] keyfiles = ci.environment.node['admin'].metadata['keyfiles'] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles) ifaces_fail = False for iface in node['network_data']: try: ifname = "%s.%s@%s" % (iface['dev'], iface['vlan'], iface['dev']) ifname_short = "%s.%s" % (iface['dev'], iface['vlan']) except KeyError: ifname = iface['dev'] iface_data = ''.join( ctrl_ssh.execute('/sbin/ip addr show dev %s' % ifname_short)['stdout']) if iface_data.find(ifname) == -1: logging.error("Interface %s is absent" % ifname_short) ifaces_fail = True else: try: if iface_data.find("inet %s" % iface['ip']) == -1: logging.error("Interface %s does not have ip %s" % (ifname_short, iface['ip'])) ifaces_fail = True except KeyError: if iface_data.find("inet ") != -1: logging.error( "Interface %s does have ip. And it should not" % ifname_short) ifaces_fail = True try: if iface_data.find("brd %s" % iface['brd']) == -1: logging.error( "Interface %s does not have broadcast %s" % (ifname_short, iface['brd'])) ifaces_fail = True except KeyError: pass self.assertEquals(ifaces_fail, False)
def __init__(self, *args, **kwargs): super(TestNode, self).__init__(*args, **kwargs) self.remote = SSHClient() self.client = HTTPClient( url="http://%s:8000" % self.get_admin_node_ip() ) self.ssh_user = "******" self.ssh_passwd = "r00tme" self.admin_host = self.get_admin_node_ip() self.remote.connect_ssh( self.admin_host, self.ssh_user, self.ssh_passwd )
class TestNailyd(Base): def __init__(self, *args, **kwargs): super(TestNailyd, self).__init__(*args, **kwargs) self.remote = SSHClient() def setUp(self): logging.info('Admin node ip: %s' % self.get_admin_node_ip()) self.ip = self.get_admin_node_ip() def tearDown(self): pass def test_nailyd_alive(self): self.remote.connect_ssh(self.ip, 'root', 'r00tme') ps_output = self.remote.execute('ps ax')['stdout'] naily_processes = filter(lambda x: 'naily master' in x, ps_output) logging.debug("Found %d naily master processes: %s" % (len(naily_processes), naily_processes)) self.assertEqual(1, len(naily_processes)) naily_processes = filter(lambda x: 'naily worker' in x, ps_output) logging.debug("Found %d naily worker processes: %s" % (len(naily_processes), naily_processes)) self.assertEqual(True, len(naily_processes) > 1)
def _check_cluster_status(self, ip, smiles_count, networks_count=1): logging.info("Checking cluster status: ip=%s smiles=%s networks=%s", ip, smiles_count, networks_count) keyfiles = ci.environment.node['admin'].metadata['keyfiles'] ctrl_ssh = SSHClient() ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles) ret = ctrl_ssh.execute('/usr/bin/nova-manage service list') nova_status = ( (ret['exit_status'] == 0) and (''.join(ret['stdout']).count(":-)") == smiles_count) and (''.join(ret['stdout']).count("XXX") == 0) ) if not nova_status: logging.warn("Nova check fails:\n%s" % ret['stdout']) ret = ctrl_ssh.execute('. /root/openrc; glance index') cirros_status = ( (ret['exit_status'] == 0) and (''.join(ret['stdout']).count("TestVM") == 1) ) if not cirros_status: logging.warn("Cirros check fails:\n%s" % ret['stdout']) ret = ctrl_ssh.execute('/usr/bin/nova-manage network list') nets_status = ( (ret['exit_status'] == 0) and (len(ret['stdout']) == networks_count + 1) ) if not nets_status: logging.warn("Networks check fails:\n%s" % ret['stdout']) return ( nova_status and cirros_status and nets_status and self._status_logserver() )
class TestNode(Base): def __init__(self, *args, **kwargs): super(TestNode, self).__init__(*args, **kwargs) self.remote = SSHClient() self.client = HTTPClient(url="http://%s:8000" % self.get_admin_node_ip()) self.ssh_user = "******" self.ssh_passwd = "r00tme" self.admin_host = self.get_admin_node_ip() self.remote.connect_ssh(self.admin_host, self.ssh_user, self.ssh_passwd) def setUp(self): pass def tearDown(self): self._wait_for_threads() try: self._stop_logserver() except AttributeError: pass def _start_logserver(self, handler=None): self._logserver_status = False if not handler: """ We define log message handler in such a way assuming that if at least one message is received logging works fine. """ def handler(message): self._logserver_status = True self.logserver = LogServer(address=self.get_host_node_ip(), port=5514) self.logserver.set_handler(handler) self.logserver.start() def _stop_logserver(self): self.logserver.stop() self._logserver_status = False def _status_logserver(self): return self._logserver_status def test_release_upload(self): self._upload_sample_release() def test_http_returns_200(self): resp = self.client.get("/") self.assertEquals(200, resp.getcode()) def test_create_empty_cluster(self): self._create_cluster(name='empty') @snapshot_errors def test_node_deploy(self): self._revert_nodes() self._bootstrap_nodes(['slave1']) @snapshot_errors def test_updating_nodes_in_cluster(self): self._revert_nodes() cluster_id = self._create_cluster(name='empty') nodes = self._bootstrap_nodes(['slave1']) self._update_nodes_in_cluster(cluster_id, nodes) @snapshot_errors def test_one_node_provisioning(self): self._revert_nodes() self._clean_clusters() self._basic_provisioning('provision', {'controller': ['slave1']}) @snapshot_errors def test_simple_cluster_flat(self): logging.info("Testing simple flat installation.") self._revert_nodes() cluster_name = 'simple_flat' nodes = {'controller': ['slave1'], 'compute': ['slave2']} cluster_id = self._basic_provisioning(cluster_name, nodes) slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 5), timeout=300) logging.info("Verifying networks for simple flat installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] for vlan in vlans: for n in (slave, slave2): self._restore_vlan_in_ebtables(n.interfaces[0].target_dev, vlan, False) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network simple flat', 60 * 2) @snapshot_errors def test_simple_cluster_vlan(self): logging.info("Testing simple vlan installation.") self._revert_nodes() cluster_name = 'simple_vlan' nodes = {'controller': ['slave1'], 'compute': ['slave2']} self._create_cluster(name=cluster_name, net_manager="VlanManager") cluster_id = self._basic_provisioning(cluster_name, nodes) slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 5, 8), timeout=300) logging.info("Verifying networks for simple vlan installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] for vlan in vlans: for n in (slave, slave2): self._restore_vlan_in_ebtables(n.interfaces[0].target_dev, vlan, False) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network simple vlan', 60 * 2) @snapshot_errors def test_ha_cluster_flat(self): logging.info("Testing ha flat installation.") self._revert_nodes() cluster_name = 'ha_flat' nodes = { 'controller': ['slave1', 'slave2', 'slave3'], 'compute': ['slave4', 'slave5'] } cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60) logging.info("Checking cluster status on slave1") slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 13), timeout=300) logging.info("Verifying networks for ha flat installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] slave3 = ci.environment.node['slave3'] slave4 = ci.environment.node['slave4'] slave5 = ci.environment.node['slave5'] for vlan in vlans: for n in (slave, slave2, slave3, slave4, slave5): self._restore_vlan_in_ebtables(n.interfaces[0].target_dev, vlan, False) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network ha flat', 60 * 2) @snapshot_errors def test_ha_cluster_vlan(self): logging.info("Testing ha vlan installation.") self._revert_nodes() cluster_name = 'ha_vlan' nodes = { 'controller': ['slave1', 'slave2', 'slave3'], 'compute': ['slave4', 'slave5'] } self._create_cluster(name=cluster_name, net_manager="VlanManager") cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60) slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 13, 8), timeout=300) logging.info("Verifying networks for ha vlan installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] slave3 = ci.environment.node['slave3'] slave4 = ci.environment.node['slave4'] slave5 = ci.environment.node['slave5'] for vlan in vlans: for n in (slave, slave2, slave3, slave4, slave5): self._restore_vlan_in_ebtables(n.interfaces[0].target_dev, vlan, False) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network ha vlan', 60 * 2) @snapshot_errors def test_network_config(self): self._revert_nodes() self._clean_clusters() self._basic_provisioning('network_config', {'controller': ['slave1']}) slave = ci.environment.node['slave1'] keyfiles = ci.environment.node['admin'].metadata['keyfiles'] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles) ifaces_fail = False for iface in node['network_data']: try: ifname = "%s.%s@%s" % (iface['dev'], iface['vlan'], iface['dev']) ifname_short = "%s.%s" % (iface['dev'], iface['vlan']) except KeyError: ifname = iface['dev'] iface_data = ''.join( ctrl_ssh.execute('/sbin/ip addr show dev %s' % ifname_short)['stdout']) if iface_data.find(ifname) == -1: logging.error("Interface %s is absent" % ifname_short) ifaces_fail = True else: try: if iface_data.find("inet %s" % iface['ip']) == -1: logging.error("Interface %s does not have ip %s" % (ifname_short, iface['ip'])) ifaces_fail = True except KeyError: if iface_data.find("inet ") != -1: logging.error( "Interface %s does have ip. And it should not" % ifname_short) ifaces_fail = True try: if iface_data.find("brd %s" % iface['brd']) == -1: logging.error( "Interface %s does not have broadcast %s" % (ifname_short, iface['brd'])) ifaces_fail = True except KeyError: pass self.assertEquals(ifaces_fail, False) @snapshot_errors def test_node_deletion(self): self._revert_nodes() cluster_name = 'node_deletion' node_name = 'slave1' nodes = {'controller': [node_name]} cluster_id = self._basic_provisioning(cluster_name, nodes) slave = ci.environment.node[node_name] node = self._get_slave_node_by_devops_node(slave) self.client.put("/api/nodes/%s/" % node['id'], {'pending_deletion': True}) task = self._launch_provisioning(cluster_id) self._task_wait(task, 'Node deletion', 60) timer = time.time() timeout = 5 * 60 while True: response = self.client.get("/api/nodes/") nodes = json.loads(response.read()) for n in nodes: if (n['mac'] == node['mac'] and n['status'] == 'discover'): return if (time.time() - timer) > timeout: raise Exception("Bootstrap boot timeout expired!") time.sleep(5) @snapshot_errors def test_network_verify_with_blocked_vlan(self): self._revert_nodes() cluster_name = 'net_verify' cluster_id = self._create_cluster(name=cluster_name) node_names = ['slave1', 'slave2'] nailgun_slave_nodes = self._bootstrap_nodes(node_names) devops_nodes = [ci.environment.node[n] for n in node_names] logging.info("Clear BROUTING table entries.") vlans = self._get_cluster_vlans(cluster_id) for vlan in vlans: for node in devops_nodes: for interface in node.interfaces: self._restore_vlan_in_ebtables(interface.target_dev, vlan, False) self._update_nodes_in_cluster(cluster_id, nailgun_slave_nodes) for node in devops_nodes: for interface in node.interfaces: self._block_vlan_in_ebtables(interface.target_dev, vlans[0]) task = self._run_network_verify(cluster_id) task = self._task_wait(task, 'Verify network in cluster with blocked vlan', 60 * 2, True) self.assertEquals(task['status'], 'error') @snapshot_errors def test_multinic_bootstrap_booting(self): self._revert_nodes() slave = filter(lambda n: n.name != 'admin' and len(n.interfaces) > 1, ci.environment.nodes)[0] nodename = slave.name logging.info("Using node %r with %d interfaces", nodename, len(slave.interfaces)) slave.stop() admin = ci.environment.node['admin'] macs = [i.mac_address for i in slave.interfaces] logging.info("Block all MACs: %s.", ', '.join([m for m in macs])) for mac in macs: self._block_mac_in_ebtables(mac) self.addCleanup(self._restore_mac_in_ebtables, mac) for mac in macs: logging.info("Trying to boot node %r via interface with MAC %s...", nodename, mac) self._restore_mac_in_ebtables(mac) slave.start() nailgun_slave = self._bootstrap_nodes([nodename])[0] self.assertEqual(mac.upper(), nailgun_slave['mac'].upper()) slave.stop() admin.restore_snapshot('initial') self._block_mac_in_ebtables(mac) @staticmethod def _block_mac_in_ebtables(mac): try: subprocess.check_output( 'sudo ebtables -t filter -A FORWARD -s %s -j DROP' % mac, stderr=subprocess.STDOUT, shell=True) logging.debug("MAC %s blocked via ebtables.", mac) except subprocess.CalledProcessError as e: raise Exception("Can't block mac %s via ebtables: %s", mac, e.output) @staticmethod def _restore_mac_in_ebtables(mac): try: subprocess.check_output( 'sudo ebtables -t filter -D FORWARD -s %s -j DROP' % mac, stderr=subprocess.STDOUT, shell=True) logging.debug("MAC %s unblocked via ebtables.", mac) except subprocess.CalledProcessError as e: logging.warn("Can't restore mac %s via ebtables: %s", mac, e.output) def _block_vlan_in_ebtables(self, target_dev, vlan): try: subprocess.check_output( 'sudo ebtables -t broute -A BROUTING -i %s -p 8021Q' ' --vlan-id %s -j DROP' % (target_dev, vlan), stderr=subprocess.STDOUT, shell=True) self.addCleanup(self._restore_vlan_in_ebtables, target_dev, vlan) logging.debug("Vlan %s on interface %s blocked via ebtables.", vlan, target_dev) except subprocess.CalledProcessError as e: raise Exception("Can't block vlan %s for interface %s" " via ebtables: %s" % (vlan, target_dev, e.output)) def _get_cluster_vlans(self, cluster_id): resp = self.client.get('/api/clusters/%d/network_configuration/' % cluster_id) self.assertEquals(200, resp.getcode()) cluster_vlans = [] networks = json.loads(resp.read())['networks'] for n in networks: amount = n.get('amount', 1) cluster_vlans.extend( range(n['vlan_start'], n['vlan_start'] + amount)) self.assertNotEqual(cluster_vlans, []) return cluster_vlans @staticmethod def _restore_vlan_in_ebtables(target_dev, vlan, log=True): try: subprocess.check_output( 'sudo ebtables -t broute -D BROUTING -i %s -p 8021Q' ' --vlan-id %s -j DROP' % (target_dev, vlan), stderr=subprocess.STDOUT, shell=True) logging.debug("Vlan %s on interface %s unblocked via ebtables.", vlan, target_dev) except subprocess.CalledProcessError as e: if log: logging.warn("Can't restore vlan %s for interface %s" " via ebtables: %s" % (vlan, target_dev, e.output)) def _run_network_verify(self, cluster_id): logging.info("Run network verify in cluster %d", cluster_id) resp = self.client.get('/api/clusters/%d/network_configuration/' % cluster_id) self.assertEquals(200, resp.getcode()) network_configuration = json.loads(resp.read()) changes = self.client.put( '/api/clusters/%d/network_configuration/verify/' % cluster_id, network_configuration) self.assertEquals(200, changes.getcode()) return json.loads(changes.read()) def _basic_provisioning(self, cluster_name, nodes_dict, task_timeout=30 * 60): self._start_logserver() self._clean_clusters() cluster_id = self._create_cluster(name=cluster_name) # Here we updating cluster editable attributes # In particular we set extra syslog server response = self.client.get("/api/clusters/%s/attributes/" % cluster_id) attrs = json.loads(response.read()) attrs["editable"]["syslog"]["syslog_server"]["value"] = \ self.get_host_node_ip() attrs["editable"]["syslog"]["syslog_port"]["value"] = \ self.logserver.bound_port() self.client.put("/api/clusters/%s/attributes/" % cluster_id, attrs) node_names = [] for role in nodes_dict: node_names += nodes_dict[role] try: if len(node_names) > 1: if len(nodes_dict['controller']) == 1: self.client.put("/api/clusters/%s/" % cluster_id, {"mode": "multinode"}) if len(nodes_dict['controller']) > 1: self.client.put("/api/clusters/%s/" % cluster_id, {"mode": "ha"}) except KeyError: pass nodes = self._bootstrap_nodes(node_names) for role in nodes_dict: for n in nodes_dict[role]: slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) self.client.put("/api/nodes/%s/" % node['id'], { "role": role, "pending_addition": True }) self._update_nodes_in_cluster(cluster_id, nodes) task = self._launch_provisioning(cluster_id) self._task_wait(task, 'Installation', task_timeout) logging.info("Checking role files on slave nodes") keyfiles = ci.environment.node['admin'].metadata['keyfiles'] for role in nodes_dict: for n in nodes_dict[role]: logging.info("Checking /tmp/%s-file on %s" % (role, n)) slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() for i in node['meta']['interfaces']: ip = i.get('ip', None) if ip: logging.debug("Trying to connect to %s via ssh", ip) try: ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles) except Exception, e: logging.debug("Unable to connect to %s: %s", ip, str(e)) continue ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role) self.assertEquals( ret['exit_status'], 0, ("File '/tmp/%s-file' not found" % role)) ctrl_ssh.disconnect() break else: i_name = i.get('name') or i.get('mac') or str(i) logging.debug("Interface doesn't have an IP: %r", i_name) self.assertNotEqual( ip, None, "Unable to fing a valid IP" " for node %s" % n) return cluster_id
def _basic_provisioning(self, cluster_name, nodes_dict, task_timeout=30 * 60): self._start_logserver() self._clean_clusters() cluster_id = self._create_cluster(name=cluster_name) # Here we updating cluster editable attributes # In particular we set extra syslog server response = self.client.get("/api/clusters/%s/attributes/" % cluster_id) attrs = json.loads(response.read()) attrs["editable"]["syslog"]["syslog_server"]["value"] = \ self.get_host_node_ip() attrs["editable"]["syslog"]["syslog_port"]["value"] = \ self.logserver.bound_port() self.client.put("/api/clusters/%s/attributes/" % cluster_id, attrs) node_names = [] for role in nodes_dict: node_names += nodes_dict[role] try: if len(node_names) > 1: if len(nodes_dict['controller']) == 1: self.client.put("/api/clusters/%s/" % cluster_id, {"mode": "multinode"}) if len(nodes_dict['controller']) > 1: self.client.put("/api/clusters/%s/" % cluster_id, {"mode": "ha"}) except KeyError: pass nodes = self._bootstrap_nodes(node_names) for role in nodes_dict: for n in nodes_dict[role]: slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) self.client.put("/api/nodes/%s/" % node['id'], { "role": role, "pending_addition": True }) self._update_nodes_in_cluster(cluster_id, nodes) task = self._launch_provisioning(cluster_id) self._task_wait(task, 'Installation', task_timeout) logging.info("Checking role files on slave nodes") keyfiles = ci.environment.node['admin'].metadata['keyfiles'] for role in nodes_dict: for n in nodes_dict[role]: logging.info("Checking /tmp/%s-file on %s" % (role, n)) slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() for i in node['meta']['interfaces']: ip = i.get('ip', None) if ip: logging.debug("Trying to connect to %s via ssh", ip) try: ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles) except Exception, e: logging.debug("Unable to connect to %s: %s", ip, str(e)) continue ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role) self.assertEquals( ret['exit_status'], 0, ("File '/tmp/%s-file' not found" % role)) ctrl_ssh.disconnect() break else: i_name = i.get('name') or i.get('mac') or str(i) logging.debug("Interface doesn't have an IP: %r", i_name) self.assertNotEqual( ip, None, "Unable to fing a valid IP" " for node %s" % n)
def __init__(self, *args, **kwargs): super(TestPuppetMaster, self).__init__(*args, **kwargs) self.remote = SSHClient()
class TestNode(Base): def __init__(self, *args, **kwargs): super(TestNode, self).__init__(*args, **kwargs) self.remote = SSHClient() self.client = HTTPClient( url="http://%s:8000" % self.get_admin_node_ip() ) self.ssh_user = "******" self.ssh_passwd = "r00tme" self.admin_host = self.get_admin_node_ip() self.remote.connect_ssh( self.admin_host, self.ssh_user, self.ssh_passwd ) def setUp(self): pass def tearDown(self): self._wait_for_threads() try: self._stop_logserver() except AttributeError: pass def _start_logserver(self, handler=None): self._logserver_status = False if not handler: """ We define log message handler in such a way assuming that if at least one message is received logging works fine. """ def handler(message): self._logserver_status = True self.logserver = LogServer( address=self.get_host_node_ip(), port=5514 ) self.logserver.set_handler(handler) self.logserver.start() def _stop_logserver(self): self.logserver.stop() self._logserver_status = False def _status_logserver(self): return self._logserver_status def test_release_upload(self): self._upload_sample_release() def test_http_returns_200(self): resp = self.client.get("/") self.assertEquals(200, resp.getcode()) def test_create_empty_cluster(self): self._create_cluster(name='empty') @snapshot_errors def test_node_deploy(self): self._revert_nodes() self._bootstrap_nodes(['slave1']) @snapshot_errors def test_updating_nodes_in_cluster(self): self._revert_nodes() cluster_id = self._create_cluster(name='empty') nodes = self._bootstrap_nodes(['slave1']) self._update_nodes_in_cluster(cluster_id, nodes) @snapshot_errors def test_one_node_provisioning(self): self._revert_nodes() self._clean_clusters() self._basic_provisioning('provision', {'controller': ['slave1']}) @snapshot_errors def test_simple_cluster_flat(self): logging.info("Testing simple flat installation.") self._revert_nodes() cluster_name = 'simple_flat' nodes = {'controller': ['slave1'], 'compute': ['slave2']} cluster_id = self._basic_provisioning(cluster_name, nodes) slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 5), timeout=300) logging.info("Verifying networks for simple flat installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] for vlan in vlans: for n in (slave, slave2): self._restore_vlan_in_ebtables( n.interfaces[0].target_dev, vlan, False ) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network simple flat', 60 * 2) @snapshot_errors def test_simple_cluster_vlan(self): logging.info("Testing simple vlan installation.") self._revert_nodes() cluster_name = 'simple_vlan' nodes = {'controller': ['slave1'], 'compute': ['slave2']} self._create_cluster(name=cluster_name, net_manager="VlanManager") cluster_id = self._basic_provisioning(cluster_name, nodes) slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 5, 8), timeout=300) logging.info("Verifying networks for simple vlan installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] for vlan in vlans: for n in (slave, slave2): self._restore_vlan_in_ebtables( n.interfaces[0].target_dev, vlan, False ) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network simple vlan', 60 * 2) @snapshot_errors def test_ha_cluster_flat(self): logging.info("Testing ha flat installation.") self._revert_nodes() cluster_name = 'ha_flat' nodes = { 'controller': ['slave1', 'slave2', 'slave3'], 'compute': ['slave4', 'slave5'] } cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60) logging.info("Checking cluster status on slave1") slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait(lambda: self._check_cluster_status(node['ip'], 13), timeout=300) logging.info("Verifying networks for ha flat installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] slave3 = ci.environment.node['slave3'] slave4 = ci.environment.node['slave4'] slave5 = ci.environment.node['slave5'] for vlan in vlans: for n in (slave, slave2, slave3, slave4, slave5): self._restore_vlan_in_ebtables( n.interfaces[0].target_dev, vlan, False ) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network ha flat', 60 * 2) @snapshot_errors def test_ha_cluster_vlan(self): logging.info("Testing ha vlan installation.") self._revert_nodes() cluster_name = 'ha_vlan' nodes = { 'controller': ['slave1', 'slave2', 'slave3'], 'compute': ['slave4', 'slave5'] } self._create_cluster(name=cluster_name, net_manager="VlanManager") cluster_id = self._basic_provisioning(cluster_name, nodes, 90 * 60) slave = ci.environment.node['slave1'] node = self._get_slave_node_by_devops_node(slave) wait( lambda: self._check_cluster_status(node['ip'], 13, 8), timeout=300 ) logging.info("Verifying networks for ha vlan installation.") vlans = self._get_cluster_vlans(cluster_id) slave2 = ci.environment.node['slave2'] slave3 = ci.environment.node['slave3'] slave4 = ci.environment.node['slave4'] slave5 = ci.environment.node['slave5'] for vlan in vlans: for n in (slave, slave2, slave3, slave4, slave5): self._restore_vlan_in_ebtables( n.interfaces[0].target_dev, vlan, False ) task = self._run_network_verify(cluster_id) self._task_wait(task, 'Verify network ha vlan', 60 * 2) @snapshot_errors def test_network_config(self): self._revert_nodes() self._clean_clusters() self._basic_provisioning('network_config', {'controller': ['slave1']}) slave = ci.environment.node['slave1'] keyfiles = ci.environment.node['admin'].metadata['keyfiles'] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() ctrl_ssh.connect_ssh(node['ip'], 'root', key_filename=keyfiles) ifaces_fail = False for iface in node['network_data']: try: ifname = "%s.%s@%s" % ( iface['dev'], iface['vlan'], iface['dev'] ) ifname_short = "%s.%s" % (iface['dev'], iface['vlan']) except KeyError: ifname = iface['dev'] iface_data = ''.join( ctrl_ssh.execute( '/sbin/ip addr show dev %s' % ifname_short )['stdout'] ) if iface_data.find(ifname) == -1: logging.error("Interface %s is absent" % ifname_short) ifaces_fail = True else: try: if iface_data.find("inet %s" % iface['ip']) == -1: logging.error( "Interface %s does not have ip %s" % ( ifname_short, iface['ip'] ) ) ifaces_fail = True except KeyError: if iface_data.find("inet ") != -1: logging.error( "Interface %s does have ip. And it should not" % ifname_short ) ifaces_fail = True try: if iface_data.find("brd %s" % iface['brd']) == -1: logging.error( "Interface %s does not have broadcast %s" % ( ifname_short, iface['brd'] ) ) ifaces_fail = True except KeyError: pass self.assertEquals(ifaces_fail, False) @snapshot_errors def test_node_deletion(self): self._revert_nodes() cluster_name = 'node_deletion' node_name = 'slave1' nodes = {'controller': [node_name]} cluster_id = self._basic_provisioning(cluster_name, nodes) slave = ci.environment.node[node_name] node = self._get_slave_node_by_devops_node(slave) self.client.put("/api/nodes/%s/" % node['id'], {'pending_deletion': True}) task = self._launch_provisioning(cluster_id) self._task_wait(task, 'Node deletion', 60) timer = time.time() timeout = 5 * 60 while True: response = self.client.get("/api/nodes/") nodes = json.loads(response.read()) for n in nodes: if (n['mac'] == node['mac'] and n['status'] == 'discover'): return if (time.time() - timer) > timeout: raise Exception("Bootstrap boot timeout expired!") time.sleep(5) @snapshot_errors def test_network_verify_with_blocked_vlan(self): self._revert_nodes() cluster_name = 'net_verify' cluster_id = self._create_cluster(name=cluster_name) node_names = ['slave1', 'slave2'] nailgun_slave_nodes = self._bootstrap_nodes(node_names) devops_nodes = [ci.environment.node[n] for n in node_names] logging.info("Clear BROUTING table entries.") vlans = self._get_cluster_vlans(cluster_id) for vlan in vlans: for node in devops_nodes: for interface in node.interfaces: self._restore_vlan_in_ebtables(interface.target_dev, vlan, False) self._update_nodes_in_cluster(cluster_id, nailgun_slave_nodes) for node in devops_nodes: for interface in node.interfaces: self._block_vlan_in_ebtables(interface.target_dev, vlans[0]) task = self._run_network_verify(cluster_id) task = self._task_wait(task, 'Verify network in cluster with blocked vlan', 60 * 2, True) self.assertEquals(task['status'], 'error') @snapshot_errors def test_multinic_bootstrap_booting(self): self._revert_nodes() slave = filter(lambda n: n.name != 'admin' and len(n.interfaces) > 1, ci.environment.nodes)[0] nodename = slave.name logging.info("Using node %r with %d interfaces", nodename, len(slave.interfaces)) slave.stop() admin = ci.environment.node['admin'] macs = [i.mac_address for i in slave.interfaces] logging.info("Block all MACs: %s.", ', '.join([m for m in macs])) for mac in macs: self._block_mac_in_ebtables(mac) self.addCleanup(self._restore_mac_in_ebtables, mac) for mac in macs: logging.info("Trying to boot node %r via interface with MAC %s...", nodename, mac) self._restore_mac_in_ebtables(mac) slave.start() nailgun_slave = self._bootstrap_nodes([nodename])[0] self.assertEqual(mac.upper(), nailgun_slave['mac'].upper()) slave.stop() admin.restore_snapshot('initial') self._block_mac_in_ebtables(mac) @staticmethod def _block_mac_in_ebtables(mac): try: subprocess.check_output( 'sudo ebtables -t filter -A FORWARD -s %s -j DROP' % mac, stderr=subprocess.STDOUT, shell=True ) logging.debug("MAC %s blocked via ebtables.", mac) except subprocess.CalledProcessError as e: raise Exception("Can't block mac %s via ebtables: %s", mac, e.output) @staticmethod def _restore_mac_in_ebtables(mac): try: subprocess.check_output( 'sudo ebtables -t filter -D FORWARD -s %s -j DROP' % mac, stderr=subprocess.STDOUT, shell=True ) logging.debug("MAC %s unblocked via ebtables.", mac) except subprocess.CalledProcessError as e: logging.warn("Can't restore mac %s via ebtables: %s", mac, e.output) def _block_vlan_in_ebtables(self, target_dev, vlan): try: subprocess.check_output( 'sudo ebtables -t broute -A BROUTING -i %s -p 8021Q' ' --vlan-id %s -j DROP' % ( target_dev, vlan ), stderr=subprocess.STDOUT, shell=True ) self.addCleanup(self._restore_vlan_in_ebtables, target_dev, vlan) logging.debug("Vlan %s on interface %s blocked via ebtables.", vlan, target_dev) except subprocess.CalledProcessError as e: raise Exception("Can't block vlan %s for interface %s" " via ebtables: %s" % (vlan, target_dev, e.output)) def _get_cluster_vlans(self, cluster_id): resp = self.client.get( '/api/clusters/%d/network_configuration/' % cluster_id) self.assertEquals(200, resp.getcode()) cluster_vlans = [] networks = json.loads(resp.read())['networks'] for n in networks: amount = n.get('amount', 1) cluster_vlans.extend(range(n['vlan_start'], n['vlan_start'] + amount)) self.assertNotEqual(cluster_vlans, []) return cluster_vlans @staticmethod def _restore_vlan_in_ebtables(target_dev, vlan, log=True): try: subprocess.check_output( 'sudo ebtables -t broute -D BROUTING -i %s -p 8021Q' ' --vlan-id %s -j DROP' % ( target_dev, vlan ), stderr=subprocess.STDOUT, shell=True ) logging.debug("Vlan %s on interface %s unblocked via ebtables.", vlan, target_dev) except subprocess.CalledProcessError as e: if log: logging.warn("Can't restore vlan %s for interface %s" " via ebtables: %s" % (vlan, target_dev, e.output)) def _run_network_verify(self, cluster_id): logging.info( "Run network verify in cluster %d", cluster_id ) resp = self.client.get( '/api/clusters/%d/network_configuration/' % cluster_id) self.assertEquals(200, resp.getcode()) network_configuration = json.loads(resp.read()) changes = self.client.put( '/api/clusters/%d/network_configuration/verify/' % cluster_id, network_configuration ) self.assertEquals(200, changes.getcode()) return json.loads(changes.read()) def _basic_provisioning(self, cluster_name, nodes_dict, task_timeout=30 * 60): self._start_logserver() self._clean_clusters() cluster_id = self._create_cluster(name=cluster_name) # Here we updating cluster editable attributes # In particular we set extra syslog server response = self.client.get( "/api/clusters/%s/attributes/" % cluster_id ) attrs = json.loads(response.read()) attrs["editable"]["syslog"]["syslog_server"]["value"] = \ self.get_host_node_ip() attrs["editable"]["syslog"]["syslog_port"]["value"] = \ self.logserver.bound_port() self.client.put( "/api/clusters/%s/attributes/" % cluster_id, attrs ) node_names = [] for role in nodes_dict: node_names += nodes_dict[role] try: if len(node_names) > 1: if len(nodes_dict['controller']) == 1: self.client.put( "/api/clusters/%s/" % cluster_id, {"mode": "multinode"} ) if len(nodes_dict['controller']) > 1: self.client.put( "/api/clusters/%s/" % cluster_id, {"mode": "ha"} ) except KeyError: pass nodes = self._bootstrap_nodes(node_names) for role in nodes_dict: for n in nodes_dict[role]: slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) self.client.put( "/api/nodes/%s/" % node['id'], {"role": role, "pending_addition": True} ) self._update_nodes_in_cluster(cluster_id, nodes) task = self._launch_provisioning(cluster_id) self._task_wait(task, 'Installation', task_timeout) logging.info("Checking role files on slave nodes") keyfiles = ci.environment.node['admin'].metadata['keyfiles'] for role in nodes_dict: for n in nodes_dict[role]: logging.info("Checking /tmp/%s-file on %s" % (role, n)) slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() for i in node['meta']['interfaces']: ip = i.get('ip', None) if ip: logging.debug("Trying to connect to %s via ssh", ip) try: ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles) except Exception, e: logging.debug("Unable to connect to %s: %s", ip, str(e)) continue ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role) self.assertEquals(ret['exit_status'], 0, ("File '/tmp/%s-file' not found" % role)) ctrl_ssh.disconnect() break else: i_name = i.get('name') or i.get('mac') or str(i) logging.debug("Interface doesn't have an IP: %r", i_name) self.assertNotEqual(ip, None, "Unable to fing a valid IP" " for node %s" % n) return cluster_id
def _basic_provisioning(self, cluster_name, nodes_dict, task_timeout=30 * 60): self._start_logserver() self._clean_clusters() cluster_id = self._create_cluster(name=cluster_name) # Here we updating cluster editable attributes # In particular we set extra syslog server response = self.client.get( "/api/clusters/%s/attributes/" % cluster_id ) attrs = json.loads(response.read()) attrs["editable"]["syslog"]["syslog_server"]["value"] = \ self.get_host_node_ip() attrs["editable"]["syslog"]["syslog_port"]["value"] = \ self.logserver.bound_port() self.client.put( "/api/clusters/%s/attributes/" % cluster_id, attrs ) node_names = [] for role in nodes_dict: node_names += nodes_dict[role] try: if len(node_names) > 1: if len(nodes_dict['controller']) == 1: self.client.put( "/api/clusters/%s/" % cluster_id, {"mode": "multinode"} ) if len(nodes_dict['controller']) > 1: self.client.put( "/api/clusters/%s/" % cluster_id, {"mode": "ha"} ) except KeyError: pass nodes = self._bootstrap_nodes(node_names) for role in nodes_dict: for n in nodes_dict[role]: slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) self.client.put( "/api/nodes/%s/" % node['id'], {"role": role, "pending_addition": True} ) self._update_nodes_in_cluster(cluster_id, nodes) task = self._launch_provisioning(cluster_id) self._task_wait(task, 'Installation', task_timeout) logging.info("Checking role files on slave nodes") keyfiles = ci.environment.node['admin'].metadata['keyfiles'] for role in nodes_dict: for n in nodes_dict[role]: logging.info("Checking /tmp/%s-file on %s" % (role, n)) slave = ci.environment.node[n] node = self._get_slave_node_by_devops_node(slave) ctrl_ssh = SSHClient() for i in node['meta']['interfaces']: ip = i.get('ip', None) if ip: logging.debug("Trying to connect to %s via ssh", ip) try: ctrl_ssh.connect_ssh(ip, 'root', key_filename=keyfiles) except Exception, e: logging.debug("Unable to connect to %s: %s", ip, str(e)) continue ret = ctrl_ssh.execute('test -f /tmp/%s-file' % role) self.assertEquals(ret['exit_status'], 0, ("File '/tmp/%s-file' not found" % role)) ctrl_ssh.disconnect() break else: i_name = i.get('name') or i.get('mac') or str(i) logging.debug("Interface doesn't have an IP: %r", i_name) self.assertNotEqual(ip, None, "Unable to fing a valid IP" " for node %s" % n)
def __init__(self, *args, **kwargs): super(TestCobbler, self).__init__(*args, **kwargs) self.remote = SSHClient()
def __init__(self, *args, **kwargs): super(TestNailyd, self).__init__(*args, **kwargs) self.remote = SSHClient()