def test_deploy_neutron_vlan_ha(self): """Deploy cluster in HA mode with Neutron VLAN Scenario: 1. Create cluster 2. Add 3 nodes with controller role 3. Add 2 nodes with compute role 4. Deploy the cluster 5. Run network verification 6. Run OSTF Duration 80m Snapshot deploy_neutron_vlan_ha """ self.manager.show_step(1) self.manager.show_step(2) self.manager.show_step(3) self.manager.show_step(4) self.manager.show_step(5) cluster_id = self._storage['cluster_id'] fuel_web = self.manager.fuel_web cluster = fuel_web.client.get_cluster(cluster_id) assert str(cluster['net_provider']) == settings.NEUTRON os_conn = os_actions.OpenStackActions( fuel_web.get_public_vip(cluster_id), user=self.cluster_config['settings']['user'], passwd=self.cluster_config['settings']['password'], tenant=self.cluster_config['settings']['tenant']) fuel_web.check_fixed_network_cidr( cluster_id, os_conn) fuel_web.verify_network(cluster_id) devops_node = fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] for _ in range(5): try: checkers.check_swift_ring(_ip) break except AssertionError: cmd = "/usr/local/bin/swift-rings-rebalance.sh" result = ssh_manager.execute(ip=_ip, cmd=cmd) logger.debug("command execution result is {0}" .format(result['exit_code'])) else: checkers.check_swift_ring(_ip) self.manager.show_step(6) fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'])
def deploy_neutron_tun_ha_base(self, snapshot_name): self.check_run(snapshot_name) self.env.revert_snapshot("ready_with_5_slaves") cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE, settings={ "net_provider": 'neutron', "net_segment_type": NEUTRON_SEGMENT['tun'], 'tenant': 'haTun', 'user': '******', 'password': '******' } ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute'], 'slave-05': ['compute'] } ) self.fuel_web.deploy_cluster_wait(cluster_id) cluster = self.fuel_web.client.get_cluster(cluster_id) assert_equal(str(cluster['net_provider']), 'neutron') self.fuel_web.verify_network(cluster_id) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_devops_node(devops_node)['ip'] with self.fuel_web.get_ssh_for_node(devops_node.name) as remote: for _ in range(5): try: checkers.check_swift_ring(_ip) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}" .format(result)) else: checkers.check_swift_ring(_ip) self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) self.env.make_snapshot(snapshot_name, is_make=True)
def _check_swift(primary_node): with self.fuel_web.get_ssh_for_node(primary_node.name) as remote: for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug( "command execution result is {0}".format(result)) else: checkers.check_swift_ring(remote)
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600): """Check Swift ring and rebalance it if needed. Replication should be performed on primary controller node. Retry check several times. Wait for replication due to LP1498368. """ ssh = SSHManager() cmd = "/usr/local/bin/swift-rings-rebalance.sh" logger.debug('Check swift ring and rebalance it.') for _ in xrange(retry_count): try: checkers.check_swift_ring(controller_ip) break except AssertionError: result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd) logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(controller_ip)
def _check_swift(node): with self.fuel_web.get_ssh_for_node(node.name) as remote: for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug( "command execution result is {0}".format(result)) if result['exit_code'] == 0: # (tleontovich) We should sleep here near 5-10 # minute and waiting for replica # LP1498368/comments/16 time.sleep(600) else: checkers.check_swift_ring(remote)
def deploy_neutron_gre_ha(self): """Deploy cluster in HA mode with Neutron GRE (DEPRECATED) Scenario: 1. Create cluster 2. Add 3 nodes with controller role 3. Add 2 nodes with compute role 4. Deploy the cluster 5. Run network verification 6. Run OSTF Duration 80m Snapshot deploy_neutron_gre_ha """ self.env.revert_snapshot("ready_with_5_slaves") cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE, settings={ "net_provider": 'neutron', "net_segment_type": NEUTRON_SEGMENT['gre'], 'tenant': 'haGre', 'user': '******', 'password': '******' } ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute'], 'slave-05': ['compute'] } ) self.fuel_web.deploy_cluster_wait(cluster_id) cluster = self.fuel_web.client.get_cluster(cluster_id) assert_equal(str(cluster['net_provider']), 'neutron') self.fuel_web.verify_network(cluster_id) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] with self.env.d_env.get_ssh_to_remote(_ip) as remote: for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}" .format(result)) else: checkers.check_swift_ring(remote) self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) self.env.make_snapshot("deploy_neutron_gre_ha")
def deploy_ha_vlan(self): """Deploy cluster in HA mode with VLAN Manager Scenario: 1. Create cluster 2. Add 3 nodes with controller roles 3. Add 2 nodes with compute roles 4. Set up cluster to use Network VLAN manager with 8 networks 5. Deploy the cluster 6. Validate cluster was set up correctly, there are no dead services, there are no errors in logs 7. Run network verification 8. Run OSTF 9. Create snapshot Duration 70m Snapshot deploy_ha_vlan """ self.env.revert_snapshot("ready_with_5_slaves") data = { 'tenant': 'novaHAVlan', 'user': '******', 'password': '******' } cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE_HA, settings=data ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute'], 'slave-05': ['compute'] } ) self.fuel_web.update_vlan_network_fixed( cluster_id, amount=8, network_size=32 ) self.fuel_web.verify_network(cluster_id) self.fuel_web.deploy_cluster_wait(cluster_id) # Network verification self.fuel_web.verify_network(cluster_id) # HAProxy backend checking controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) for node in controller_nodes: remote = self.env.d_env.get_ssh_to_remote(node['ip']) logger.info("Check all HAProxy backends on {}".format( node['meta']['system']['fqdn'])) haproxy_status = checkers.check_haproxy_backend( remote, ignore_services=['nova-metadata-api']) assert_equal(haproxy_status['exit_code'], 1, "HAProxy backends are DOWN. {0}".format( haproxy_status)) remote.clear() os_conn = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id), data['user'], data['password'], data['tenant']) self.fuel_web.assert_cluster_ready( os_conn, smiles_count=16, networks_count=8, timeout=300) _ip = self.fuel_web.get_nailgun_node_by_name('slave-01')['ip'] self.fuel_web.check_fixed_nova_splited_cidr( os_conn, self.fuel_web.get_nailgun_cidr_nova(cluster_id), self.env.d_env.get_ssh_to_remote(_ip)) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] remote = self.env.d_env.get_ssh_to_remote(_ip) for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(remote) self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) self.env.make_snapshot("deploy_ha_vlan")
def ha_flat_scalability(self): """Check HA mode on scalability Scenario: 1. Create cluster 2. Add 1 controller node 3. Deploy the cluster 4. Add 2 controller nodes 5. Deploy changes 6. Run network verification 7. Add 2 controller nodes 8. Deploy changes 9. Run network verification 10. Run OSTF Duration 110m Snapshot ha_flat_scalability """ self.env.revert_snapshot("ready_with_5_slaves") cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE_HA ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'] } ) self.fuel_web.deploy_cluster_wait(cluster_id) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] remote = self.env.d_env.get_ssh_to_remote(_ip) for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(remote) self.fuel_web.update_nodes( cluster_id, {'slave-02': ['controller'], 'slave-03': ['controller']}, True, False ) self.fuel_web.deploy_cluster_wait(cluster_id) for devops_node in self.env.d_env.nodes().slaves[:3]: self.fuel_web.assert_pacemaker( devops_node.name, self.env.d_env.nodes().slaves[:3], []) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] remote = self.env.d_env.get_ssh_to_remote(_ip) for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(remote) self.fuel_web.update_nodes( cluster_id, {'slave-04': ['controller'], 'slave-05': ['controller']}, True, False ) self.fuel_web.deploy_cluster_wait(cluster_id) for devops_node in self.env.d_env.nodes().slaves[:5]: self.fuel_web.assert_pacemaker( devops_node.name, self.env.d_env.nodes().slaves[:5], []) ret = self.fuel_web.get_pacemaker_status(devops_node.name) assert_true( re.search('vip__management\s+\(ocf::fuel:ns_IPaddr2\):' '\s+Started node', ret), 'vip management started') assert_true( re.search('vip__public\s+\(ocf::fuel:ns_IPaddr2\):' '\s+Started node', ret), 'vip public started') self.fuel_web.security.verify_firewall(cluster_id) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name))\ _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] remote = self.env.d_env.get_ssh_to_remote(_ip) for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(remote) self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'sanity']) self.env.make_snapshot("ha_flat_scalability")
def deploy_ha_flat(self): """Deploy cluster in HA mode with flat nova-network Scenario: 1. Create cluster 2. Add 3 nodes with controller roles 3. Add 2 nodes with compute roles 4. Deploy the cluster 5. Validate cluster was set up correctly, there are no dead services, there are no errors in logs 6. Run verify networks 7. Run OSTF 8. Make snapshot Duration 70m Snapshot deploy_ha_flat """ try: self.check_run("deploy_ha_flat") except SkipTest: return self.env.revert_snapshot("ready_with_5_slaves") data = { 'tenant': 'novaHaFlat', 'user': '******', 'password': '******' } cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE_HA, settings=data ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute'], 'slave-05': ['compute'] } ) self.fuel_web.deploy_cluster_wait(cluster_id) os_conn = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id), data['user'], data['password'], data['tenant']) self.fuel_web.assert_cluster_ready( os_conn, smiles_count=16, networks_count=1, timeout=300) self.fuel_web.verify_network(cluster_id) devops_node = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) logger.debug("devops node name is {0}".format(devops_node.name)) _ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip'] remote = self.env.d_env.get_ssh_to_remote(_ip) for i in range(5): try: checkers.check_swift_ring(remote) break except AssertionError: result = remote.execute( "/usr/local/bin/swift-rings-rebalance.sh") logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(remote) self.fuel_web.security.verify_firewall(cluster_id) self.fuel_web.run_ostf( cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) self.env.make_snapshot("deploy_ha_flat", is_make=True)