def test_deploy_neutron_vlan_ha(self):
"""Deploy cluster in HA mode with Neutron VLAN
Scenario:
1. Create cluster
2. Add 3 nodes with controller role
3. Add 2 nodes with compute role
4. Deploy the cluster
5. Run network verification
6. Run OSTF
Duration 80m
Snapshot deploy_neutron_vlan_ha
"""
self.manager.show_step(1)
self.manager.show_step(2)
self.manager.show_step(3)
self.manager.show_step(4)
self.manager.show_step(5)
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
cluster = fuel_web.client.get_cluster(cluster_id)
assert str(cluster['net_provider']) == settings.NEUTRON
os_conn = os_actions.OpenStackActions(
fuel_web.get_public_vip(cluster_id),
user=self.cluster_config['settings']['user'],
passwd=self.cluster_config['settings']['password'],
tenant=self.cluster_config['settings']['tenant'])
fuel_web.check_fixed_network_cidr(
cluster_id, os_conn)
fuel_web.verify_network(cluster_id)
devops_node = fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
for _ in range(5):
try:
checkers.check_swift_ring(_ip)
break
except AssertionError:
cmd = "/usr/local/bin/swift-rings-rebalance.sh"
result = ssh_manager.execute(ip=_ip, cmd=cmd)
logger.debug("command execution result is {0}"
.format(result['exit_code']))
else:
checkers.check_swift_ring(_ip)
self.manager.show_step(6)
fuel_web.run_ostf(
cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity'])
def deploy_neutron_tun_ha_base(self, snapshot_name):
self.check_run(snapshot_name)
self.env.revert_snapshot("ready_with_5_slaves")
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=DEPLOYMENT_MODE,
settings={
"net_provider": 'neutron',
"net_segment_type": NEUTRON_SEGMENT['tun'],
'tenant': 'haTun',
'user': '******',
'password': '******'
}
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['compute']
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
cluster = self.fuel_web.client.get_cluster(cluster_id)
assert_equal(str(cluster['net_provider']), 'neutron')
self.fuel_web.verify_network(cluster_id)
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = self.fuel_web.get_nailgun_node_by_devops_node(devops_node)['ip']
with self.fuel_web.get_ssh_for_node(devops_node.name) as remote:
for _ in range(5):
try:
checkers.check_swift_ring(_ip)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}"
.format(result))
else:
checkers.check_swift_ring(_ip)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot(snapshot_name, is_make=True)
def _check_swift(primary_node):
with self.fuel_web.get_ssh_for_node(primary_node.name) as remote:
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug(
"command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
def _check_swift(primary_node):
with self.fuel_web.get_ssh_for_node(primary_node.name) as remote:
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug(
"command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600):
"""Check Swift ring and rebalance it if needed.
Replication should be performed on primary controller node.
Retry check several times. Wait for replication due to LP1498368.
"""
ssh = SSHManager()
cmd = "/usr/local/bin/swift-rings-rebalance.sh"
logger.debug('Check swift ring and rebalance it.')
for _ in xrange(retry_count):
try:
checkers.check_swift_ring(controller_ip)
break
except AssertionError:
result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd)
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(controller_ip)
def _check_swift(node):
with self.fuel_web.get_ssh_for_node(node.name) as remote:
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug(
"command execution result is {0}".format(result))
if result['exit_code'] == 0:
# (tleontovich) We should sleep here near 5-10
# minute and waiting for replica
# LP1498368/comments/16
time.sleep(600)
else:
checkers.check_swift_ring(remote)
def _check_swift(node):
with self.fuel_web.get_ssh_for_node(node.name) as remote:
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug(
"command execution result is {0}".format(result))
if result['exit_code'] == 0:
# (tleontovich) We should sleep here near 5-10
# minute and waiting for replica
# LP1498368/comments/16
time.sleep(600)
else:
checkers.check_swift_ring(remote)
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600):
"""Check Swift ring and rebalance it if needed.
Replication should be performed on primary controller node.
Retry check several times. Wait for replication due to LP1498368.
"""
ssh = SSHManager()
cmd = "/usr/local/bin/swift-rings-rebalance.sh"
logger.debug('Check swift ring and rebalance it.')
for _ in xrange(retry_count):
try:
checkers.check_swift_ring(controller_ip)
break
except AssertionError:
result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd)
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(controller_ip)
def deploy_neutron_gre_ha(self):
"""Deploy cluster in HA mode with Neutron GRE (DEPRECATED)
Scenario:
1. Create cluster
2. Add 3 nodes with controller role
3. Add 2 nodes with compute role
4. Deploy the cluster
5. Run network verification
6. Run OSTF
Duration 80m
Snapshot deploy_neutron_gre_ha
"""
self.env.revert_snapshot("ready_with_5_slaves")
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=DEPLOYMENT_MODE,
settings={
"net_provider": 'neutron',
"net_segment_type": NEUTRON_SEGMENT['gre'],
'tenant': 'haGre',
'user': '******',
'password': '******'
}
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['compute']
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
cluster = self.fuel_web.client.get_cluster(cluster_id)
assert_equal(str(cluster['net_provider']), 'neutron')
self.fuel_web.verify_network(cluster_id)
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}"
.format(result))
else:
checkers.check_swift_ring(remote)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot("deploy_neutron_gre_ha")
def deploy_ha_vlan(self):
"""Deploy cluster in HA mode with VLAN Manager
Scenario:
1. Create cluster
2. Add 3 nodes with controller roles
3. Add 2 nodes with compute roles
4. Set up cluster to use Network VLAN manager with 8 networks
5. Deploy the cluster
6. Validate cluster was set up correctly, there are no dead
services, there are no errors in logs
7. Run network verification
8. Run OSTF
9. Create snapshot
Duration 70m
Snapshot deploy_ha_vlan
"""
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'tenant': 'novaHAVlan',
'user': '******',
'password': '******'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_HA,
settings=data
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['compute']
}
)
self.fuel_web.update_vlan_network_fixed(
cluster_id, amount=8, network_size=32
)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.deploy_cluster_wait(cluster_id)
# Network verification
self.fuel_web.verify_network(cluster_id)
# HAProxy backend checking
controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
for node in controller_nodes:
remote = self.env.d_env.get_ssh_to_remote(node['ip'])
logger.info("Check all HAProxy backends on {}".format(
node['meta']['system']['fqdn']))
haproxy_status = checkers.check_haproxy_backend(
remote, ignore_services=['nova-metadata-api'])
assert_equal(haproxy_status['exit_code'], 1,
"HAProxy backends are DOWN. {0}".format(
haproxy_status))
remote.clear()
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id),
data['user'], data['password'], data['tenant'])
self.fuel_web.assert_cluster_ready(
os_conn, smiles_count=16, networks_count=8, timeout=300)
_ip = self.fuel_web.get_nailgun_node_by_name('slave-01')['ip']
self.fuel_web.check_fixed_nova_splited_cidr(
os_conn, self.fuel_web.get_nailgun_cidr_nova(cluster_id),
self.env.d_env.get_ssh_to_remote(_ip))
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
remote = self.env.d_env.get_ssh_to_remote(_ip)
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot("deploy_ha_vlan")
def ha_flat_scalability(self):
"""Check HA mode on scalability
Scenario:
1. Create cluster
2. Add 1 controller node
3. Deploy the cluster
4. Add 2 controller nodes
5. Deploy changes
6. Run network verification
7. Add 2 controller nodes
8. Deploy changes
9. Run network verification
10. Run OSTF
Duration 110m
Snapshot ha_flat_scalability
"""
self.env.revert_snapshot("ready_with_5_slaves")
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_HA
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller']
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
remote = self.env.d_env.get_ssh_to_remote(_ip)
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
self.fuel_web.update_nodes(
cluster_id, {'slave-02': ['controller'],
'slave-03': ['controller']},
True, False
)
self.fuel_web.deploy_cluster_wait(cluster_id)
for devops_node in self.env.d_env.nodes().slaves[:3]:
self.fuel_web.assert_pacemaker(
devops_node.name,
self.env.d_env.nodes().slaves[:3], [])
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
remote = self.env.d_env.get_ssh_to_remote(_ip)
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
self.fuel_web.update_nodes(
cluster_id, {'slave-04': ['controller'],
'slave-05': ['controller']},
True, False
)
self.fuel_web.deploy_cluster_wait(cluster_id)
for devops_node in self.env.d_env.nodes().slaves[:5]:
self.fuel_web.assert_pacemaker(
devops_node.name,
self.env.d_env.nodes().slaves[:5], [])
ret = self.fuel_web.get_pacemaker_status(devops_node.name)
assert_true(
re.search('vip__management\s+\(ocf::fuel:ns_IPaddr2\):'
'\s+Started node', ret), 'vip management started')
assert_true(
re.search('vip__public\s+\(ocf::fuel:ns_IPaddr2\):'
'\s+Started node', ret), 'vip public started')
self.fuel_web.security.verify_firewall(cluster_id)
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))\
_ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
remote = self.env.d_env.get_ssh_to_remote(_ip)
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'sanity'])
self.env.make_snapshot("ha_flat_scalability")
def deploy_ha_flat(self):
"""Deploy cluster in HA mode with flat nova-network
Scenario:
1. Create cluster
2. Add 3 nodes with controller roles
3. Add 2 nodes with compute roles
4. Deploy the cluster
5. Validate cluster was set up correctly, there are no dead
services, there are no errors in logs
6. Run verify networks
7. Run OSTF
8. Make snapshot
Duration 70m
Snapshot deploy_ha_flat
"""
try:
self.check_run("deploy_ha_flat")
except SkipTest:
return
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'tenant': 'novaHaFlat',
'user': '******',
'password': '******'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_HA,
settings=data
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['compute']
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id),
data['user'], data['password'], data['tenant'])
self.fuel_web.assert_cluster_ready(
os_conn, smiles_count=16, networks_count=1, timeout=300)
self.fuel_web.verify_network(cluster_id)
devops_node = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug("devops node name is {0}".format(devops_node.name))
_ip = self.fuel_web.get_nailgun_node_by_name(devops_node.name)['ip']
remote = self.env.d_env.get_ssh_to_remote(_ip)
for i in range(5):
try:
checkers.check_swift_ring(remote)
break
except AssertionError:
result = remote.execute(
"/usr/local/bin/swift-rings-rebalance.sh")
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(remote)
self.fuel_web.security.verify_firewall(cluster_id)
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot("deploy_ha_flat", is_make=True)
