def delete_instances(os_conn, instances):
instances_ids = [x.id for x in instances]
# Stop instances (to prevent error during deletion)
for instance_id in instances_ids:
os_conn.nova.servers.stop(instance_id)
def isnstances_shutdowned():
instances = [x
for x in os_conn.nova.servers.list()
if x.id in instances_ids]
if any([x.status == 'ERROR' for x in instances]):
raise Exception(
'Some server(s) became to ERROR state after stop')
return all([x.status == 'SHUTOFF' for x in instances])
common.wait(isnstances_shutdowned, timeout_seconds=10 * 60)
# Delete instances
for instance_id in instances_ids:
os_conn.nova.servers.delete(instance_id)
def instances_deleted():
not_deleted = [x
for x in os_conn.nova.servers.list()
if x.id in instances_ids]
if len(not_deleted) == 0:
return True
if any([x.status == 'ERROR' for x in not_deleted]):
raise Exception(
'Some server(s) became to ERROR state after deletion')
common.wait(instances_deleted, timeout_seconds=2 * 60)
def ironic_nodes(request, env, ironic_drivers_params, ironic, devops_env):
node_count = getattr(request, 'param', 1)
devops_nodes = []
nodes = []
for i, config in enumerate(ironic_drivers_params[:node_count]):
devops_node, node = make_ironic_node(config=config,
devops_env=devops_env,
ironic=ironic,
name='baremetal_{i}'.format(i=i),
fuel_env=env)
nodes.append(node)
if devops_node is not None:
devops_nodes.append(devops_node)
common.wait(lambda: env.is_ostf_tests_pass('sanity'),
timeout_seconds=60 * 5,
waiting_for='OSTF sanity tests to pass')
yield nodes
for node in nodes:
ironic.delete_node(node)
for node in devops_nodes:
devops_env.del_node(node)
def check_vm_is_accessible_with_ssh(self,
vm_ip,
username=None,
password=None,
pkeys=None):
"""Check that instance is accessible with ssh via floating_ip.
:param vm_ip: floating_ip of instance
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
error_msg = ('Instance with ip {0} '
'is not accessible with ssh.').format(vm_ip)
def is_accessible():
ssh_client = self.env.get_ssh_to_vm(vm_ip, username, password,
pkeys)
return ssh_client.check_connection() is True
wait(is_accessible,
sleep_seconds=10,
timeout_seconds=60,
waiting_for=error_msg)
def check_vm_is_accessible_with_ssh(self, vm_ip, username=None,
password=None, pkeys=None):
"""Check that instance is accessible with ssh via floating_ip.
:param vm_ip: floating_ip of instance
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
error_msg = 'Instance with ip {0} is not accessible with ssh.'\
.format(vm_ip)
def is_accessible():
try:
with self.env.get_ssh_to_vm(
vm_ip, username, password, pkeys) as vm_remote:
vm_remote.execute("date")
return True
except ssh_exception.SSHException:
return False
except ssh_exception.NoValidConnectionsError:
return False
wait(is_accessible,
sleep_seconds=10, timeout_seconds=60,
waiting_for=error_msg)
def check_vm_is_available(self, vm,
username=None, password=None, pkeys=None):
"""Check that instance is available for connect from controller.
:param vm: instance to ping from it compute node
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
vm = self.os_conn.get_instance_detail(vm)
srv_host = self.env.find_node_by_fqdn(
self.os_conn.get_srv_hypervisor_name(vm)).data['ip']
vm_ip = self.os_conn.get_nova_instance_ips(vm)['floating']
with self.env.get_ssh_to_node(srv_host) as remote:
cmd = "ping -c1 {0}".format(vm_ip)
waiting_for_msg = (
'Waiting for instance with ip {0} has '
'connectivity from node with ip {1}.').format(vm_ip, srv_host)
wait(lambda: remote.execute(cmd)['exit_code'] == 0,
sleep_seconds=10, timeout_seconds=3 * 60,
waiting_for=waiting_for_msg)
return self.check_vm_is_accessible_with_ssh(
vm_ip, username=username, password=password, pkeys=pkeys)
def clear_dhcp_agent(self, node_to_clear, host, network_name=None,
wait_for_rescheduling=True):
"""Clear DHCP agent after ban and wait until agents rescheduling.
:param node_to_clear: dhcp-agent host to clear
:param host: host or ip of controller onto execute ban command
:param network_name: name of network to determine node with dhcp agents
:param wait_for_rescheduling: wait until dhcp-agent reschedule
:returns: str, name of cleared node
"""
list_dhcp_agents = lambda: self.os_conn.list_all_neutron_agents(
agent_type='dhcp', filter_attr='host')
if network_name:
network = self.os_conn.neutron.list_networks(
name=network_name)['networks'][0]
list_dhcp_agents = (
lambda: self.os_conn.get_node_with_dhcp_for_network(
network['id']))
# clear dhcp agent on provided node
with self.env.get_ssh_to_node(host) as remote:
remote.check_call(
"pcs resource clear neutron-dhcp-agent {0}".format(
node_to_clear))
# Wait to reschedule dhcp agent
if wait_for_rescheduling:
wait(
lambda: (node_to_clear in list_dhcp_agents()),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for="DHCP agent {0} to reschedule".format(
node_to_clear))
return node_to_clear
def test_reboot_conductor(env, ironic, os_conn, ironic_nodes, make_image,
flavors, keypair, devops_env, boot_instance_before):
"""Check ironic state after restart conductor node
Scenario:
1. Boot Ironic instance (if `boot_instance_before`)
2. Reboot Ironic conductor.
3. Wait 5-10 minutes.
4. Run network verification.
5. Run OSTF including Ironic tests.
6. Verify that CLI ironicclient can list nodes, ports, chassis, drivers
7. Boot new Ironic instance (if not `boot_instance_before`).
"""
flavor, ironic_node = zip(flavors, ironic_nodes)[0]
image = make_image(node_driver=ironic_node.driver)
if boot_instance_before:
instance = ironic.boot_instance(image=image,
flavor=flavor,
keypair=keypair)
conductor = env.get_nodes_by_role('ironic')[0]
devops_node = devops_env.get_node_by_fuel_node(conductor)
devops_node.reset()
time.sleep(10)
common.wait(conductor.is_ssh_avaliable,
timeout_seconds=60 * 10,
sleep_seconds=20,
waiting_for='ironic conductor node to reboot')
def is_ironic_available():
try:
ironic.client.driver.list()
return True
except Exception:
return False
common.wait(is_ironic_available,
timeout_seconds=60 * 5,
sleep_seconds=20,
waiting_for='ironic conductor service to start')
result = env.wait_network_verification()
assert result.status == 'ready', 'Result data:\n{0}'.format(result.data)
env.wait_for_ostf_pass(['sanity'], timeout_seconds=60 * 5)
with env.get_nodes_by_role('controller')[0].ssh() as remote:
ironic_cli = os_cli.Ironic(remote)
for cmd in ['node-list', 'port-list', 'chassis-list', 'driver-list']:
ironic_cli(cmd)
if not boot_instance_before:
instance = ironic.boot_instance(image=image,
flavor=flavor,
keypair=keypair)
assert os_conn.nova.servers.get(instance.id).status == 'ACTIVE'
def create_server(self, name, image_id=None, flavor=1, userdata=None,
files=None, key_name=None, timeout=300,
wait_for_active=True, wait_for_avaliable=True, **kwargs):
if image_id is None:
image_id = self._get_cirros_image().id
srv = self.nova.servers.create(name=name,
image=image_id,
flavor=flavor,
userdata=userdata,
files=files,
key_name=key_name,
**kwargs)
if wait_for_active:
wait(lambda: self.is_server_active(srv),
timeout_seconds=timeout, sleep_seconds=5,
waiting_for='instance {0} changes status to ACTIVE'.format(
name))
# wait for ssh ready
if wait_for_avaliable:
if self.env is not None:
wait(lambda: self.is_server_ssh_ready(srv),
timeout_seconds=timeout,
waiting_for='server available via ssh')
logger.info('the server {0} is ready'.format(srv.name))
return self.get_instance_detail(srv.id)
def check_vm_is_accessible_with_ssh(self, vm_ip, username=None,
password=None, pkeys=None):
"""Check that instance is accessible with ssh via floating_ip.
:param vm_ip: floating_ip of instance
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
error_msg = 'Instance with ip {0} is not accessible with ssh.'\
.format(vm_ip)
def is_accessible():
try:
with self.env.get_ssh_to_vm(
vm_ip, username, password, pkeys) as vm_remote:
vm_remote.execute("date")
return True
except ssh_exception.SSHException:
return False
except ssh_exception.NoValidConnectionsError:
return False
wait(is_accessible,
sleep_seconds=10, timeout_seconds=60,
waiting_for=error_msg)
def test_force_delete_inst_before_deferred_cleanup(
self, set_recl_inst_interv, instances, volumes):
"""Force delete of instance before deferred cleanup
Actions:
1. Update '/etc/nova/nova.conf' with long 'reclaim_instance_interval'
and restart Nova on all nodes;
2. Create net and subnet;
3. Create and run two instances (vm1, vm2) inside same net;
4. Create a volume and attach it to an instance vm1;
5. Delete instance vm1 and check that it's in 'SOFT_DELETE' state;
6. Delete instance vm1 with 'force' option and check that it's not
present.
7. Check that volume is released now and has an Available state;
8. Attach the volume to vm2 instance to ensure that the volume's reuse
doesn't call any errors.
"""
timeout = 60 # (sec) timeout to wait instance for status change
# Create two vms
vm1, vm2 = instances
# Create a volume and attach it to an instance vm1
volume = common_functions.create_volume(
self.os_conn.cinder, image_id=None)
self.os_conn.nova.volumes.create_server_volume(
server_id=vm1.id, volume_id=volume.id, device='/dev/vdb')
volumes.append(volume)
# Delete instance vm1 and check that it's in "SOFT_DELETED" state
common_functions.delete_instance(self.os_conn.nova, vm1.id)
assert vm1 not in self.os_conn.get_servers()
common_functions.wait(
lambda: self.os_conn.server_status_is(vm1, 'SOFT_DELETED'),
timeout_seconds=timeout, sleep_seconds=5,
waiting_for='instance {0} changes status to SOFT_DELETED'.format(
vm1.name))
# Force delete and check vm1 not present
common_functions.delete_instance(self.os_conn.nova, vm1.id, force=True)
common_functions.wait(
lambda: self.os_conn.is_server_deleted(vm1.id),
timeout_seconds=timeout, sleep_seconds=5,
waiting_for='instance {0} to be forced deleted'.format(vm1.name))
# Check that volume is released now and has an Available state
assert common_functions.check_volume_status(
self.os_conn.cinder, volume.id, 'available', 1)
# Check volume is not attached
assert self.os_conn.cinder.volumes.get(volume.id).attachments == []
# Attach the volume to vm2 instance
self.os_conn.nova.volumes.create_server_volume(
server_id=vm2.id, volume_id=volume.id, device='/dev/vdb')
# Check volume status is 'in-use' after re-attach
assert common_functions.check_volume_status(
self.os_conn.cinder, volume.id, 'in-use', 1)
# Check that volume has correct server id
volume = self.os_conn.cinder.volumes.get(volume.id)
assert volume.attachments[0]['server_id'] == vm2.id
def wait_instance_to_boot(self):
common_functions.wait(
lambda: self.is_instance_ready(self.instance),
timeout_seconds=45 * 60,
sleep_seconds=60,
waiting_for="windows instance to boot",
)
def deploy_env(self):
"""Deploy env and wait till it will be deployed"""
deploy_task = self.env.deploy_changes()
wait(lambda: is_task_ready(deploy_task),
timeout_seconds=60 * self.TIMEOUT_FOR_DEPLOY,
sleep_seconds=60,
waiting_for='changes to be deployed')
def delete_env(self, timeout=2):
self.env.reset()
wait(lambda: self.env.status == 'new',
timeout_seconds=60 * timeout,
sleep_seconds=20,
waiting_for="Env reset finish")
self.env.delete()
def cleanup(os_conn):
def instances_cleanup(os_conn):
instances = os_conn.nova.servers.list()
for instance in instances:
instance.delete()
common.wait(lambda: len(os_conn.nova.servers.list()) == 0,
timeout_seconds=10 * 60, waiting_for='instances cleanup')
initial_images = os_conn.nova.images.list()
instances_cleanup(os_conn)
yield
instances_cleanup(os_conn)
images = [image for image in os_conn.nova.images.list() if
image not in initial_images]
for image in images:
image.delete()
common.wait(lambda: len(os_conn.nova.images.list()) == len(initial_images),
timeout_seconds=10 * 60, waiting_for='images cleanup')
for volume in os_conn.cinder.volumes.list():
if volume.name != 'nfv_volume':
volume.delete()
common.wait(lambda: volume not in os_conn.cinder.volumes.list(),
timeout_seconds=10 * 60, waiting_for='volumes cleanup')
def assign_floating_ip(self, srv, use_neutron=False):
if use_neutron:
# Find external net id for tenant
nets = self.neutron.list_networks()['networks']
err_msg = "Active external network not found in nets:{}"
ext_net_ids = [
net['id'] for net in nets
if net['router:external'] and net['status'] == "ACTIVE"]
assert ext_net_ids, err_msg.format(nets)
net_id = ext_net_ids[0]
# Find instance port
ports = self.neutron.list_ports(device_id=srv.id)['ports']
err_msg = "Not found active ports for instance:{}"
assert ports, err_msg.format(srv.id)
port = ports[0]
# Create floating IP
body = {'floatingip': {'floating_network_id': net_id,
'port_id': port['id']}}
flip = self.neutron.create_floatingip(body)
# Wait active state for port
port_id = flip['floatingip']['port_id']
wait(lambda: self.neutron.show_port(port_id)['port']['status'] ==
"ACTIVE",
timeout_seconds=60,
waiting_for="floating_ip port is active")
return flip['floatingip']
fl_ips_pool = self.nova.floating_ip_pools.list()
if fl_ips_pool:
floating_ip = self.nova.floating_ips.create(
pool=fl_ips_pool[0].name)
self.nova.servers.add_floating_ip(srv, floating_ip)
return floating_ip
def boot_instance(self, image, flavor, keypair, **kwargs):
"""Boot and return ironic instance
:param os_conn: initialized `os_conn` fixture
:type os_conn: mos_tests.environment.os_actions.OpenStackActions
:param image: image to boot instance with it
:type image: warlock.core.image
:param flavor: baremetal flavor
:type flavor: novaclient.v2.flavors.Flavor
:param keypair: SSH keypair to instance
:type keypair: novaclient.v2.keypairs.Keypair
:return: created instance
:rtype: novaclient.v2.servers.Server
"""
common.wait(self.all_nodes_provisioned,
timeout_seconds=3 * 60,
sleep_seconds=15,
waiting_for='ironic nodes to be provisioned')
baremetal_net = self.os_conn.nova.networks.find(label='baremetal')
return self.os_conn.create_server('ironic-server',
image_id=image.id,
flavor=flavor.id,
key_name=keypair.name,
nics=[{'net-id': baremetal_net.id}],
timeout=60 * 10,
**kwargs)
def clear_dhcp_agent(self, node_to_clear, host, network_name=None,
wait_for_rescheduling=True):
"""Clear DHCP agent after ban and wait until agents rescheduling.
:param node_to_clear: dhcp-agent host to clear
:param host: host or ip of controller onto execute ban command
:param network_name: name of network to determine node with dhcp agents
:param wait_for_rescheduling: wait until dhcp-agent reschedule
:returns: str, name of cleared node
"""
list_dhcp_agents = lambda: self.os_conn.list_all_neutron_agents(
agent_type='dhcp', filter_attr='host')
if network_name:
network = self.os_conn.neutron.list_networks(
name=network_name)['networks'][0]
list_dhcp_agents = (
lambda: self.os_conn.get_node_with_dhcp_for_network(
network['id']))
# clear dhcp agent on provided node
with self.env.get_ssh_to_node(host) as remote:
remote.check_call(
"pcs resource clear neutron-dhcp-agent {0}".format(
node_to_clear))
# Wait to reschedule dhcp agent
if wait_for_rescheduling:
wait(
lambda: (node_to_clear in list_dhcp_agents()),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for="DHCP agent {0} to reschedule".format(
node_to_clear))
return node_to_clear
def delete_volumes(self, volumes):
names = ', '.join([x.name for x in volumes])
for volume in volumes:
volume.get()
# if volume attached to any instance
for attach in volume.attachments:
serv_id = attach['server_id']
self.nova.volumes.delete_server_volume(serv_id, volume.id)
# if volume have snapshots
snapshots = self.cinder.volume_snapshots.findall(
volume_id=volume.id)
for snapshot in snapshots:
self.cinder.volume_snapshots.delete(snapshot)
wait(lambda: not any([self.cinder.volume_snapshots.findall(
volume_id=x.id) for x in volumes]),
timeout_seconds=60 * 5,
sleep_seconds=10,
waiting_for=('snapshots from volumes [{names}] '
'to be deleted').format(names=names))
wait(lambda: all([self.cinder.volumes.get(x.id).status == 'available'
for x in volumes]),
timeout_seconds=60 * 5,
sleep_seconds=10,
waiting_for=('volumes [{names}] '
'to became available').format(names=names))
for volume in volumes:
self.cinder.volumes.delete(volume.id)
# Too fast deletion requests make deletion too long
time.sleep(2)
self.wait_volumes_deleted(volumes)
def assign_floating_ip(self, srv, use_neutron=False):
if use_neutron:
# Find external net id for tenant
nets = self.neutron.list_networks()['networks']
err_msg = "Active external network not found in nets:{}"
ext_net_ids = [
net['id'] for net in nets
if net['router:external'] and net['status'] == "ACTIVE"]
assert ext_net_ids, err_msg.format(nets)
net_id = ext_net_ids[0]
# Find instance port
ports = self.neutron.list_ports(device_id=srv.id)['ports']
err_msg = "Not found active ports for instance:{}"
assert ports, err_msg.format(srv.id)
port = ports[0]
# Create floating IP
body = {'floatingip': {'floating_network_id': net_id,
'port_id': port['id']}}
flip = self.neutron.create_floatingip(body)
# Wait active state for port
port_id = flip['floatingip']['port_id']
wait(lambda:
self.neutron.show_port(port_id)['port']['status'] == "ACTIVE",
timeout_seconds=60,
waiting_for="floating_ip port is active")
return flip['floatingip']
fl_ips_pool = self.nova.floating_ip_pools.list()
if fl_ips_pool:
floating_ip = self.nova.floating_ips.create(
pool=fl_ips_pool[0].name)
self.nova.servers.add_floating_ip(srv, floating_ip)
return floating_ip
def check_vm_is_available(self, vm,
username=None, password=None, pkeys=None):
"""Check that instance is available for connect from controller.
:param vm: instance to ping from it compute node
:param username: username to login to instance
:param password: password to connect to instance
:param pkeys: private keys to connect to instance
"""
vm = self.os_conn.get_instance_detail(vm)
srv_host = self.env.find_node_by_fqdn(
self.os_conn.get_srv_hypervisor_name(vm)).data['ip']
vm_ip = self.os_conn.get_nova_instance_ips(vm)['floating']
with self.env.get_ssh_to_node(srv_host) as remote:
cmd = "ping -c1 {0}".format(vm_ip)
waiting_for_msg = (
'Waiting for instance with ip {0} has '
'connectivity from node with ip {1}.').format(vm_ip, srv_host)
wait(lambda: remote.execute(cmd)['exit_code'] == 0,
sleep_seconds=10, timeout_seconds=3 * 60,
waiting_for=waiting_for_msg)
return self.check_vm_is_accessible_with_ssh(
vm_ip, username=username, password=password, pkeys=pkeys)
def reset_computes(self, hostnames, env_name):
def get_hypervisors():
return [x for x in self.os_conn.nova.hypervisors.list()
if x.hypervisor_hostname in hostnames]
node_states = defaultdict(list)
def is_nodes_started():
for hypervisor in get_hypervisors():
state = hypervisor.state
prev_states = node_states[hypervisor.hypervisor_hostname]
if len(prev_states) == 0 or state != prev_states[-1]:
prev_states.append(state)
return all(x[-2:] == ['down', 'up'] for x in node_states.values())
logger.info('Resetting computes {}'.format(hostnames))
for hostname in hostnames:
node = self.env.find_node_by_fqdn(hostname)
devops_node = DevopsClient.get_node_by_mac(env_name=env_name,
mac=node.data['mac'])
devops_node.reset()
wait(is_nodes_started, timeout_seconds=10 * 60)
def test_image_create_delete_from_url(glance, suffix, option):
"""Check image creation and deletion from URL
Scenario:
1. Create image from URL
2. Wait until image has active `status`
3. Delete image
4. Check that image deleted
"""
name = 'Test_{}'.format(suffix)
image_url = settings.GLANCE_IMAGE_URL
cmd = ('image-create --name {name} --container-format bare '
'--disk-format qcow2 {option} {image_url} --progress'.format(
name=name,
option=option,
image_url=image_url))
image = parser.details(glance(cmd))
def is_image_active():
image_data = parser.details(glance('image-show {id}'.format(**image)))
return image_data['status'] == 'active'
wait(is_image_active, timeout_seconds=60, waiting_for='image is active')
glance('image-delete {id}'.format(**image))
check_image_not_in_list(glance, image)
def force_l3_reschedule(self,
router_id,
new_l3_agt_id=None,
current_l3_agt_id=None):
logger.info('going to reschedule the router on new agent')
if current_l3_agt_id is None:
l3_agents = self.neutron.list_l3_agent_hosting_routers(
router_id)['agents']
if len(l3_agents) != 1:
raise Exception("Can't determine l3 agent to move router from")
current_l3_agt_id = l3_agents[0]['id']
if new_l3_agt_id is None:
all_l3_agts = self.neutron.list_agents(
binary='neutron-l3-agent')['agents']
available_l3_agts = [
agt for agt in all_l3_agts if agt['id'] != current_l3_agt_id
]
new_l3_agt_id = available_l3_agts[0]['id']
self.neutron.remove_router_from_l3_agent(current_l3_agt_id, router_id)
self.neutron.add_router_to_l3_agent(new_l3_agt_id,
{"router_id": router_id})
wait(lambda: self.neutron.list_l3_agent_hosting_routers(router_id),
timeout_seconds=5 * 60,
waiting_for="router moved to new agent")
def test_image_size_attributes(instance, os_conn):
"""Check the OS-EXT-IMG-SIZE:size extended attribute
Scenario:
1. Create net and subnet
2. Check that TestVM image has an OS-EXT-IMG-SIZE:size attribute
3. Boot instance with TestVM image on net
4. Wait for instance to reach ACTIVE status
5. Create new image as snapshot of instance
6. Check that the created snapshot has an
OS-EXT-IMG-SIZE:size attribute
"""
attr = 'OS-EXT-IMG-SIZE:size'
test_vm_image = os_conn.nova.images.find(name='TestVM')
assert hasattr(test_vm_image, attr)
common.wait(lambda: os_conn.is_server_active(instance),
timeout_seconds=60,
waiting_for='instance to became to ACTIVE status')
instance = os_conn.nova.servers.get(instance.id)
snapshot_id = instance.create_image('snap1')
snapshot = os_conn.nova.images.get(snapshot_id)
assert hasattr(snapshot, attr)
def test_ban_l3_agents_and_clear_first(self):
"""Ban all l3-agents, clear first of them and check health of l3-agent
Scenario:
1. Revert snapshot with neutron cluster
2. Create network1, network2
3. Create router1 and connect it with network1, network2 and
external net
4. Boot vm1 in network1 and associate floating ip
5. Boot vm2 in network2
6. Add rules for ping
7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other
8. Ban l3-agent on what router1 is
9. Wait for route rescheduling
10. Repeat steps 7-8
11. Ban l3-agent on what router1 is
12. Clear first banned L3 agent
13. Check that router moved to the health l3-agent
14. Boot one more VM (VM3) in network1
15. Boot vm3 in network1
16. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip)
from each other
Duration 10m
"""
net_id = self.os_conn.neutron.list_networks(name="net01")["networks"][0]["id"]
devops_node = self.get_node_with_dhcp(net_id)
ip = devops_node.data["ip"]
# ban l3 agents
first_banned_node = self.ban_l3_agent(router_name="router01", _ip=ip)
self.ban_l3_agent(router_name="router01", _ip=ip)
self.ban_l3_agent(router_name="router01", _ip=ip, wait_for_migrate=False, wait_for_die=False)
# clear first banned l3 agent
self.clear_l3_agent(_ip=ip, router_name="router01", node=first_banned_node, wait_for_alive=True)
# wait for router migrate to cleared node
router = self.os_conn.neutron.list_routers(name="router01")["routers"][0]
waiting_for = "l3 agent wasn't migrate to {0}"
wait(
lambda: first_banned_node == self.os_conn.get_l3_agent_hosts(router["id"])[0],
timeout_seconds=60 * 3,
waiting_for=waiting_for.format(first_banned_node),
sleep_seconds=(1, 60),
)
# create another server on net01
net01 = self.os_conn.nova.networks.find(label="net01")
self.os_conn.create_server(
name="server03",
availability_zone="{}:{}".format(self.zone.zoneName, self.hosts[0]),
key_name=self.instance_keypair.name,
nics=[{"net-id": net01.id}],
security_groups=[self.security_group.id],
)
# check pings
self.check_vm_connectivity()
def test_os_instance_attributes(request, error_instance, os_conn, attrs):
"""Check instance extended attributes
Scenario:
1. Create net and subnet
2. Boot instance on net
3. Check that attributes `attrs` are visible in instance attributes
4. Wait instance to reach ACTIVE status
5. Check that attributes `attrs` are visible in instance attributes
6. Boot instance in ERROR status
7. Check that attributes `attrs` are visible in instance attributes
"""
instance = request.getfuncargvalue('instance')
common.wait(lambda: os_conn.server_status_is(instance, 'BUILD'),
timeout_seconds=60,
waiting_for='instance to became to BUILD status')
instance = os_conn.nova.servers.get(instance.id)
for attr in attrs:
assert hasattr(instance, attr)
common.wait(lambda: os_conn.is_server_active(instance),
timeout_seconds=60,
waiting_for='instance to became to ACTIVE status')
instance = os_conn.nova.servers.get(instance.id)
for attr in attrs:
assert hasattr(instance, attr)
error_instance = os_conn.nova.servers.get(error_instance.id)
for attr in attrs:
assert hasattr(error_instance, attr)
def _prepare_neutron_server_and_env(self, net_count):
"""Prepares neutron service network count on dhcp agent
and prepares env.
:param net_count: how many networks musth dhcp agent handle
"""
def _check_neutron_restart():
try:
self.os_conn.list_networks()['networks']
except Exception as e:
logger.debug(e)
return False
return True
all_controllers = self.env.get_nodes_by_role('controller')
for controller in all_controllers:
with controller.ssh() as remote:
self._apply_new_neutron_param_value(remote, net_count)
wait(
lambda: _check_neutron_restart(),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for='neutron to be up')
self._prepare_openstack_state()
def wait_hypervisor_be_free(self, hypervisor):
hyp_id = hypervisor.id
wait(lambda: (self.nova.hypervisors.get(hyp_id).running_vms == 0),
timeout_seconds=2 * 60,
sleep_seconds=5,
waiting_for='hypervisor {0} to be free'.format(
hypervisor.hypervisor_hostname))
def _prepare_neutron_server_and_env(self, net_count):
"""Prepares neutron service network count on dhcp agent
and prepares env.
:param net_count: how many networks musth dhcp agent handle
"""
def _check_neutron_restart():
try:
self.os_conn.list_networks()['networks']
except Exception as e:
logger.debug(e)
return False
return True
all_controllers = self.env.get_nodes_by_role('controller')
for controller in all_controllers:
with controller.ssh() as remote:
res = self._apply_new_neutron_param_value(remote, net_count)
error_msg = (
'Neutron service restart with new value failed, '
'exit code {exit_code},'
'stdout {stdout}, stderr {stderr}').format(**res)
assert 0 == res['exit_code'], error_msg
wait(
lambda: _check_neutron_restart(),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for='neutron to be up')
self._prepare_openstack_state()
def clear_l3_agent(self, _ip, router_name, node, wait_for_alive=False):
"""Clear L3 agent ban and wait until router moved to this node
Clear previously banned L3 agent on node wait until router moved
to this node
:param _ip: ip of server to to execute clear command
:param router_name: name of router to wait until it move to node
:param node: name of node to clear
:param wait_for_alive:
"""
router = self.os_conn.neutron.list_routers(
name=router_name)['routers'][0]
with self.env.get_ssh_to_node(_ip) as remote:
remote.check_call(
"pcs resource clear neutron-l3-agent {0}".format(node))
logger.info("Clear L3 agent on node {0}".format(node))
# wait for l3 agent alive
if wait_for_alive:
wait(
lambda: self.os_conn.get_l3_for_router(
router['id'])['agents'][0]['alive'] is True,
timeout_seconds=60 * 3, waiting_for="L3 agent is alive",
sleep_seconds=(1, 60)
)
def test_destroy_master_rabbit_node(self):
"""Destroy one of RabbitMQ nodes .
Actions:
1. Poweroff master standalone rabbitmq node;
2. Wait some time for rabbitmq cluster to recover;
3. Check RabbitMQ health with rabbitmqctl;
4. Check that old master is offline;
5. Check that new master != old master;
"""
timeout = 5 # minutes, wait for rabbit recover
# Get master standalone rabbit node for disabling
old_master = self.rabbit_node('master')
old_master_fqdn = old_master.data['fqdn']
# Disable master rabbit node
logger.debug("Disabling RabbitMQ master node")
self.disable_node(old_master)
# Wait for rabbit cluster to recover
logger.debug("Sleeping for %s minutes" % timeout)
sleep(60 * timeout)
# Check rabbit status,
wait(lambda: self.is_rabbit_cluster_ok(),
timeout_seconds=60 * timeout,
sleep_seconds=30,
waiting_for="RabbitMQ became online")
# Check that old master now offline
assert self.rabbit_nodes_statuses()[old_master_fqdn] == 'offline'
# Check that now we have a new master
assert old_master_fqdn not in self.rabbit_nodes_roles()
def ironic_nodes(request, env, ironic_drivers_params, ironic, devops_env):
node_count = getattr(request, 'param', 1)
devops_nodes = []
nodes = []
for i, config in enumerate(ironic_drivers_params[:node_count]):
devops_node, node = make_ironic_node(config=config,
devops_env=devops_env,
ironic=ironic,
name='baremetal_{i}'.format(i=i),
fuel_env=env)
nodes.append(node)
if devops_node is not None:
devops_nodes.append(devops_node)
common.wait(lambda: env.is_ostf_tests_pass('sanity'),
timeout_seconds=60 * 5,
waiting_for='OSTF sanity tests to pass')
yield nodes
for node in nodes:
ironic.delete_node(node)
for node in devops_nodes:
devops_env.del_node(node)
def _prepare_neutron_server_and_env(self, net_count):
"""Prepares neutron service network count on dhcp agent
and prepares env.
:param net_count: how many networks musth dhcp agent handle
"""
def _check_neutron_restart():
try:
self.os_conn.list_networks()['networks']
except Exception as e:
logger.debug(e)
return False
return True
all_controllers = self.env.get_nodes_by_role('controller')
for controller in all_controllers:
with controller.ssh() as remote:
res = self._apply_new_neutron_param_value(remote, net_count)
error_msg = (
'Neutron service restart with new value failed, '
'exit code {exit_code},'
'stdout {stdout}, stderr {stderr}').format(**res)
assert 0 == res['exit_code'], error_msg
wait_msg = "Waiting for neutron is up"
wait(
lambda: _check_neutron_restart(),
timeout_seconds=60 * 3,
sleep_seconds=(1, 60, 5),
waiting_for=wait_msg)
self._prepare_openstack_state()
def servers(os_conn, os_clients, networks, sec_groups):
# boot instances
zone = os_conn.nova.availability_zones.find(zoneName="nova")
hostname = zone.hosts.keys()[0]
servers = []
for i, (os_conn, network,
sec_group) in enumerate(zip(os_clients, networks, sec_groups)):
server1 = os_conn.create_server(name='server%02d' % (i * 2 + 1),
availability_zone='{}:{}'.format(
zone.zoneName, hostname),
nics=[{
'net-id': network['id']
}],
security_groups=[sec_group['id']],
fixed_ip='10.0.0.4',
wait_for_active=False,
wait_for_avaliable=False)
server2 = os_conn.create_server(name='server%02d' % (i * 2 + 2),
availability_zone='{}:{}'.format(
zone.zoneName, hostname),
nics=[{
'net-id': network['id']
}],
security_groups=[sec_group['id']],
fixed_ip='10.0.0.5',
wait_for_active=False,
wait_for_avaliable=False)
servers.extend([server1, server2])
def is_all_instances_ready():
for os_conn in os_clients:
for server in os_conn.nova.servers.list():
if not os_conn.is_server_active(server):
return False
if not os_conn.is_server_ssh_ready(server):
return False
return True
wait(is_all_instances_ready,
timeout_seconds=3 * 60,
waiting_for='all instances are ready')
# update states
for i, server in enumerate(servers):
servers[i] = server.manager.get(server)
yield servers
for server in servers:
server.delete()
def is_instances_deleted():
for os_conn in os_clients:
if not all(os_conn.is_server_deleted(x.id) for x in servers):
return False
return True
wait(is_instances_deleted,
timeout_seconds=60,
waiting_for='instances deleted')
def create_volume_from_vm(self, os_conn, vm, size=1):
image = os_conn.nova.servers.create_image(vm, image_name="image_vm2")
common.wait(lambda: os_conn.nova.images.get(image).status == 'ACTIVE',
timeout_seconds=10 * 60,
waiting_for='image changes status to ACTIVE')
volume = common.create_volume(os_conn.cinder, image, size=size)
return volume.id
def test_delete_node(self, env, roles, ironic, make_image, flavors,
keypair, os_conn, ironic_nodes):
"""Delete one of multiple ironic nodes.
Scenario:
1. Remove created ironic node from cluster
2. Boot new ironic instance
3. Check ironic instance status is ACTIVE
"""
fuel_node = [x for x in env.get_all_nodes()
if x.data['name'] == self.node_name][0]
if 'ceph-osd' in roles:
with fuel_node.ssh() as remote:
remove_ceph_from_node(remote)
env.unassign([fuel_node.id])
# Deploy changes
task = env.deploy_changes()
common.wait(lambda: common.is_task_ready(task),
timeout_seconds=60 * 60,
sleep_seconds=60,
waiting_for='changes to be deployed')
image = make_image(node_driver=ironic_nodes[0].driver)
instance = ironic.boot_instance(image=image,
flavor=flavors[0],
keypair=keypair)
assert os_conn.nova.servers.get(instance.id).status == 'ACTIVE'
def create_server(self, name, image_id=None, flavor=1, scenario='',
files=None, key_name=None, timeout=300,
wait_for_active=True, wait_for_avaliable=True, **kwargs):
try:
if scenario:
with open(scenario, "r+") as f:
scenario = f.read()
except Exception as exc:
logger.info("Error opening file: %s" % exc)
raise Exception()
if image_id is None:
image_id = self._get_cirros_image().id
srv = self.nova.servers.create(name=name,
image=image_id,
flavor=flavor,
userdata=scenario,
files=files,
key_name=key_name,
**kwargs)
if wait_for_active:
wait(lambda: self.is_server_active(srv),
timeout_seconds=timeout, sleep_seconds=5,
waiting_for='instance {0} changes status to ACTIVE'.format(
name))
# wait for ssh ready
if wait_for_avaliable:
if self.env is not None:
wait(lambda: self.is_server_ssh_ready(srv),
timeout_seconds=timeout,
waiting_for='server available via ssh')
logger.info('the server {0} is ready'.format(srv.name))
return self.get_instance_detail(srv.id)
def remove_ceph_from_node(remote):
def is_pg_clean():
result = remote.check_call('ceph pg stat -f json-pretty',
verbose=False)
pg_stat = json.loads(result.stdout_string)
states = pg_stat['num_pg_by_state']
return len(states) == 1 and states[0]['name'] == 'active+clean'
hostname = remote.check_call('hostname -f', verbose=False).stdout_string
result = remote.check_call('ceph report', verbose=False)
ceph_data = json.loads(result.stdout_string)
osd_ids = [x['id'] for x in ceph_data['osd_metadata']
if x['hostname'] == hostname]
for osd_id in osd_ids:
remote.check_call('ceph osd out {0}'.format(osd_id), verbose=False)
common.wait(is_pg_clean,
timeout_seconds=5 * 60,
sleep_seconds=15,
waiting_for='Ceph data migration to be done')
for osd_id in osd_ids:
remote.check_call("stop ceph-osd id={}".format(osd_id),
verbose=False)
remote.check_call("ceph osd crush remove osd.{}".format(osd_id),
verbose=False)
remote.check_call("ceph auth del osd.{}".format(osd_id),
verbose=False)
remote.check_call("ceph osd rm osd.{}".format(osd_id),
verbose=False)
remote.check_call("ceph osd crush remove {}".format(hostname),
verbose=False)
def create_server(self, name, image_id=None, flavor=1, userdata=None,
files=None, key_name=None, timeout=300,
wait_for_active=True, wait_for_avaliable=True, **kwargs):
if image_id is None:
image_id = self._get_cirros_image().id
srv = self.nova.servers.create(name=name,
image=image_id,
flavor=flavor,
userdata=userdata,
files=files,
key_name=key_name,
**kwargs)
if wait_for_active:
wait(lambda: self.is_server_active(srv),
timeout_seconds=timeout, sleep_seconds=5,
waiting_for='instance {0} changes status to ACTIVE'.format(
name))
# wait for ssh ready
if wait_for_avaliable:
if self.env is not None:
wait(lambda: self.is_server_ssh_ready(srv),
timeout_seconds=timeout,
waiting_for='server available via ssh')
logger.info('the server {0} is ready'.format(srv.name))
return self.get_instance_detail(srv.id)
def boot_instance(self, image, flavor, keypair, **kwargs):
"""Boot and return ironic instance
:param os_conn: initialized `os_conn` fixture
:type os_conn: mos_tests.environment.os_actions.OpenStackActions
:param image: image to boot instance with it
:type image: warlock.core.image
:param flavor: baremetal flavor
:type flavor: novaclient.v2.flavors.Flavor
:param keypair: SSH keypair to instance
:type keypair: novaclient.v2.keypairs.Keypair
:return: created instance
:rtype: novaclient.v2.servers.Server
"""
common.wait(self.get_provisioned_node,
timeout_seconds=3 * 60,
sleep_seconds=15,
waiting_for='ironic node to be provisioned')
baremetal_net = self.os_conn.nova.networks.find(label='baremetal')
return self.os_conn.create_server('ironic-server',
image_id=image.id,
flavor=flavor.id,
key_name=keypair.name,
nics=[{
'net-id': baremetal_net.id
}],
timeout=60 * 10,
**kwargs)
def prepare_openstack(self):
"""Prepare OpenStack for scenarios run
Steps:
1. Create network net01, subnet net01__subnet with CIDR 10.1.1.0/24
2. Create new security group sec_group1
3. Add Ingress rule for TCP protocol to sec_group1
4. Boot vm1 net01 with sec_group1
"""
# init variables
security_group = self.os_conn.create_sec_group_for_ssh()
self.instance_keypair = self.os_conn.create_key(key_name='instancekey')
self.os_conn.nova.security_group_rules.create(security_group.id,
ip_protocol='tcp',
from_port=1,
to_port=65535,
cidr='0.0.0.0/0')
net, subnet = self.create_internal_network_with_subnet(
cidr="10.1.1.0/24")
# create router
router = self.create_router_between_nets(self.os_conn.ext_network,
subnet)['router']
self.server = self.os_conn.create_server(
name='server01',
key_name=self.instance_keypair.name,
nics=[{'net-id': net['network']['id']}],
security_groups=[security_group.id])
# add floating ip to first server
self.floating_ip = self.os_conn.assign_floating_ip(self.server,
use_neutron=True)
pkeys = self.convert_private_key_for_vm(
[self.instance_keypair.private_key])
self.check_vm_is_accessible_with_ssh(
vm_ip=self.floating_ip['floating_ip_address'],
pkeys=pkeys,
**self.cirros_creds)
yield
self.server.delete()
wait(lambda: not self.os_conn.nova.servers.findall(id=self.server.id),
timeout_seconds=2 * 60,
waiting_for="instance to be deleted")
self.os_conn.neutron.delete_floatingip(self.floating_ip['id'])
self.os_conn.router_interface_delete(router['id'],
subnet_id=subnet['subnet']['id'])
self.os_conn.neutron.delete_router(router['id'])
self.os_conn.neutron.delete_network(net['network']['id'])
security_group.delete()
self.instance_keypair.delete()
def wait_instances_to_boot(os_conn, instances):
common.wait(lambda: all(os_conn.is_server_active(x) for x in instances),
timeout_seconds=5 * 60,
waiting_for="instances became to active state")
common.wait(lambda: all(BOOT_MARKER in x.get_console_output()
for x in instances),
timeout_seconds=5 * 60,
waiting_for="instances to be ready")
def warm_start_nodes(self, devops_nodes):
for node in devops_nodes:
logger.info('Starting node {}'.format(node.name))
node.create()
wait(self.check_nodes_get_online_state, timeout_seconds=10 * 60)
logger.info('wait until the nodes get online state')
for node in self.get_all_nodes():
logger.info('online state of node {0} now is {1}'.format(
node.data['name'], node.data['online']))
def create_volume_from_vm(self, os_conn, vm):
image = os_conn.nova.servers.create_image(vm, image_name="image_vm2")
common.wait(lambda: os_conn.nova.images.get(image).status == 'ACTIVE',
timeout_seconds=10 * 60,
waiting_for='image changes status to ACTIVE')
volume = common.create_volume(os_conn.cinder,
image,
volume_type='volumes_lvm')
return volume.id
def test_start_rpc_srv_client_iptables_modify(env,
fixt_open_5000_port_on_nodes,
fixt_kill_rpc_server_client,
patch_iptables, controller):
"""Tests:
Start RabbitMQ RPC server and client and apply IPTABLES DROP rules
for RabbitMQ ports on one controller.
Actions:
1. Apply IPTables Drop OR Reject rules to controller where
'oslo_msg_check_server' will be launched;
2. To be able to use port 5000 from any node open it in IPTables;
3. Install 'oslo.messaging-check-tool' on controller and compute;
4. Prepare config file for both nodes above;
5. Run 'oslo_msg_check_client' on compute node;
6. Run 'oslo_msg_check_server' on controller node (remember point 1);
7. Send GET curl request from host server to 'oslo_msg_check_client'
located on compute node and check that response will '200';
8. Remove all modifications of rules from IPTables and kill serv/client.
"""
exp_resp = 200 # expected response code from curl from RPC client
timeout_min = 2 # (minutes) time to wait for RPC server/client start
compute = random.choice(env.get_nodes_by_role('compute'))
# Get management IPs of all controllers
ctrl_ips = get_mngmnt_ip_of_ctrllrs(env)
# Install and configure tool on controller and compute
for node in (controller, compute):
with node.ssh() as remote:
kwargs = vars_config(remote)
install_oslomessagingchecktool(remote, **kwargs)
configure_oslomessagingchecktool(remote, ctrl_ips,
kwargs['nova_user'],
kwargs['nova_pass'],
kwargs['cfg_file_path'],
kwargs['sample_cfg_file_path'])
# Client: Run 'oslo_msg_check_client' on compute
with compute.ssh() as remote:
rpc_client_ip = rabbit_rpc_client_start(remote,
kwargs['cfg_file_path'])
# Server: Run 'oslo_msg_check_server' on controller
with controller.ssh() as remote:
rabbit_rpc_server_start(remote, kwargs['cfg_file_path'])
# host srv -> client: Check GET
logger.debug('GET: [host server] -> [{0}]'.format(rpc_client_ip))
# need to wait for server/client start
wait(lambda: get_http_code(rpc_client_ip) == exp_resp,
timeout_seconds=60 * timeout_min,
sleep_seconds=20,
waiting_for='RPC server/client to start')
assert get_http_code(rpc_client_ip) == exp_resp
def test_boot_instances_on_different_tenants(env, os_conn, ubuntu_image,
ironic_nodes, ironic, flavors,
tenants_clients):
"""Check instance statuses during instance restart
Scenario:
1. Boot 1st Ironic instance under 1st tenant
2. Boot 2nd Ironic instance under 2nd tenant
3. Check Ironic instances statuses
4. Login via SSH to Ironic instances.
5. Check that instances are accessible for each other in baremetal
network
"""
common.wait(ironic.get_provisioned_node,
timeout_seconds=3 * 60,
sleep_seconds=15,
waiting_for='ironic node to be provisioned')
instances, keypairs, ips = [], [], []
for flavor, tenant_conn in zip(flavors, tenants_clients):
tenant_keypair = tenant_conn.create_key(key_name='ironic-key')
brm_net = tenant_conn.nova.networks.find(label='baremetal')
instance = tenant_conn.create_server('ironic-server',
image_id=ubuntu_image.id,
flavor=flavor.id,
key_name=tenant_keypair.name,
nics=[{
'net-id': brm_net.id
}],
timeout=60 * 10,
wait_for_avaliable=False)
keypairs.append(tenant_keypair)
instances.append(instance)
ips.append(tenant_conn.get_nova_instance_ips(instance)['fixed'])
for instance, tenant_keypair, ip in zip(instances, keypairs, ips[::-1]):
with os_conn.ssh_to_instance(env,
instance,
vm_keypair=tenant_keypair,
username='******') as remote:
result = remote.execute('ping -c 10 {}'.format(ip))
received_packets = int(result['stdout'][-2].split()[3])
assert received_packets > 0
def is_instance_deleted():
return instance not in tenant_conn.nova.servers.list()
for instance, tenant_conn in zip(instances, tenants_clients):
instance.delete()
common.wait(is_instance_deleted,
timeout_seconds=60 * 5,
sleep_seconds=20,
waiting_for="instance is deleted")
def wait_for_rabbit_running_nodes(remote, exp_nodes, timeout_min=5):
"""Waits until number of 'running_nodes' from 'rabbitmqctl cluster_status'
will be as expected number of controllers.
:param remote: SSH connection point to controller.
:param exp_nodes: Expected number of rabbit nodes.
:param timeout_min: Timeout in minutes to wait.
"""
wait(lambda: num_of_rabbit_running_nodes(remote) == exp_nodes,
timeout_seconds=60 * timeout_min,
sleep_seconds=30,
waiting_for='number of running nodes will be %s.' % exp_nodes)
def test_evacuate(devops_env, env, os_conn, instances, keypair):
"""Evacuate instances from failed compute node
Scenario:
1. Create net01, net01__subnet:
neutron net-create net01
neutron subnet-create net01 192.168.1.0/24 --enable-dhcp \
--name net01__subnet
2. Boot instances vm1 and vm2 in net01 on a single compute node:
3. Destroy a compute node where instances are scheduled
4. Evacuate instances vm1 and vm2:
nova evacuate vm1 && nova evacuate vm2
5. Check that they are rescheduled onto another compute node and
are in ACTIVE state:
6. Check that pings between vm1 and vm2 are successful
"""
compute_host = getattr(instances[0], 'OS-EXT-SRV-ATTR:hypervisor_hostname')
compute_node = env.find_node_by_fqdn(compute_host)
devops_node = devops_env.get_node_by_mac(compute_node.data['mac'])
devops_node.destroy()
def is_hypervisor_down():
hypervisor = os_conn.nova.hypervisors.find(
hypervisor_hostname=compute_host)
return hypervisor.state == 'down'
common.wait(
is_hypervisor_down,
timeout_seconds=5 * 60,
waiting_for='hypervisor {0} to be in down state'.format(compute_host))
for instance in instances:
os_conn.nova.servers.evacuate(instance)
def is_instances_migrate():
for instance in os_conn.nova.servers.list():
if instance not in instances:
continue
if instance.status == 'ERROR':
raise Exception('Instance {0.name} is in ERROR status\n'
'{0.fault[message]}\n'
'{0.fault[details]}'.format(instance))
if not os_conn.server_status_is(instance, 'ACTIVE'):
return False
if getattr(instance,
'OS-EXT-SRV-ATTR:hypervisor_hostname') == compute_host:
return False
return True
common.wait(is_instances_migrate,
timeout_seconds=5 * 60,
waiting_for='instances to migrate to another compute')
network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
def test_kill_conductor_service(env, os_conn, ironic_nodes, ubuntu_image,
flavors, keypair, env_name, ironic):
"""Kill ironic-conductor service with one bare-metal node
Scenario:
1. Launch baremetal instance
2. Kill Ironic-conductor service for conductor node that had booted
instance
3. Wait some time
4. Baremetal node must be reassigned to another Ironic-conductor
5. Run OSTF including Ironic tests.
6. Check that Ironic instance still ACTIVE and operable
"""
flavor, ironic_node = zip(flavors, ironic_nodes)[0]
def find_conductor_node(ironic_node_uuid, conductors):
cmd = 'ls /var/log/remote/ironic/{0}/'.format(ironic_node_uuid)
for conductor in conductors:
with conductor.ssh() as remote:
result = remote.execute(cmd)
if result.is_ok:
return conductor
instance = ironic.boot_instance(image=ubuntu_image,
flavor=flavor,
keypair=keypair)
conductors = env.get_nodes_by_role('ironic')
conductor = find_conductor_node(ironic_node.uuid, conductors)
if conductor is None:
raise Exception("Can't find conductor node booted istance")
with conductor.ssh() as remote:
remote.check_call('service ironic-conductor stop')
conductors.remove(conductor)
common.wait(
lambda: (find_conductor_node(ironic_node.uuid, conductors)
not in (conductor, None)), # yapf: disable
timeout_seconds=10 * 60,
waiting_for='node to migrate to another conductor',
sleep_seconds=20)
common.wait(lambda: env.is_ostf_tests_pass('sanity'),
timeout_seconds=5 * 60,
waiting_for='OSTF sanity tests to pass')
assert os_conn.nova.servers.get(instance.id).status == 'ACTIVE'
with os_conn.ssh_to_instance(env,
instance,
vm_keypair=keypair,
username='******') as remote:
remote.check_call('uname')
def force_dhcp_reschedule(self, net_id, new_dhcp_agt_id):
logger.info('going to reschedule network to specified '
'controller dhcp agent')
current_dhcp_agt_id = self.neutron.list_dhcp_agent_hosting_networks(
net_id)['agents'][0]['id']
self.neutron.remove_network_from_dhcp_agent(current_dhcp_agt_id,
net_id)
self.neutron.add_network_to_dhcp_agent(new_dhcp_agt_id,
{'network_id': net_id})
wait(lambda: self.neutron.list_dhcp_agent_hosting_networks(net_id),
timeout_seconds=5 * 60,
waiting_for="network reschedule to new dhcp agent")
def test_deploy_baremetal_nodes_on_2_envs(env, ironic, ubuntu_image, flavors,
keypair, env2, env2_ironic,
env2_ubuntu_image, env2_flavors,
env2_keypair):
"""Check deploy bare-metal nodes on two environments concurrently
Scenario:
1. Launch baremetal instance from env1
2. Launch baremetal instance from env2
3. Verify that "Provisioning State" became "available" on baremetal
nodes in both Envs.
4. Check that baremetal instances ase SSH available
"""
instance1 = ironic.boot_instance(image=ubuntu_image,
flavor=flavors[0],
keypair=keypair,
wait_for_active=False,
wait_for_avaliable=False)
instance2 = env2_ironic.boot_instance(image=env2_ubuntu_image,
flavor=env2_flavors[0],
keypair=env2_keypair,
wait_for_active=False,
wait_for_avaliable=False)
def is_instances_active():
return (env.os_conn.is_server_active(instance1)
and env2.os_conn.is_server_active(instance2))
common.wait(is_instances_active,
timeout_seconds=10 * 60,
waiting_for='instances became active state')
def is_ssh_ready():
return (env.os_conn.is_server_ssh_ready(instance1)
and env2.os_conn.is_server_ssh_ready(instance2))
common.wait(is_ssh_ready,
timeout_seconds=5 * 60,
waiting_for='instances to be available via SSH')
with env.os_conn.ssh_to_instance(env,
instance1,
vm_keypair=keypair,
username='******') as remote:
remote.check_call('uname')
with env2.os_conn.ssh_to_instance(env2,
instance2,
vm_keypair=env2_keypair,
username='******') as remote:
remote.check_call('uname')
def test_shutdown_non_primary_controller_dhcp_agent(self):
"""Shutdown non-primary controller and check dhcp-agent
Scenario:
2. Create network1, subnet1, router1
3. Launch instances vm1 in network1
4. Find non-primary controller
5. Check on what agents is network1. Reschedule agents if needed.
6. Run udhcp on vm1
7. Shutdown non-primary controller
8. Check that all networks reschedule from non-primary controller
9. Run udhcp on vm1
Duration 10m
"""
self._prepare_openstack()
agents_hosts = self.os_conn.get_node_with_dhcp_for_network(
self.networks[0])
# Check if the agent is not on the non-primary controller
# Reschedule if needed
if not any(self.non_primary_host in host for host in agents_hosts):
self.os_conn.reschedule_dhcp_agent(self.networks[0],
self.non_primary_host)
# Get non-primary controller agent id
agent_ids = [
agt['id'] for agt in self.os_conn.neutron.list_agents(
binary='neutron-dhcp-agent')['agents']
if self.non_primary_host in agt['host']
]
# Run udhcp on vm
self.run_udhcpc_on_vm(self.server1)
# Destroy non-primary controller
self.env.destroy_nodes([self.non_primary_node])
# Wait some time while agents become down
self.os_conn.wait_agents_down(agent_ids)
# Check that all networks reschedule from non-primary controller
wait(lambda: len(
self.os_conn.neutron.list_networks_on_dhcp_agent(agent_ids[0])[
'networks']) == 0,
timeout_seconds=60 * 5,
sleep_seconds=(1, 60, 5),
waiting_for="migrating all networks from died dhcp agent")
# Run udhcp on vm
self.run_udhcpc_on_vm(self.server1)
def migrate(self, os_conn, vm):
os_conn.nova.servers.migrate(vm)
common.wait(
lambda: os_conn.nova.servers.get(vm).status == 'VERIFY_RESIZE',
timeout_seconds=3 * 60,
waiting_for='instance {} changes status to VERIFY_RESIZE during '
'migration'.format(vm.name))
os_conn.nova.servers.confirm_resize(vm)
common.wait(lambda: os_conn.is_server_active(vm),
timeout_seconds=5 * 60,
waiting_for='instance {} changes status to ACTIVE after '
'migration'.format(vm.name))
return os_conn.nova.servers.get(vm)
def test_instance_terminate(env, ironic, os_conn, ironic_nodes, ubuntu_image,
flavors, keypair, instance):
"""Check terminate instance
Scenario:
1. Boot Ironic instance
2. Terminate Ironic instance
3. Wait and check that instance not present in nova list
"""
instance.delete()
common.wait(lambda: not os_conn.nova.servers.list().count(instance),
timeout_seconds=60,
waiting_for="instance is terminated")
def test_reboot_all_ironic_conductors(env, env_name):
"""Check ironic state after restart all conductor nodes
Scenario:
1. Ensure ironic conductor service works correctly on each conductor
node (e.g., ensure "ironic driver-list" returns correct list of
drivers)
2. Shutdown all ironic conductor nodes
3. Turn on all ironic conductor nodes
4. SSH to every conductor and ensure conductor service works fine.
"""
controller = env.get_nodes_by_role('controller')[0]
with controller.ssh() as remote:
with remote.open('/root/openrc') as f:
openrc = f.read()
conductors = env.get_nodes_by_role('ironic')
drivers = None
for conductor in conductors:
with conductor.ssh() as remote:
with remote.open('/root/openrc', 'w') as f:
f.write(openrc)
drivers_data = os_cli.Ironic(remote)('driver-list').listing()
assert len(drivers_data) > 0
if drivers is None:
drivers = drivers_data
else:
assert drivers == drivers_data
devops_nodes = [devops_client.DevopsClient.get_node_by_mac(
env_name=env_name,
mac=x.data['mac']) for x in conductors]
for node in devops_nodes:
node.destroy()
for node in devops_nodes:
node.start()
common.wait(lambda: all(x.is_ssh_avaliable() for x in conductors),
timeout_seconds=10 * 60,
waiting_for='conductor nodes to boot')
for conductor in conductors:
with conductor.ssh() as remote:
with remote.open('/root/openrc', 'w') as f:
f.write(openrc)
ironic_cli = os_cli.Ironic(remote)
assert ironic_cli('driver-list').listing() == drivers
def test_instance_rebuild(env, ironic, os_conn, ironic_nodes, ubuntu_image,
flavors, keypair, instance):
"""Check rebuild instance
Scenario:
1. Boot Ironic instance
2. Rebuild Ironic instance (nova rebuild <server> <image>)
3. Check that instance status became REBUILD
4. Wait until instance returns back to ACTIVE status.
"""
server = os_conn.rebuild_server(instance, ubuntu_image.id)
common.wait(lambda: os_conn.nova.servers.get(server).status == 'ACTIVE',
timeout_seconds=60 * 10,
waiting_for="instance is active")
def kill_rabbitmq_on_node(remote, timeout_min=7):
"""Waiting for rabbit startup and got pid, then kill-9 it"""
def get_pid():
cmd = "rabbitmqctl status | grep '{pid' | tr -dc '0-9'"
try:
return remote.check_call(cmd)['stdout'][0].strip()
except Exception:
return None
wait(get_pid,
timeout_seconds=60 * timeout_min,
sleep_seconds=30,
waiting_for='Rabbit get its pid on %s.' % remote.host)
cmd = "kill -9 %s" % get_pid()
remote.check_call(cmd)
def wait_for_deploy(self, environment):
def is_murano_env_deployed():
status = self.murano.environments.get(environment.id).status
if status == 'deploy failure':
raise Exception('Environment deploy finished with errors')
return status == 'ready'
wait(is_murano_env_deployed,
timeout_seconds=1200,
waiting_for='environment is ready')
environment = self.murano.environments.get(environment.id)
logs = self.get_log(environment)
assert 'Deployment finished' in logs
return environment
