def delete_instances(os_conn, instances): instances_ids = [x.id for x in instances] # Stop instances (to prevent error during deletion) for instance_id in instances_ids: os_conn.nova.servers.stop(instance_id) def isnstances_shutdowned(): instances = [x for x in os_conn.nova.servers.list() if x.id in instances_ids] if any([x.status == 'ERROR' for x in instances]): raise Exception( 'Some server(s) became to ERROR state after stop') return all([x.status == 'SHUTOFF' for x in instances]) common.wait(isnstances_shutdowned, timeout_seconds=10 * 60) # Delete instances for instance_id in instances_ids: os_conn.nova.servers.delete(instance_id) def instances_deleted(): not_deleted = [x for x in os_conn.nova.servers.list() if x.id in instances_ids] if len(not_deleted) == 0: return True if any([x.status == 'ERROR' for x in not_deleted]): raise Exception( 'Some server(s) became to ERROR state after deletion') common.wait(instances_deleted, timeout_seconds=2 * 60)
def ironic_nodes(request, env, ironic_drivers_params, ironic, devops_env): node_count = getattr(request, 'param', 1) devops_nodes = [] nodes = [] for i, config in enumerate(ironic_drivers_params[:node_count]): devops_node, node = make_ironic_node(config=config, devops_env=devops_env, ironic=ironic, name='baremetal_{i}'.format(i=i), fuel_env=env) nodes.append(node) if devops_node is not None: devops_nodes.append(devops_node) common.wait(lambda: env.is_ostf_tests_pass('sanity'), timeout_seconds=60 * 5, waiting_for='OSTF sanity tests to pass') yield nodes for node in nodes: ironic.delete_node(node) for node in devops_nodes: devops_env.del_node(node)
def check_vm_is_accessible_with_ssh(self, vm_ip, username=None, password=None, pkeys=None): """Check that instance is accessible with ssh via floating_ip. :param vm_ip: floating_ip of instance :param username: username to login to instance :param password: password to connect to instance :param pkeys: private keys to connect to instance """ error_msg = ('Instance with ip {0} ' 'is not accessible with ssh.').format(vm_ip) def is_accessible(): ssh_client = self.env.get_ssh_to_vm(vm_ip, username, password, pkeys) return ssh_client.check_connection() is True wait(is_accessible, sleep_seconds=10, timeout_seconds=60, waiting_for=error_msg)
def check_vm_is_accessible_with_ssh(self, vm_ip, username=None, password=None, pkeys=None): """Check that instance is accessible with ssh via floating_ip. :param vm_ip: floating_ip of instance :param username: username to login to instance :param password: password to connect to instance :param pkeys: private keys to connect to instance """ error_msg = 'Instance with ip {0} is not accessible with ssh.'\ .format(vm_ip) def is_accessible(): try: with self.env.get_ssh_to_vm( vm_ip, username, password, pkeys) as vm_remote: vm_remote.execute("date") return True except ssh_exception.SSHException: return False except ssh_exception.NoValidConnectionsError: return False wait(is_accessible, sleep_seconds=10, timeout_seconds=60, waiting_for=error_msg)
def check_vm_is_available(self, vm, username=None, password=None, pkeys=None): """Check that instance is available for connect from controller. :param vm: instance to ping from it compute node :param username: username to login to instance :param password: password to connect to instance :param pkeys: private keys to connect to instance """ vm = self.os_conn.get_instance_detail(vm) srv_host = self.env.find_node_by_fqdn( self.os_conn.get_srv_hypervisor_name(vm)).data['ip'] vm_ip = self.os_conn.get_nova_instance_ips(vm)['floating'] with self.env.get_ssh_to_node(srv_host) as remote: cmd = "ping -c1 {0}".format(vm_ip) waiting_for_msg = ( 'Waiting for instance with ip {0} has ' 'connectivity from node with ip {1}.').format(vm_ip, srv_host) wait(lambda: remote.execute(cmd)['exit_code'] == 0, sleep_seconds=10, timeout_seconds=3 * 60, waiting_for=waiting_for_msg) return self.check_vm_is_accessible_with_ssh( vm_ip, username=username, password=password, pkeys=pkeys)
def clear_dhcp_agent(self, node_to_clear, host, network_name=None, wait_for_rescheduling=True): """Clear DHCP agent after ban and wait until agents rescheduling. :param node_to_clear: dhcp-agent host to clear :param host: host or ip of controller onto execute ban command :param network_name: name of network to determine node with dhcp agents :param wait_for_rescheduling: wait until dhcp-agent reschedule :returns: str, name of cleared node """ list_dhcp_agents = lambda: self.os_conn.list_all_neutron_agents( agent_type='dhcp', filter_attr='host') if network_name: network = self.os_conn.neutron.list_networks( name=network_name)['networks'][0] list_dhcp_agents = ( lambda: self.os_conn.get_node_with_dhcp_for_network( network['id'])) # clear dhcp agent on provided node with self.env.get_ssh_to_node(host) as remote: remote.check_call( "pcs resource clear neutron-dhcp-agent {0}".format( node_to_clear)) # Wait to reschedule dhcp agent if wait_for_rescheduling: wait( lambda: (node_to_clear in list_dhcp_agents()), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for="DHCP agent {0} to reschedule".format( node_to_clear)) return node_to_clear
def test_reboot_conductor(env, ironic, os_conn, ironic_nodes, make_image, flavors, keypair, devops_env, boot_instance_before): """Check ironic state after restart conductor node Scenario: 1. Boot Ironic instance (if `boot_instance_before`) 2. Reboot Ironic conductor. 3. Wait 5-10 minutes. 4. Run network verification. 5. Run OSTF including Ironic tests. 6. Verify that CLI ironicclient can list nodes, ports, chassis, drivers 7. Boot new Ironic instance (if not `boot_instance_before`). """ flavor, ironic_node = zip(flavors, ironic_nodes)[0] image = make_image(node_driver=ironic_node.driver) if boot_instance_before: instance = ironic.boot_instance(image=image, flavor=flavor, keypair=keypair) conductor = env.get_nodes_by_role('ironic')[0] devops_node = devops_env.get_node_by_fuel_node(conductor) devops_node.reset() time.sleep(10) common.wait(conductor.is_ssh_avaliable, timeout_seconds=60 * 10, sleep_seconds=20, waiting_for='ironic conductor node to reboot') def is_ironic_available(): try: ironic.client.driver.list() return True except Exception: return False common.wait(is_ironic_available, timeout_seconds=60 * 5, sleep_seconds=20, waiting_for='ironic conductor service to start') result = env.wait_network_verification() assert result.status == 'ready', 'Result data:\n{0}'.format(result.data) env.wait_for_ostf_pass(['sanity'], timeout_seconds=60 * 5) with env.get_nodes_by_role('controller')[0].ssh() as remote: ironic_cli = os_cli.Ironic(remote) for cmd in ['node-list', 'port-list', 'chassis-list', 'driver-list']: ironic_cli(cmd) if not boot_instance_before: instance = ironic.boot_instance(image=image, flavor=flavor, keypair=keypair) assert os_conn.nova.servers.get(instance.id).status == 'ACTIVE'
def create_server(self, name, image_id=None, flavor=1, userdata=None, files=None, key_name=None, timeout=300, wait_for_active=True, wait_for_avaliable=True, **kwargs): if image_id is None: image_id = self._get_cirros_image().id srv = self.nova.servers.create(name=name, image=image_id, flavor=flavor, userdata=userdata, files=files, key_name=key_name, **kwargs) if wait_for_active: wait(lambda: self.is_server_active(srv), timeout_seconds=timeout, sleep_seconds=5, waiting_for='instance {0} changes status to ACTIVE'.format( name)) # wait for ssh ready if wait_for_avaliable: if self.env is not None: wait(lambda: self.is_server_ssh_ready(srv), timeout_seconds=timeout, waiting_for='server available via ssh') logger.info('the server {0} is ready'.format(srv.name)) return self.get_instance_detail(srv.id)
def test_force_delete_inst_before_deferred_cleanup( self, set_recl_inst_interv, instances, volumes): """Force delete of instance before deferred cleanup Actions: 1. Update '/etc/nova/nova.conf' with long 'reclaim_instance_interval' and restart Nova on all nodes; 2. Create net and subnet; 3. Create and run two instances (vm1, vm2) inside same net; 4. Create a volume and attach it to an instance vm1; 5. Delete instance vm1 and check that it's in 'SOFT_DELETE' state; 6. Delete instance vm1 with 'force' option and check that it's not present. 7. Check that volume is released now and has an Available state; 8. Attach the volume to vm2 instance to ensure that the volume's reuse doesn't call any errors. """ timeout = 60 # (sec) timeout to wait instance for status change # Create two vms vm1, vm2 = instances # Create a volume and attach it to an instance vm1 volume = common_functions.create_volume( self.os_conn.cinder, image_id=None) self.os_conn.nova.volumes.create_server_volume( server_id=vm1.id, volume_id=volume.id, device='/dev/vdb') volumes.append(volume) # Delete instance vm1 and check that it's in "SOFT_DELETED" state common_functions.delete_instance(self.os_conn.nova, vm1.id) assert vm1 not in self.os_conn.get_servers() common_functions.wait( lambda: self.os_conn.server_status_is(vm1, 'SOFT_DELETED'), timeout_seconds=timeout, sleep_seconds=5, waiting_for='instance {0} changes status to SOFT_DELETED'.format( vm1.name)) # Force delete and check vm1 not present common_functions.delete_instance(self.os_conn.nova, vm1.id, force=True) common_functions.wait( lambda: self.os_conn.is_server_deleted(vm1.id), timeout_seconds=timeout, sleep_seconds=5, waiting_for='instance {0} to be forced deleted'.format(vm1.name)) # Check that volume is released now and has an Available state assert common_functions.check_volume_status( self.os_conn.cinder, volume.id, 'available', 1) # Check volume is not attached assert self.os_conn.cinder.volumes.get(volume.id).attachments == [] # Attach the volume to vm2 instance self.os_conn.nova.volumes.create_server_volume( server_id=vm2.id, volume_id=volume.id, device='/dev/vdb') # Check volume status is 'in-use' after re-attach assert common_functions.check_volume_status( self.os_conn.cinder, volume.id, 'in-use', 1) # Check that volume has correct server id volume = self.os_conn.cinder.volumes.get(volume.id) assert volume.attachments[0]['server_id'] == vm2.id
def wait_instance_to_boot(self): common_functions.wait( lambda: self.is_instance_ready(self.instance), timeout_seconds=45 * 60, sleep_seconds=60, waiting_for="windows instance to boot", )
def deploy_env(self): """Deploy env and wait till it will be deployed""" deploy_task = self.env.deploy_changes() wait(lambda: is_task_ready(deploy_task), timeout_seconds=60 * self.TIMEOUT_FOR_DEPLOY, sleep_seconds=60, waiting_for='changes to be deployed')
def delete_env(self, timeout=2): self.env.reset() wait(lambda: self.env.status == 'new', timeout_seconds=60 * timeout, sleep_seconds=20, waiting_for="Env reset finish") self.env.delete()
def cleanup(os_conn): def instances_cleanup(os_conn): instances = os_conn.nova.servers.list() for instance in instances: instance.delete() common.wait(lambda: len(os_conn.nova.servers.list()) == 0, timeout_seconds=10 * 60, waiting_for='instances cleanup') initial_images = os_conn.nova.images.list() instances_cleanup(os_conn) yield instances_cleanup(os_conn) images = [image for image in os_conn.nova.images.list() if image not in initial_images] for image in images: image.delete() common.wait(lambda: len(os_conn.nova.images.list()) == len(initial_images), timeout_seconds=10 * 60, waiting_for='images cleanup') for volume in os_conn.cinder.volumes.list(): if volume.name != 'nfv_volume': volume.delete() common.wait(lambda: volume not in os_conn.cinder.volumes.list(), timeout_seconds=10 * 60, waiting_for='volumes cleanup')
def assign_floating_ip(self, srv, use_neutron=False): if use_neutron: # Find external net id for tenant nets = self.neutron.list_networks()['networks'] err_msg = "Active external network not found in nets:{}" ext_net_ids = [ net['id'] for net in nets if net['router:external'] and net['status'] == "ACTIVE"] assert ext_net_ids, err_msg.format(nets) net_id = ext_net_ids[0] # Find instance port ports = self.neutron.list_ports(device_id=srv.id)['ports'] err_msg = "Not found active ports for instance:{}" assert ports, err_msg.format(srv.id) port = ports[0] # Create floating IP body = {'floatingip': {'floating_network_id': net_id, 'port_id': port['id']}} flip = self.neutron.create_floatingip(body) # Wait active state for port port_id = flip['floatingip']['port_id'] wait(lambda: self.neutron.show_port(port_id)['port']['status'] == "ACTIVE", timeout_seconds=60, waiting_for="floating_ip port is active") return flip['floatingip'] fl_ips_pool = self.nova.floating_ip_pools.list() if fl_ips_pool: floating_ip = self.nova.floating_ips.create( pool=fl_ips_pool[0].name) self.nova.servers.add_floating_ip(srv, floating_ip) return floating_ip
def boot_instance(self, image, flavor, keypair, **kwargs): """Boot and return ironic instance :param os_conn: initialized `os_conn` fixture :type os_conn: mos_tests.environment.os_actions.OpenStackActions :param image: image to boot instance with it :type image: warlock.core.image :param flavor: baremetal flavor :type flavor: novaclient.v2.flavors.Flavor :param keypair: SSH keypair to instance :type keypair: novaclient.v2.keypairs.Keypair :return: created instance :rtype: novaclient.v2.servers.Server """ common.wait(self.all_nodes_provisioned, timeout_seconds=3 * 60, sleep_seconds=15, waiting_for='ironic nodes to be provisioned') baremetal_net = self.os_conn.nova.networks.find(label='baremetal') return self.os_conn.create_server('ironic-server', image_id=image.id, flavor=flavor.id, key_name=keypair.name, nics=[{'net-id': baremetal_net.id}], timeout=60 * 10, **kwargs)
def delete_volumes(self, volumes): names = ', '.join([x.name for x in volumes]) for volume in volumes: volume.get() # if volume attached to any instance for attach in volume.attachments: serv_id = attach['server_id'] self.nova.volumes.delete_server_volume(serv_id, volume.id) # if volume have snapshots snapshots = self.cinder.volume_snapshots.findall( volume_id=volume.id) for snapshot in snapshots: self.cinder.volume_snapshots.delete(snapshot) wait(lambda: not any([self.cinder.volume_snapshots.findall( volume_id=x.id) for x in volumes]), timeout_seconds=60 * 5, sleep_seconds=10, waiting_for=('snapshots from volumes [{names}] ' 'to be deleted').format(names=names)) wait(lambda: all([self.cinder.volumes.get(x.id).status == 'available' for x in volumes]), timeout_seconds=60 * 5, sleep_seconds=10, waiting_for=('volumes [{names}] ' 'to became available').format(names=names)) for volume in volumes: self.cinder.volumes.delete(volume.id) # Too fast deletion requests make deletion too long time.sleep(2) self.wait_volumes_deleted(volumes)
def reset_computes(self, hostnames, env_name): def get_hypervisors(): return [x for x in self.os_conn.nova.hypervisors.list() if x.hypervisor_hostname in hostnames] node_states = defaultdict(list) def is_nodes_started(): for hypervisor in get_hypervisors(): state = hypervisor.state prev_states = node_states[hypervisor.hypervisor_hostname] if len(prev_states) == 0 or state != prev_states[-1]: prev_states.append(state) return all(x[-2:] == ['down', 'up'] for x in node_states.values()) logger.info('Resetting computes {}'.format(hostnames)) for hostname in hostnames: node = self.env.find_node_by_fqdn(hostname) devops_node = DevopsClient.get_node_by_mac(env_name=env_name, mac=node.data['mac']) devops_node.reset() wait(is_nodes_started, timeout_seconds=10 * 60)
def test_image_create_delete_from_url(glance, suffix, option): """Check image creation and deletion from URL Scenario: 1. Create image from URL 2. Wait until image has active `status` 3. Delete image 4. Check that image deleted """ name = 'Test_{}'.format(suffix) image_url = settings.GLANCE_IMAGE_URL cmd = ('image-create --name {name} --container-format bare ' '--disk-format qcow2 {option} {image_url} --progress'.format( name=name, option=option, image_url=image_url)) image = parser.details(glance(cmd)) def is_image_active(): image_data = parser.details(glance('image-show {id}'.format(**image))) return image_data['status'] == 'active' wait(is_image_active, timeout_seconds=60, waiting_for='image is active') glance('image-delete {id}'.format(**image)) check_image_not_in_list(glance, image)
def force_l3_reschedule(self, router_id, new_l3_agt_id=None, current_l3_agt_id=None): logger.info('going to reschedule the router on new agent') if current_l3_agt_id is None: l3_agents = self.neutron.list_l3_agent_hosting_routers( router_id)['agents'] if len(l3_agents) != 1: raise Exception("Can't determine l3 agent to move router from") current_l3_agt_id = l3_agents[0]['id'] if new_l3_agt_id is None: all_l3_agts = self.neutron.list_agents( binary='neutron-l3-agent')['agents'] available_l3_agts = [ agt for agt in all_l3_agts if agt['id'] != current_l3_agt_id ] new_l3_agt_id = available_l3_agts[0]['id'] self.neutron.remove_router_from_l3_agent(current_l3_agt_id, router_id) self.neutron.add_router_to_l3_agent(new_l3_agt_id, {"router_id": router_id}) wait(lambda: self.neutron.list_l3_agent_hosting_routers(router_id), timeout_seconds=5 * 60, waiting_for="router moved to new agent")
def test_image_size_attributes(instance, os_conn): """Check the OS-EXT-IMG-SIZE:size extended attribute Scenario: 1. Create net and subnet 2. Check that TestVM image has an OS-EXT-IMG-SIZE:size attribute 3. Boot instance with TestVM image on net 4. Wait for instance to reach ACTIVE status 5. Create new image as snapshot of instance 6. Check that the created snapshot has an OS-EXT-IMG-SIZE:size attribute """ attr = 'OS-EXT-IMG-SIZE:size' test_vm_image = os_conn.nova.images.find(name='TestVM') assert hasattr(test_vm_image, attr) common.wait(lambda: os_conn.is_server_active(instance), timeout_seconds=60, waiting_for='instance to became to ACTIVE status') instance = os_conn.nova.servers.get(instance.id) snapshot_id = instance.create_image('snap1') snapshot = os_conn.nova.images.get(snapshot_id) assert hasattr(snapshot, attr)
def test_ban_l3_agents_and_clear_first(self): """Ban all l3-agents, clear first of them and check health of l3-agent Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. Ban l3-agent on what router1 is 9. Wait for route rescheduling 10. Repeat steps 7-8 11. Ban l3-agent on what router1 is 12. Clear first banned L3 agent 13. Check that router moved to the health l3-agent 14. Boot one more VM (VM3) in network1 15. Boot vm3 in network1 16. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other Duration 10m """ net_id = self.os_conn.neutron.list_networks(name="net01")["networks"][0]["id"] devops_node = self.get_node_with_dhcp(net_id) ip = devops_node.data["ip"] # ban l3 agents first_banned_node = self.ban_l3_agent(router_name="router01", _ip=ip) self.ban_l3_agent(router_name="router01", _ip=ip) self.ban_l3_agent(router_name="router01", _ip=ip, wait_for_migrate=False, wait_for_die=False) # clear first banned l3 agent self.clear_l3_agent(_ip=ip, router_name="router01", node=first_banned_node, wait_for_alive=True) # wait for router migrate to cleared node router = self.os_conn.neutron.list_routers(name="router01")["routers"][0] waiting_for = "l3 agent wasn't migrate to {0}" wait( lambda: first_banned_node == self.os_conn.get_l3_agent_hosts(router["id"])[0], timeout_seconds=60 * 3, waiting_for=waiting_for.format(first_banned_node), sleep_seconds=(1, 60), ) # create another server on net01 net01 = self.os_conn.nova.networks.find(label="net01") self.os_conn.create_server( name="server03", availability_zone="{}:{}".format(self.zone.zoneName, self.hosts[0]), key_name=self.instance_keypair.name, nics=[{"net-id": net01.id}], security_groups=[self.security_group.id], ) # check pings self.check_vm_connectivity()
def test_os_instance_attributes(request, error_instance, os_conn, attrs): """Check instance extended attributes Scenario: 1. Create net and subnet 2. Boot instance on net 3. Check that attributes `attrs` are visible in instance attributes 4. Wait instance to reach ACTIVE status 5. Check that attributes `attrs` are visible in instance attributes 6. Boot instance in ERROR status 7. Check that attributes `attrs` are visible in instance attributes """ instance = request.getfuncargvalue('instance') common.wait(lambda: os_conn.server_status_is(instance, 'BUILD'), timeout_seconds=60, waiting_for='instance to became to BUILD status') instance = os_conn.nova.servers.get(instance.id) for attr in attrs: assert hasattr(instance, attr) common.wait(lambda: os_conn.is_server_active(instance), timeout_seconds=60, waiting_for='instance to became to ACTIVE status') instance = os_conn.nova.servers.get(instance.id) for attr in attrs: assert hasattr(instance, attr) error_instance = os_conn.nova.servers.get(error_instance.id) for attr in attrs: assert hasattr(error_instance, attr)
def _prepare_neutron_server_and_env(self, net_count): """Prepares neutron service network count on dhcp agent and prepares env. :param net_count: how many networks musth dhcp agent handle """ def _check_neutron_restart(): try: self.os_conn.list_networks()['networks'] except Exception as e: logger.debug(e) return False return True all_controllers = self.env.get_nodes_by_role('controller') for controller in all_controllers: with controller.ssh() as remote: self._apply_new_neutron_param_value(remote, net_count) wait( lambda: _check_neutron_restart(), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for='neutron to be up') self._prepare_openstack_state()
def wait_hypervisor_be_free(self, hypervisor): hyp_id = hypervisor.id wait(lambda: (self.nova.hypervisors.get(hyp_id).running_vms == 0), timeout_seconds=2 * 60, sleep_seconds=5, waiting_for='hypervisor {0} to be free'.format( hypervisor.hypervisor_hostname))
def _prepare_neutron_server_and_env(self, net_count): """Prepares neutron service network count on dhcp agent and prepares env. :param net_count: how many networks musth dhcp agent handle """ def _check_neutron_restart(): try: self.os_conn.list_networks()['networks'] except Exception as e: logger.debug(e) return False return True all_controllers = self.env.get_nodes_by_role('controller') for controller in all_controllers: with controller.ssh() as remote: res = self._apply_new_neutron_param_value(remote, net_count) error_msg = ( 'Neutron service restart with new value failed, ' 'exit code {exit_code},' 'stdout {stdout}, stderr {stderr}').format(**res) assert 0 == res['exit_code'], error_msg wait( lambda: _check_neutron_restart(), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for='neutron to be up') self._prepare_openstack_state()
def clear_l3_agent(self, _ip, router_name, node, wait_for_alive=False): """Clear L3 agent ban and wait until router moved to this node Clear previously banned L3 agent on node wait until router moved to this node :param _ip: ip of server to to execute clear command :param router_name: name of router to wait until it move to node :param node: name of node to clear :param wait_for_alive: """ router = self.os_conn.neutron.list_routers( name=router_name)['routers'][0] with self.env.get_ssh_to_node(_ip) as remote: remote.check_call( "pcs resource clear neutron-l3-agent {0}".format(node)) logger.info("Clear L3 agent on node {0}".format(node)) # wait for l3 agent alive if wait_for_alive: wait( lambda: self.os_conn.get_l3_for_router( router['id'])['agents'][0]['alive'] is True, timeout_seconds=60 * 3, waiting_for="L3 agent is alive", sleep_seconds=(1, 60) )
def test_destroy_master_rabbit_node(self): """Destroy one of RabbitMQ nodes . Actions: 1. Poweroff master standalone rabbitmq node; 2. Wait some time for rabbitmq cluster to recover; 3. Check RabbitMQ health with rabbitmqctl; 4. Check that old master is offline; 5. Check that new master != old master; """ timeout = 5 # minutes, wait for rabbit recover # Get master standalone rabbit node for disabling old_master = self.rabbit_node('master') old_master_fqdn = old_master.data['fqdn'] # Disable master rabbit node logger.debug("Disabling RabbitMQ master node") self.disable_node(old_master) # Wait for rabbit cluster to recover logger.debug("Sleeping for %s minutes" % timeout) sleep(60 * timeout) # Check rabbit status, wait(lambda: self.is_rabbit_cluster_ok(), timeout_seconds=60 * timeout, sleep_seconds=30, waiting_for="RabbitMQ became online") # Check that old master now offline assert self.rabbit_nodes_statuses()[old_master_fqdn] == 'offline' # Check that now we have a new master assert old_master_fqdn not in self.rabbit_nodes_roles()
def _prepare_neutron_server_and_env(self, net_count): """Prepares neutron service network count on dhcp agent and prepares env. :param net_count: how many networks musth dhcp agent handle """ def _check_neutron_restart(): try: self.os_conn.list_networks()['networks'] except Exception as e: logger.debug(e) return False return True all_controllers = self.env.get_nodes_by_role('controller') for controller in all_controllers: with controller.ssh() as remote: res = self._apply_new_neutron_param_value(remote, net_count) error_msg = ( 'Neutron service restart with new value failed, ' 'exit code {exit_code},' 'stdout {stdout}, stderr {stderr}').format(**res) assert 0 == res['exit_code'], error_msg wait_msg = "Waiting for neutron is up" wait( lambda: _check_neutron_restart(), timeout_seconds=60 * 3, sleep_seconds=(1, 60, 5), waiting_for=wait_msg) self._prepare_openstack_state()
def servers(os_conn, os_clients, networks, sec_groups): # boot instances zone = os_conn.nova.availability_zones.find(zoneName="nova") hostname = zone.hosts.keys()[0] servers = [] for i, (os_conn, network, sec_group) in enumerate(zip(os_clients, networks, sec_groups)): server1 = os_conn.create_server(name='server%02d' % (i * 2 + 1), availability_zone='{}:{}'.format( zone.zoneName, hostname), nics=[{ 'net-id': network['id'] }], security_groups=[sec_group['id']], fixed_ip='10.0.0.4', wait_for_active=False, wait_for_avaliable=False) server2 = os_conn.create_server(name='server%02d' % (i * 2 + 2), availability_zone='{}:{}'.format( zone.zoneName, hostname), nics=[{ 'net-id': network['id'] }], security_groups=[sec_group['id']], fixed_ip='10.0.0.5', wait_for_active=False, wait_for_avaliable=False) servers.extend([server1, server2]) def is_all_instances_ready(): for os_conn in os_clients: for server in os_conn.nova.servers.list(): if not os_conn.is_server_active(server): return False if not os_conn.is_server_ssh_ready(server): return False return True wait(is_all_instances_ready, timeout_seconds=3 * 60, waiting_for='all instances are ready') # update states for i, server in enumerate(servers): servers[i] = server.manager.get(server) yield servers for server in servers: server.delete() def is_instances_deleted(): for os_conn in os_clients: if not all(os_conn.is_server_deleted(x.id) for x in servers): return False return True wait(is_instances_deleted, timeout_seconds=60, waiting_for='instances deleted')
def create_volume_from_vm(self, os_conn, vm, size=1): image = os_conn.nova.servers.create_image(vm, image_name="image_vm2") common.wait(lambda: os_conn.nova.images.get(image).status == 'ACTIVE', timeout_seconds=10 * 60, waiting_for='image changes status to ACTIVE') volume = common.create_volume(os_conn.cinder, image, size=size) return volume.id
def test_delete_node(self, env, roles, ironic, make_image, flavors, keypair, os_conn, ironic_nodes): """Delete one of multiple ironic nodes. Scenario: 1. Remove created ironic node from cluster 2. Boot new ironic instance 3. Check ironic instance status is ACTIVE """ fuel_node = [x for x in env.get_all_nodes() if x.data['name'] == self.node_name][0] if 'ceph-osd' in roles: with fuel_node.ssh() as remote: remove_ceph_from_node(remote) env.unassign([fuel_node.id]) # Deploy changes task = env.deploy_changes() common.wait(lambda: common.is_task_ready(task), timeout_seconds=60 * 60, sleep_seconds=60, waiting_for='changes to be deployed') image = make_image(node_driver=ironic_nodes[0].driver) instance = ironic.boot_instance(image=image, flavor=flavors[0], keypair=keypair) assert os_conn.nova.servers.get(instance.id).status == 'ACTIVE'
def create_server(self, name, image_id=None, flavor=1, scenario='', files=None, key_name=None, timeout=300, wait_for_active=True, wait_for_avaliable=True, **kwargs): try: if scenario: with open(scenario, "r+") as f: scenario = f.read() except Exception as exc: logger.info("Error opening file: %s" % exc) raise Exception() if image_id is None: image_id = self._get_cirros_image().id srv = self.nova.servers.create(name=name, image=image_id, flavor=flavor, userdata=scenario, files=files, key_name=key_name, **kwargs) if wait_for_active: wait(lambda: self.is_server_active(srv), timeout_seconds=timeout, sleep_seconds=5, waiting_for='instance {0} changes status to ACTIVE'.format( name)) # wait for ssh ready if wait_for_avaliable: if self.env is not None: wait(lambda: self.is_server_ssh_ready(srv), timeout_seconds=timeout, waiting_for='server available via ssh') logger.info('the server {0} is ready'.format(srv.name)) return self.get_instance_detail(srv.id)
def remove_ceph_from_node(remote): def is_pg_clean(): result = remote.check_call('ceph pg stat -f json-pretty', verbose=False) pg_stat = json.loads(result.stdout_string) states = pg_stat['num_pg_by_state'] return len(states) == 1 and states[0]['name'] == 'active+clean' hostname = remote.check_call('hostname -f', verbose=False).stdout_string result = remote.check_call('ceph report', verbose=False) ceph_data = json.loads(result.stdout_string) osd_ids = [x['id'] for x in ceph_data['osd_metadata'] if x['hostname'] == hostname] for osd_id in osd_ids: remote.check_call('ceph osd out {0}'.format(osd_id), verbose=False) common.wait(is_pg_clean, timeout_seconds=5 * 60, sleep_seconds=15, waiting_for='Ceph data migration to be done') for osd_id in osd_ids: remote.check_call("stop ceph-osd id={}".format(osd_id), verbose=False) remote.check_call("ceph osd crush remove osd.{}".format(osd_id), verbose=False) remote.check_call("ceph auth del osd.{}".format(osd_id), verbose=False) remote.check_call("ceph osd rm osd.{}".format(osd_id), verbose=False) remote.check_call("ceph osd crush remove {}".format(hostname), verbose=False)
def boot_instance(self, image, flavor, keypair, **kwargs): """Boot and return ironic instance :param os_conn: initialized `os_conn` fixture :type os_conn: mos_tests.environment.os_actions.OpenStackActions :param image: image to boot instance with it :type image: warlock.core.image :param flavor: baremetal flavor :type flavor: novaclient.v2.flavors.Flavor :param keypair: SSH keypair to instance :type keypair: novaclient.v2.keypairs.Keypair :return: created instance :rtype: novaclient.v2.servers.Server """ common.wait(self.get_provisioned_node, timeout_seconds=3 * 60, sleep_seconds=15, waiting_for='ironic node to be provisioned') baremetal_net = self.os_conn.nova.networks.find(label='baremetal') return self.os_conn.create_server('ironic-server', image_id=image.id, flavor=flavor.id, key_name=keypair.name, nics=[{ 'net-id': baremetal_net.id }], timeout=60 * 10, **kwargs)
def prepare_openstack(self): """Prepare OpenStack for scenarios run Steps: 1. Create network net01, subnet net01__subnet with CIDR 10.1.1.0/24 2. Create new security group sec_group1 3. Add Ingress rule for TCP protocol to sec_group1 4. Boot vm1 net01 with sec_group1 """ # init variables security_group = self.os_conn.create_sec_group_for_ssh() self.instance_keypair = self.os_conn.create_key(key_name='instancekey') self.os_conn.nova.security_group_rules.create(security_group.id, ip_protocol='tcp', from_port=1, to_port=65535, cidr='0.0.0.0/0') net, subnet = self.create_internal_network_with_subnet( cidr="10.1.1.0/24") # create router router = self.create_router_between_nets(self.os_conn.ext_network, subnet)['router'] self.server = self.os_conn.create_server( name='server01', key_name=self.instance_keypair.name, nics=[{'net-id': net['network']['id']}], security_groups=[security_group.id]) # add floating ip to first server self.floating_ip = self.os_conn.assign_floating_ip(self.server, use_neutron=True) pkeys = self.convert_private_key_for_vm( [self.instance_keypair.private_key]) self.check_vm_is_accessible_with_ssh( vm_ip=self.floating_ip['floating_ip_address'], pkeys=pkeys, **self.cirros_creds) yield self.server.delete() wait(lambda: not self.os_conn.nova.servers.findall(id=self.server.id), timeout_seconds=2 * 60, waiting_for="instance to be deleted") self.os_conn.neutron.delete_floatingip(self.floating_ip['id']) self.os_conn.router_interface_delete(router['id'], subnet_id=subnet['subnet']['id']) self.os_conn.neutron.delete_router(router['id']) self.os_conn.neutron.delete_network(net['network']['id']) security_group.delete() self.instance_keypair.delete()
def wait_instances_to_boot(os_conn, instances): common.wait(lambda: all(os_conn.is_server_active(x) for x in instances), timeout_seconds=5 * 60, waiting_for="instances became to active state") common.wait(lambda: all(BOOT_MARKER in x.get_console_output() for x in instances), timeout_seconds=5 * 60, waiting_for="instances to be ready")
def warm_start_nodes(self, devops_nodes): for node in devops_nodes: logger.info('Starting node {}'.format(node.name)) node.create() wait(self.check_nodes_get_online_state, timeout_seconds=10 * 60) logger.info('wait until the nodes get online state') for node in self.get_all_nodes(): logger.info('online state of node {0} now is {1}'.format( node.data['name'], node.data['online']))
def create_volume_from_vm(self, os_conn, vm): image = os_conn.nova.servers.create_image(vm, image_name="image_vm2") common.wait(lambda: os_conn.nova.images.get(image).status == 'ACTIVE', timeout_seconds=10 * 60, waiting_for='image changes status to ACTIVE') volume = common.create_volume(os_conn.cinder, image, volume_type='volumes_lvm') return volume.id
def test_start_rpc_srv_client_iptables_modify(env, fixt_open_5000_port_on_nodes, fixt_kill_rpc_server_client, patch_iptables, controller): """Tests: Start RabbitMQ RPC server and client and apply IPTABLES DROP rules for RabbitMQ ports on one controller. Actions: 1. Apply IPTables Drop OR Reject rules to controller where 'oslo_msg_check_server' will be launched; 2. To be able to use port 5000 from any node open it in IPTables; 3. Install 'oslo.messaging-check-tool' on controller and compute; 4. Prepare config file for both nodes above; 5. Run 'oslo_msg_check_client' on compute node; 6. Run 'oslo_msg_check_server' on controller node (remember point 1); 7. Send GET curl request from host server to 'oslo_msg_check_client' located on compute node and check that response will '200'; 8. Remove all modifications of rules from IPTables and kill serv/client. """ exp_resp = 200 # expected response code from curl from RPC client timeout_min = 2 # (minutes) time to wait for RPC server/client start compute = random.choice(env.get_nodes_by_role('compute')) # Get management IPs of all controllers ctrl_ips = get_mngmnt_ip_of_ctrllrs(env) # Install and configure tool on controller and compute for node in (controller, compute): with node.ssh() as remote: kwargs = vars_config(remote) install_oslomessagingchecktool(remote, **kwargs) configure_oslomessagingchecktool(remote, ctrl_ips, kwargs['nova_user'], kwargs['nova_pass'], kwargs['cfg_file_path'], kwargs['sample_cfg_file_path']) # Client: Run 'oslo_msg_check_client' on compute with compute.ssh() as remote: rpc_client_ip = rabbit_rpc_client_start(remote, kwargs['cfg_file_path']) # Server: Run 'oslo_msg_check_server' on controller with controller.ssh() as remote: rabbit_rpc_server_start(remote, kwargs['cfg_file_path']) # host srv -> client: Check GET logger.debug('GET: [host server] -> [{0}]'.format(rpc_client_ip)) # need to wait for server/client start wait(lambda: get_http_code(rpc_client_ip) == exp_resp, timeout_seconds=60 * timeout_min, sleep_seconds=20, waiting_for='RPC server/client to start') assert get_http_code(rpc_client_ip) == exp_resp
def test_boot_instances_on_different_tenants(env, os_conn, ubuntu_image, ironic_nodes, ironic, flavors, tenants_clients): """Check instance statuses during instance restart Scenario: 1. Boot 1st Ironic instance under 1st tenant 2. Boot 2nd Ironic instance under 2nd tenant 3. Check Ironic instances statuses 4. Login via SSH to Ironic instances. 5. Check that instances are accessible for each other in baremetal network """ common.wait(ironic.get_provisioned_node, timeout_seconds=3 * 60, sleep_seconds=15, waiting_for='ironic node to be provisioned') instances, keypairs, ips = [], [], [] for flavor, tenant_conn in zip(flavors, tenants_clients): tenant_keypair = tenant_conn.create_key(key_name='ironic-key') brm_net = tenant_conn.nova.networks.find(label='baremetal') instance = tenant_conn.create_server('ironic-server', image_id=ubuntu_image.id, flavor=flavor.id, key_name=tenant_keypair.name, nics=[{ 'net-id': brm_net.id }], timeout=60 * 10, wait_for_avaliable=False) keypairs.append(tenant_keypair) instances.append(instance) ips.append(tenant_conn.get_nova_instance_ips(instance)['fixed']) for instance, tenant_keypair, ip in zip(instances, keypairs, ips[::-1]): with os_conn.ssh_to_instance(env, instance, vm_keypair=tenant_keypair, username='******') as remote: result = remote.execute('ping -c 10 {}'.format(ip)) received_packets = int(result['stdout'][-2].split()[3]) assert received_packets > 0 def is_instance_deleted(): return instance not in tenant_conn.nova.servers.list() for instance, tenant_conn in zip(instances, tenants_clients): instance.delete() common.wait(is_instance_deleted, timeout_seconds=60 * 5, sleep_seconds=20, waiting_for="instance is deleted")
def wait_for_rabbit_running_nodes(remote, exp_nodes, timeout_min=5): """Waits until number of 'running_nodes' from 'rabbitmqctl cluster_status' will be as expected number of controllers. :param remote: SSH connection point to controller. :param exp_nodes: Expected number of rabbit nodes. :param timeout_min: Timeout in minutes to wait. """ wait(lambda: num_of_rabbit_running_nodes(remote) == exp_nodes, timeout_seconds=60 * timeout_min, sleep_seconds=30, waiting_for='number of running nodes will be %s.' % exp_nodes)
def test_evacuate(devops_env, env, os_conn, instances, keypair): """Evacuate instances from failed compute node Scenario: 1. Create net01, net01__subnet: neutron net-create net01 neutron subnet-create net01 192.168.1.0/24 --enable-dhcp \ --name net01__subnet 2. Boot instances vm1 and vm2 in net01 on a single compute node: 3. Destroy a compute node where instances are scheduled 4. Evacuate instances vm1 and vm2: nova evacuate vm1 && nova evacuate vm2 5. Check that they are rescheduled onto another compute node and are in ACTIVE state: 6. Check that pings between vm1 and vm2 are successful """ compute_host = getattr(instances[0], 'OS-EXT-SRV-ATTR:hypervisor_hostname') compute_node = env.find_node_by_fqdn(compute_host) devops_node = devops_env.get_node_by_mac(compute_node.data['mac']) devops_node.destroy() def is_hypervisor_down(): hypervisor = os_conn.nova.hypervisors.find( hypervisor_hostname=compute_host) return hypervisor.state == 'down' common.wait( is_hypervisor_down, timeout_seconds=5 * 60, waiting_for='hypervisor {0} to be in down state'.format(compute_host)) for instance in instances: os_conn.nova.servers.evacuate(instance) def is_instances_migrate(): for instance in os_conn.nova.servers.list(): if instance not in instances: continue if instance.status == 'ERROR': raise Exception('Instance {0.name} is in ERROR status\n' '{0.fault[message]}\n' '{0.fault[details]}'.format(instance)) if not os_conn.server_status_is(instance, 'ACTIVE'): return False if getattr(instance, 'OS-EXT-SRV-ATTR:hypervisor_hostname') == compute_host: return False return True common.wait(is_instances_migrate, timeout_seconds=5 * 60, waiting_for='instances to migrate to another compute') network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
def test_kill_conductor_service(env, os_conn, ironic_nodes, ubuntu_image, flavors, keypair, env_name, ironic): """Kill ironic-conductor service with one bare-metal node Scenario: 1. Launch baremetal instance 2. Kill Ironic-conductor service for conductor node that had booted instance 3. Wait some time 4. Baremetal node must be reassigned to another Ironic-conductor 5. Run OSTF including Ironic tests. 6. Check that Ironic instance still ACTIVE and operable """ flavor, ironic_node = zip(flavors, ironic_nodes)[0] def find_conductor_node(ironic_node_uuid, conductors): cmd = 'ls /var/log/remote/ironic/{0}/'.format(ironic_node_uuid) for conductor in conductors: with conductor.ssh() as remote: result = remote.execute(cmd) if result.is_ok: return conductor instance = ironic.boot_instance(image=ubuntu_image, flavor=flavor, keypair=keypair) conductors = env.get_nodes_by_role('ironic') conductor = find_conductor_node(ironic_node.uuid, conductors) if conductor is None: raise Exception("Can't find conductor node booted istance") with conductor.ssh() as remote: remote.check_call('service ironic-conductor stop') conductors.remove(conductor) common.wait( lambda: (find_conductor_node(ironic_node.uuid, conductors) not in (conductor, None)), # yapf: disable timeout_seconds=10 * 60, waiting_for='node to migrate to another conductor', sleep_seconds=20) common.wait(lambda: env.is_ostf_tests_pass('sanity'), timeout_seconds=5 * 60, waiting_for='OSTF sanity tests to pass') assert os_conn.nova.servers.get(instance.id).status == 'ACTIVE' with os_conn.ssh_to_instance(env, instance, vm_keypair=keypair, username='******') as remote: remote.check_call('uname')
def force_dhcp_reschedule(self, net_id, new_dhcp_agt_id): logger.info('going to reschedule network to specified ' 'controller dhcp agent') current_dhcp_agt_id = self.neutron.list_dhcp_agent_hosting_networks( net_id)['agents'][0]['id'] self.neutron.remove_network_from_dhcp_agent(current_dhcp_agt_id, net_id) self.neutron.add_network_to_dhcp_agent(new_dhcp_agt_id, {'network_id': net_id}) wait(lambda: self.neutron.list_dhcp_agent_hosting_networks(net_id), timeout_seconds=5 * 60, waiting_for="network reschedule to new dhcp agent")
def test_deploy_baremetal_nodes_on_2_envs(env, ironic, ubuntu_image, flavors, keypair, env2, env2_ironic, env2_ubuntu_image, env2_flavors, env2_keypair): """Check deploy bare-metal nodes on two environments concurrently Scenario: 1. Launch baremetal instance from env1 2. Launch baremetal instance from env2 3. Verify that "Provisioning State" became "available" on baremetal nodes in both Envs. 4. Check that baremetal instances ase SSH available """ instance1 = ironic.boot_instance(image=ubuntu_image, flavor=flavors[0], keypair=keypair, wait_for_active=False, wait_for_avaliable=False) instance2 = env2_ironic.boot_instance(image=env2_ubuntu_image, flavor=env2_flavors[0], keypair=env2_keypair, wait_for_active=False, wait_for_avaliable=False) def is_instances_active(): return (env.os_conn.is_server_active(instance1) and env2.os_conn.is_server_active(instance2)) common.wait(is_instances_active, timeout_seconds=10 * 60, waiting_for='instances became active state') def is_ssh_ready(): return (env.os_conn.is_server_ssh_ready(instance1) and env2.os_conn.is_server_ssh_ready(instance2)) common.wait(is_ssh_ready, timeout_seconds=5 * 60, waiting_for='instances to be available via SSH') with env.os_conn.ssh_to_instance(env, instance1, vm_keypair=keypair, username='******') as remote: remote.check_call('uname') with env2.os_conn.ssh_to_instance(env2, instance2, vm_keypair=env2_keypair, username='******') as remote: remote.check_call('uname')
def test_shutdown_non_primary_controller_dhcp_agent(self): """Shutdown non-primary controller and check dhcp-agent Scenario: 2. Create network1, subnet1, router1 3. Launch instances vm1 in network1 4. Find non-primary controller 5. Check on what agents is network1. Reschedule agents if needed. 6. Run udhcp on vm1 7. Shutdown non-primary controller 8. Check that all networks reschedule from non-primary controller 9. Run udhcp on vm1 Duration 10m """ self._prepare_openstack() agents_hosts = self.os_conn.get_node_with_dhcp_for_network( self.networks[0]) # Check if the agent is not on the non-primary controller # Reschedule if needed if not any(self.non_primary_host in host for host in agents_hosts): self.os_conn.reschedule_dhcp_agent(self.networks[0], self.non_primary_host) # Get non-primary controller agent id agent_ids = [ agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-dhcp-agent')['agents'] if self.non_primary_host in agt['host'] ] # Run udhcp on vm self.run_udhcpc_on_vm(self.server1) # Destroy non-primary controller self.env.destroy_nodes([self.non_primary_node]) # Wait some time while agents become down self.os_conn.wait_agents_down(agent_ids) # Check that all networks reschedule from non-primary controller wait(lambda: len( self.os_conn.neutron.list_networks_on_dhcp_agent(agent_ids[0])[ 'networks']) == 0, timeout_seconds=60 * 5, sleep_seconds=(1, 60, 5), waiting_for="migrating all networks from died dhcp agent") # Run udhcp on vm self.run_udhcpc_on_vm(self.server1)
def migrate(self, os_conn, vm): os_conn.nova.servers.migrate(vm) common.wait( lambda: os_conn.nova.servers.get(vm).status == 'VERIFY_RESIZE', timeout_seconds=3 * 60, waiting_for='instance {} changes status to VERIFY_RESIZE during ' 'migration'.format(vm.name)) os_conn.nova.servers.confirm_resize(vm) common.wait(lambda: os_conn.is_server_active(vm), timeout_seconds=5 * 60, waiting_for='instance {} changes status to ACTIVE after ' 'migration'.format(vm.name)) return os_conn.nova.servers.get(vm)
def test_instance_terminate(env, ironic, os_conn, ironic_nodes, ubuntu_image, flavors, keypair, instance): """Check terminate instance Scenario: 1. Boot Ironic instance 2. Terminate Ironic instance 3. Wait and check that instance not present in nova list """ instance.delete() common.wait(lambda: not os_conn.nova.servers.list().count(instance), timeout_seconds=60, waiting_for="instance is terminated")
def test_reboot_all_ironic_conductors(env, env_name): """Check ironic state after restart all conductor nodes Scenario: 1. Ensure ironic conductor service works correctly on each conductor node (e.g., ensure "ironic driver-list" returns correct list of drivers) 2. Shutdown all ironic conductor nodes 3. Turn on all ironic conductor nodes 4. SSH to every conductor and ensure conductor service works fine. """ controller = env.get_nodes_by_role('controller')[0] with controller.ssh() as remote: with remote.open('/root/openrc') as f: openrc = f.read() conductors = env.get_nodes_by_role('ironic') drivers = None for conductor in conductors: with conductor.ssh() as remote: with remote.open('/root/openrc', 'w') as f: f.write(openrc) drivers_data = os_cli.Ironic(remote)('driver-list').listing() assert len(drivers_data) > 0 if drivers is None: drivers = drivers_data else: assert drivers == drivers_data devops_nodes = [devops_client.DevopsClient.get_node_by_mac( env_name=env_name, mac=x.data['mac']) for x in conductors] for node in devops_nodes: node.destroy() for node in devops_nodes: node.start() common.wait(lambda: all(x.is_ssh_avaliable() for x in conductors), timeout_seconds=10 * 60, waiting_for='conductor nodes to boot') for conductor in conductors: with conductor.ssh() as remote: with remote.open('/root/openrc', 'w') as f: f.write(openrc) ironic_cli = os_cli.Ironic(remote) assert ironic_cli('driver-list').listing() == drivers
def test_instance_rebuild(env, ironic, os_conn, ironic_nodes, ubuntu_image, flavors, keypair, instance): """Check rebuild instance Scenario: 1. Boot Ironic instance 2. Rebuild Ironic instance (nova rebuild <server> <image>) 3. Check that instance status became REBUILD 4. Wait until instance returns back to ACTIVE status. """ server = os_conn.rebuild_server(instance, ubuntu_image.id) common.wait(lambda: os_conn.nova.servers.get(server).status == 'ACTIVE', timeout_seconds=60 * 10, waiting_for="instance is active")
def kill_rabbitmq_on_node(remote, timeout_min=7): """Waiting for rabbit startup and got pid, then kill-9 it""" def get_pid(): cmd = "rabbitmqctl status | grep '{pid' | tr -dc '0-9'" try: return remote.check_call(cmd)['stdout'][0].strip() except Exception: return None wait(get_pid, timeout_seconds=60 * timeout_min, sleep_seconds=30, waiting_for='Rabbit get its pid on %s.' % remote.host) cmd = "kill -9 %s" % get_pid() remote.check_call(cmd)
def wait_for_deploy(self, environment): def is_murano_env_deployed(): status = self.murano.environments.get(environment.id).status if status == 'deploy failure': raise Exception('Environment deploy finished with errors') return status == 'ready' wait(is_murano_env_deployed, timeout_seconds=1200, waiting_for='environment is ready') environment = self.murano.environments.get(environment.id) logs = self.get_log(environment) assert 'Deployment finished' in logs return environment