def auth_session(self):
if not self.__auth_session:
log.debug(
"Creating auth session with args {}",
{k: v
for k, v in self.__auth.items() if k != 'password'})
plugin = Password(**self.__auth)
self.__auth_session = Session(auth=plugin,
timeout=self.request_timeout)
return self.__auth_session
class OpenStackActions(object):
"""OpenStack base services clients and helper actions"""
def __init__(self, controller_ip, keystone_version=2, domain='Default',
user='******', password='******', tenant='admin',
cert=None, env=None, proxy_session=None):
logger.debug('Init OpenStack clients on {0}'.format(controller_ip))
self.controller_ip = controller_ip
self.username = user
self.password = password
self.tenant = tenant
if cert is None:
if keystone_version == 2:
auth_url = 'http://{0}:5000/v2.0/'.format(self.controller_ip)
else:
auth_url = 'http://{0}:5000/v3/'.format(self.controller_ip)
self.path_to_cert = None
self.insecure = True
else:
if keystone_version == 2:
auth_url = 'https://{0}:5000/v2.0/'.format(self.controller_ip)
else:
auth_url = 'https://{0}:5000/v3/'.format(self.controller_ip)
with gen_temp_file(prefix="fuel_cert_", suffix=".pem") as f:
f.write(cert)
self.path_to_cert = f.name
self.insecure = False
logger.debug('Auth URL is {0}'.format(auth_url))
if keystone_version == 2:
self.auth = KeystonePassword(username=user,
password=password,
auth_url=auth_url,
tenant_name=tenant)
self.session = session.Session(auth=self.auth,
verify=self.path_to_cert)
self.keystone = KeystoneClient(session=self.session)
else:
self.auth = v3.Password(auth_url=auth_url,
user_domain_name=domain,
username=user,
password=password,
project_domain_name=domain,
project_name=tenant)
self.session = sessionV3.Session(auth=self.auth,
verify=self.path_to_cert)
self.keystone = KeystoneClientV3(session=self.session)
self.keystone.management_url = auth_url
self.nova = nova_client.Client(version=2, session=self.session)
self.cinder = cinderclient.Client(version=2, session=self.session)
if keystone_version == 2:
self.neutron = neutron_client.Client(session=self.session)
else:
self.neutron = neutron_client.Client(session=self.session,
project_name=tenant)
self.glance = GlanceClient(session=self.session)
# TODO(akuznetsova): Need to refactor initialization of heatclient
self.endpoint_url = self.session.get_endpoint(
service_type='orchestration',
endpoint_type='publicURL'
)
token = self.session.get_token()
self.heat = HeatClient(endpoint=self.endpoint_url, token=token)
self.env = env
def get_auth_token(self):
return self.auth.get_auth_ref(self.session).auth_token
def reinit_heat_client(self):
token = self.get_auth_token()
self.heat = HeatClient(endpoint=self.endpoint_url, token=token)
def _get_cirros_image(self):
for image in self.glance.images.list():
if image.name.startswith("TestVM"):
return image
def is_nova_ready(self):
"""Checks that all nova computes are available"""
def get_hosts():
zone = self.nova.availability_zones.find(zoneName="nova")
hosts = [x for y in zone.hosts.values() for x in y.values()]
for host in hosts:
host['updated_at'] = dateparse(host['updated_at'])
return hosts
last_updated = max([x['updated_at'] for x in get_hosts()])
wait(lambda: all([x['updated_at'] > last_updated
for x in get_hosts()]),
timeout_seconds=2 * 60,
sleep_seconds=10,
waiting_for='nova host data to be updated')
return all(x['available'] for x in get_hosts() if x['active'])
def get_instance_detail(self, server):
details = self.nova.servers.get(server)
return details
def get_servers(self):
servers = self.nova.servers.list()
if servers:
return servers
def get_srv_hypervisor_name(self, srv):
srv = self.nova.servers.get(srv.id)
return getattr(srv, "OS-EXT-SRV-ATTR:hypervisor_hostname")
def server_status_is(self, server, status):
server = self.nova.servers.get(server)
if server.status == 'ERROR':
raise InstanceError(server)
return server.status == status
def image_status_is(self, image, status):
image = self.glance.images.get(image.get('id'))
return image.get('status') == status
def is_server_active(self, server):
return self.server_status_is(server, 'ACTIVE')
def is_image_active(self, image):
return self.image_status_is(image, 'active')
def wait_servers_active(self, servers, timeout=10 * 60):
wait(lambda: all(self.is_server_active(x) for x in servers),
timeout_seconds=timeout,
sleep_seconds=10,
waiting_for='instances to become at ACTIVE status')
def wait_images_active(self, images, timeout=10 * 60):
wait(lambda: all(self.is_image_active(x) for x in images),
timeout_seconds=timeout,
sleep_seconds=10,
waiting_for='images to become at ACTIVE status')
def wait_servers_ssh_ready(self, servers, timeout=10 * 60):
wait(lambda: all(self.is_server_ssh_ready(x) for x in servers),
timeout_seconds=timeout,
sleep_seconds=10,
waiting_for='instances to be ssh ready')
def wait_servers_deleted(self, servers, timeout=3 * 60):
wait(lambda: all(self.is_server_deleted(x) for x in servers),
timeout_seconds=timeout,
waiting_for='instances to be deleted')
def wait_marker_in_servers_log(self, servers, marker, timeout=10 * 60):
wait(lambda: all(marker in x.get_console_output() for x in servers),
timeout_seconds=timeout,
waiting_for='marker appears in all servers log')
def create_server(self, name, image_id=None, flavor=1, userdata=None,
files=None, key_name=None, timeout=600,
wait_for_active=True, wait_for_avaliable=True, **kwargs):
if image_id is None:
image_id = self._get_cirros_image().id
srv = self.nova.servers.create(name=name,
image=image_id,
flavor=flavor,
userdata=userdata,
files=files,
key_name=key_name,
**kwargs)
if wait_for_active:
self.wait_servers_active([srv], timeout=timeout)
# wait for ssh ready
if wait_for_avaliable:
self.wait_servers_ssh_ready([srv], timeout=timeout)
return self.get_instance_detail(srv.id)
def is_server_ssh_ready(self, server):
"""Check ssh connect to server"""
ssh_client = self.ssh_to_instance(self.env,
server,
username='******',
password='******')
return bool(ssh_client.check_connection())
def is_server_deleted(self, server_id):
try:
instance = self.nova.servers.get(server_id)
if instance.status == 'ERROR':
raise InstanceError(instance)
return False
except nova_exceptions.NotFound:
return True
def get_hypervisor_capacity(self, hypervisor, flavor):
"""Return max available count of instances, which can be booted on
hypervisor with choosed flavor
:returns: possible instances count
"""
if hypervisor.vcpus < flavor.vcpus:
return 0
if flavor.disk > 0:
return min(hypervisor.disk_available_least // flavor.disk,
hypervisor.free_ram_mb // flavor.ram)
else:
return hypervisor.free_ram_mb // flavor.ram
def wait_hypervisor_be_free(self, hypervisor):
hyp_id = hypervisor.id
wait(lambda: (self.nova.hypervisors.get(hyp_id).running_vms == 0),
timeout_seconds=2 * 60,
sleep_seconds=5,
waiting_for='hypervisor {0} to be free'.format(
hypervisor.hypervisor_hostname))
def get_nova_instance_ips(self, srv):
"""Return all nova instance ip addresses as dict
Example return:
{'floating': '10.109.2.2',
'fixed': '192.168.1.2'}
:param srv: nova instance
:rtype: dict
:return: Dict with server ips
"""
srv.get()
return {x['OS-EXT-IPS:type']: x['addr']
for y in srv.addresses.values()
for x in y}
def get_node_with_dhcp_for_network(self, net_id, filter_attr='host',
is_alive=True):
filter_fn = lambda x: x[filter_attr] if filter_attr else x
result = self.list_dhcp_agents_for_network(net_id)
nodes = [filter_fn(node) for node in result['agents']
if node['alive'] == is_alive]
return nodes
def get_node_with_dhcp_for_network_by_host(self, net_id, hostname):
result = self.list_dhcp_agents_for_network(net_id)
nodes = [node for node in result['agents'] if node['host'] == hostname]
return nodes
def list_all_neutron_agents(self, agent_type=None,
filter_attr=None, is_alive=True):
agents_type_map = {
'dhcp': 'neutron-dhcp-agent',
'ovs': 'neutron-openvswitch-agent',
'metadata': 'neutron-metadata-agent',
'l3': 'neutron-l3-agent',
None: ''
}
filter_fn = lambda x: x[filter_attr] if filter_attr else x
agents = [
filter_fn(agent) for agent in self.neutron.list_agents(
binary=agents_type_map[agent_type])['agents']
if agent['alive'] == is_alive]
return agents
def list_dhcp_agents_for_network(self, net_id):
return self.neutron.list_dhcp_agent_hosting_networks(net_id)
def get_networks_on_dhcp_agent(self, agent_id):
return self.list_networks_on_dhcp_agent(agent_id)['networks']
def list_networks_on_dhcp_agent(self, agent_id):
return self.neutron.list_networks_on_dhcp_agent(agent_id)
def add_network_to_dhcp_agent(self, agent_id, network_id):
self.neutron.add_network_to_dhcp_agent(
agent_id, body={'network_id': network_id})
def remove_network_from_dhcp_agent(self, agent_id, network_id):
self.neutron.remove_network_from_dhcp_agent(agent_id, network_id)
def add_router_to_l3_agent(self, router_id, l3_agent_id):
return self.neutron.add_router_to_l3_agent(l3_agent_id,
{'router_id': router_id})
def remove_router_from_l3_agent(self, router_id, l3_agent_id):
return self.neutron.remove_router_from_l3_agent(router_id=router_id,
l3_agent=l3_agent_id)
def list_ports_for_network(self, network_id, device_owner):
return self.neutron.list_ports(
network_id=network_id, device_owner=device_owner)['ports']
def create_port(self, network_id):
return self.neutron.create_port({'port': {'network_id': network_id}})
def list_l3_agents(self):
return self.list_all_neutron_agents('l3')
def get_l3_agent_hosts(self, router_id):
agents = self.get_l3_for_router(router_id)
hosts = [i['host'] for i in agents]
return hosts
def get_l3_for_router(self, router_id, condition=lambda x: True):
result = self.neutron.list_l3_agent_hosting_routers(router_id)
return list(filter(condition, result['agents']))
def create_network(self, name, tenant_id=None, qos_policy_id=None):
network = {'name': name, 'admin_state_up': True}
if tenant_id is not None:
network['tenant_id'] = tenant_id
if qos_policy_id is not None:
network['qos_policy_id'] = qos_policy_id
return self.neutron.create_network({'network': network})
def delete_network(self, id):
return self.neutron.delete_network(id)
def create_subnet(self, network_id, name, cidr, tenant_id=None,
dns_nameservers=('8.8.8.8', '8.8.4.4')):
subnet = {
"network_id": network_id,
"ip_version": 4,
"cidr": cidr,
"name": name
}
if tenant_id is not None:
subnet['tenant_id'] = tenant_id
if dns_nameservers is not None:
subnet['dns_nameservers'] = dns_nameservers
return self.neutron.create_subnet({'subnet': subnet})
def delete_subnet(self, id):
return self.neutron.delete_subnet(id)
def delete_net_subnet_smart(self, net_id):
"""Delete network and it's subnetwork(s).
If subnet has dependencies like instances or routers - remove interface
from instance or router.
:param net_id: ID of network
"""
def subnet_in_router_ports(subnet_id):
routers_ports = self.neutron.list_ports(
device_owner='network:router_interface')['ports']
return subnet_id in str(routers_ports)
net_info = self.neutron.list_networks(id=net_id)['networks']
if len(net_info) == 0:
logger.debug('Network [{0}] not present. '
'Nothing to delete.'.format(net_id))
return
net_name = net_info[0]['name']
subnets_id = net_info[0]['subnets']
# delete all subnets from net
del_msg = 'Deleting SubNetwork [{sn}] from Network [{n}] ... '
for subnet_id in subnets_id:
subnet_name = self.neutron.show_subnet(subnet_id)['subnet']['name']
logger.debug(del_msg.format(n=net_name, sn=subnet_name) + 'start')
try:
self.neutron.delete_subnet(subnet_id)
except NeutronConflict:
# Inst/router has assigned IP/port from subnet
logger.debug("Seems that SubNetwork still in use. "
"Deleting interfaces from dependencies.")
# -- Delete IPs from subnet from instances --
# get instances with IPs from net
insts_with_net = [x for x in self.nova.servers.findall()
if net_name in getattr(x, 'networks', [])]
for inst in insts_with_net:
# get ports(ips) from instances
inst_ports = self.neutron.list_ports(
device_id=inst.id, network_id=net_id)['ports']
inst_ports_ids = [x['id'] for x in inst_ports]
# detach interface/port/ip from instance
for port_id in inst_ports_ids:
self.nova.servers.interface_detach(inst.id, port_id)
# wait till interface will be deleted from instance
wait(
lambda: net_name not in str(
self.nova.servers.find(id=inst.id).networks),
timeout_seconds=60,
waiting_for="interface deletion from instance")
# -- Delete Internal Interface from router --
routers_ports = self.neutron.list_ports(
device_owner='network:router_interface')['ports']
for router_ports in routers_ports:
# get router id that has attached port from subnet
routers_with_subnet_id = [
router_ports['device_id']
for x in router_ports['fixed_ips']
if x['subnet_id'] == subnet_id]
# delete interface from router
for router_id in routers_with_subnet_id:
self.router_interface_delete(
router_id=router_id, subnet_id=subnet_id)
# wait till no subnet_id in list of router's ports
wait(lambda: subnet_in_router_ports(subnet_id) is False,
timeout_seconds=60,
waiting_for="interface deletion from router")
# -- Delete subnet --
self.neutron.delete_subnet(subnet_id)
logger.debug(del_msg.format(n=net_name, sn=subnet_name) + 'done')
# Delete Network
logger.debug('Deleting Network [{n}]'.format(n=net_name))
self.neutron.delete_network(net_id)
def list_subnets(self):
return self.neutron.list_subnets()
def list_networks(self):
return self.neutron.list_networks()
def list_subnetpools(self):
return self.neutron.list_subnepools()
def list_address_scopes(self):
return self.neutron.list_address_scopes()
def assign_floating_ip(self, srv, use_neutron=False):
if use_neutron:
# Find external net id for tenant
nets = self.neutron.list_networks()['networks']
err_msg = "Active external network not found in nets:{}"
ext_net_ids = [
net['id'] for net in nets
if net['router:external'] and net['status'] == "ACTIVE"]
assert ext_net_ids, err_msg.format(nets)
net_id = ext_net_ids[0]
# Find instance port
ports = self.neutron.list_ports(device_id=srv.id)['ports']
err_msg = "Not found active ports for instance:{}"
assert ports, err_msg.format(srv.id)
port = ports[0]
# Create floating IP
body = {'floatingip': {'floating_network_id': net_id,
'port_id': port['id']}}
flip = self.neutron.create_floatingip(body)
# Wait active state for port
port_id = flip['floatingip']['port_id']
wait(lambda:
self.neutron.show_port(port_id)['port']['status'] == "ACTIVE",
timeout_seconds=60,
waiting_for="floating_ip port is active")
return flip['floatingip']
fl_ips_pool = self.nova.floating_ip_pools.list()
if fl_ips_pool:
floating_ip = self.nova.floating_ips.create(
pool=fl_ips_pool[0].name)
self.nova.servers.add_floating_ip(srv, floating_ip)
return floating_ip
def disassociate_floating_ip(self, srv, floating_ip, use_neutron=False):
def is_floating_ip_down():
fl_ip = self.neutron.show_floatingip(identifier)
return fl_ip['floatingip']['status'] == 'DOWN'
if use_neutron:
try:
self.neutron.update_floatingip(
floatingip=floating_ip['id'],
body={'floatingip': {}})
identifier = floating_ip['id']
wait(is_floating_ip_down, timeout_seconds=60)
except NeutronClientException:
logger.info('The floatingip {} can not be disassociated.'
.format(floating_ip['id']))
else:
try:
self.nova.servers.remove_floating_ip(srv, floating_ip)
except nova_exceptions.ClientException:
logger.info('The floatingip {} can not be disassociated.'
.format(floating_ip))
def delete_floating_ip(self, floating_ip, use_neutron=False):
if use_neutron:
try:
self.neutron.delete_floatingip(floating_ip['id'])
except NeutronClientException:
logger.info('floating_ip {} is not deletable'
.format(floating_ip['id']))
else:
try:
self.nova.floating_ips.delete(floating_ip)
except nova_exceptions.ClientException:
logger.info('floating_ip {} is not deletable'
.format(floating_ip))
else:
wait(
lambda: len(
self.nova.floating_ips.findall(
ip=floating_ip.ip)) == 0,
timeout_seconds=1 * 60,
waiting_for='floating {0} removal'.format(floating_ip.ip))
def create_router(self, name, tenant_id=None, distributed=False):
router = {'name': name, 'distributed': distributed}
if tenant_id is not None:
router['tenant_id'] = tenant_id
return self.neutron.create_router({'router': router})
def router_interface_add(self, router_id, subnet_id=None, port_id=None):
body = {}
if subnet_id:
body['subnet_id'] = subnet_id
elif port_id:
body['port_id'] = port_id
else:
raise ValueError("subnet_id or port_id must be indicated.")
self.neutron.add_interface_router(router_id, body)
def router_interface_delete(self, router_id, subnet_id=None, port_id=None):
body = {}
if subnet_id:
body['subnet_id'] = subnet_id
elif port_id:
body['port_id'] = port_id
else:
raise ValueError("subnet_id or port_id must be indicated.")
self.neutron.remove_interface_router(router_id, body)
def router_gateway_add(self, router_id, network_id):
network = {
'network_id': network_id
}
self.neutron.add_gateway_router(router_id, network)
def delete_router(self, router_id):
binded_ports = self.neutron.list_ports(
device_id=router_id, device_owner=u'network:router_interface'
)['ports']
for port in binded_ports:
self.router_interface_delete(router_id, port_id=port['id'])
self.neutron.delete_router(router_id)
def create_qos_policy(self, name):
return self.neutron.create_qos_policy({'policy': {'name': name}})
def delete_qos_policy(self, policy_id):
self.neutron.delete_qos_policy(policy_id)
def create_sec_group_for_ssh(self):
name = "test-sg" + str(random.randint(1, 0x7fffffff))
secgroup = self.nova.security_groups.create(
name, "descr")
rulesets = [
{
# ssh
'ip_protocol': 'tcp',
'from_port': 22,
'to_port': 22,
'cidr': '0.0.0.0/0',
},
{
# ping
'ip_protocol': 'icmp',
'from_port': -1,
'to_port': -1,
'cidr': '0.0.0.0/0',
}
]
for ruleset in rulesets:
self.nova.security_group_rules.create(
secgroup.id, **ruleset)
return secgroup
def delete_security_group(self, sg):
"""Delete security group.
If security group used by instances -> delete it from instances first.
:param sg: security group to delete
:type sg: novaclient.v2.security_groups.SecurityGroup
"""
logger.debug("Deleting security group '{sg.name}'".format(sg=sg))
# if sec group in use -> remove it from instance
sg_search = {'name': sg.name}
srvs_with_sg = [x
for x in self.nova.servers.findall()
if sg_search in getattr(x, 'security_groups', [])]
for srv in srvs_with_sg:
# remove sec group from instance
logger.debug(('Removing sec group "{sg.name}" from instance '
'[{srv.name}:{srv.id}]').format(sg=sg,
srv=srv))
self.nova.servers.remove_security_group(srv.id, sg.id)
# delete security group
self.nova.security_groups.delete(sg)
logger.debug("Deleting security group "
"'{sg.name}' ... done".format(sg=sg))
def create_key(self, key_name):
return self.nova.keypairs.create(key_name)
def delete_key(self, key_name):
return self.nova.keypairs.delete(key_name)
def get_port_by_fixed_ip(self, ip):
"""Returns neutron port by instance fixed ip"""
for port in self.neutron.list_ports()['ports']:
for ips in port['fixed_ips']:
if ip == ips['ip_address']:
return port
@property
def ext_network(self):
ext_networks = self.neutron.list_networks(
**{'router:external': True, 'status': 'ACTIVE'})
return ext_networks['networks'][0]
@property
def int_networks(self):
return self.neutron.list_networks(**{'router:external': False,
'status': 'ACTIVE'})['networks']
def delete_subnets(self, networks):
# Subnets and ports are simply filtered by network ids
for subnet in self.neutron.list_subnets()['subnets']:
if subnet['network_id'] not in networks:
continue
try:
self.neutron.delete_subnet(subnet['id'])
except NeutronClientException:
logger.info(
'the subnet {} is not deletable'.format(subnet['id']))
def delete_routers(self):
# Did not find the better way to detect the fuel admin router
# Looks like it just always has fixed name router04
for router in self.neutron.list_routers()['routers']:
if router['name'] == 'router04':
continue
try:
self.neutron.delete_router(router['id'])
except NeutronClientException:
logger.info('the router {} is not deletable'.format(router))
def delete_floating_ips(self):
for floating_ip in self.nova.floating_ips.list():
try:
self.nova.floating_ips.delete(floating_ip)
except nova_exceptions.ClientException:
self.delete_floating_ip(floating_ip, use_neutron=True)
def delete_servers(self):
for server in self.nova.servers.list():
try:
self.nova.servers.delete(server)
except nova_exceptions.ClientException:
logger.info('nova server {} is not deletable'.format(server))
def delete_keypairs(self):
for key_pair in self.nova.keypairs.list():
try:
self.nova.keypairs.delete(key_pair)
except nova_exceptions.ClientException:
logger.info('key pair {} is not deletable'.format(key_pair.id))
def delete_security_groups(self):
for sg in self.nova.security_groups.list():
if sg.description == 'Default security group':
continue
try:
self.nova.security_groups.delete(sg)
except nova_exceptions.ClientException:
logger.info(
'The Security Group {} is not deletable'.format(sg))
def delete_ports(self, networks):
# After some experiments the following sequence for deletion was found
# router_interface and ports -> subnets -> routers -> nets
# Delete router interface and ports
# TBD some ports are still kept after the cleanup.
# Need to find why and delete them as well
# But it does not fail the execution so far.
for port in self.neutron.list_ports()['ports']:
if port['network_id'] not in networks:
continue
try:
# TBD Looks like the port might be used either by router or
# l3 agent
# in case of router this condition is true
# port['network'] == 'router_interface'
# dunno what will happen in case of the l3 agent
for fixed_ip in port['fixed_ips']:
self.neutron.remove_interface_router(
port['device_id'],
{
'router_id': port['device_id'],
'subnet_id': fixed_ip['subnet_id'],
}
)
except NeutronClientException:
logger.info('the port {} is not deletable'
.format(port['id']))
def remove_port(self, port_id):
"""Remove port by port_id.
:param port_id: id of port that should be deleted.
"""
port = self.neutron.show_port(port_id)['port']
try:
for fixed_ip in port['fixed_ips']:
self.neutron.remove_interface_router(
port['device_id'],
{
'router_id': port['device_id'],
'subnet_id': fixed_ip['subnet_id'],
}
)
except NeutronClientException:
logger.info('port {} could not be deleted'.format(port['id']))
def cleanup_network(self, networks_to_skip=tuple()):
"""Clean up the neutron networks.
:param networks_to_skip: list of networks names that should be kept
"""
# net ids with the names from networks_to_skip are filtered out
networks = [x['id'] for x in self.neutron.list_networks()['networks']
if x['name'] not in networks_to_skip]
self.delete_keypairs()
self.delete_floating_ips()
self.delete_servers()
self.delete_security_groups()
self.delete_ports(networks)
self.delete_subnets(networks)
self.delete_routers()
# Delete nets
for net in networks:
try:
self.neutron.delete_network(net)
except NeutronClientException:
logger.info('the net {} is not deletable'
.format(net))
def execute_through_host(self, ssh, vm_host, cmd, creds=()):
logger.debug("Making intermediate transport")
intermediate_transport = ssh._ssh.get_transport()
logger.debug("Opening channel to VM")
intermediate_channel = intermediate_transport.open_channel(
'direct-tcpip', (vm_host, 22), (ssh.host, 0))
logger.debug("Opening paramiko transport")
transport = paramiko.Transport(intermediate_channel)
logger.debug("Starting client")
transport.start_client()
logger.info("Passing authentication to VM: {}".format(creds))
if not creds:
creds = ('cirros', 'cubswin:)')
transport.auth_password(creds[0], creds[1])
logger.debug("Opening session")
channel = transport.open_session()
logger.info("Executing command: {}".format(cmd))
channel.exec_command(cmd)
result = {
'stdout': [],
'stderr': [],
'exit_code': 0
}
logger.debug("Receiving exit_code")
result['exit_code'] = channel.recv_exit_status()
logger.debug("Receiving stdout")
result['stdout'] = channel.recv(1024)
logger.debug("Receiving stderr")
result['stderr'] = channel.recv_stderr(1024)
logger.debug("Closing channel")
channel.close()
return result
def ssh_to_instance(self,
env,
vm,
vm_keypair=None,
username='******',
password=None,
proxy_node=None,
vm_ip=None):
"""Returns direct ssh client to instance via proxy"""
# Update vm data
vm.get()
instance_ips = {ip['addr']: {'type': ip['OS-EXT-IPS:type'],
'mac': ip['OS-EXT-IPS-MAC:mac_addr'],
'net': n}
for n, ips in vm.addresses.items() for ip in ips}
if vm_ip is not None:
ip_type = instance_ips[vm_ip]['type']
else:
ip_type = 'fixed'
vm_ip = next(k for k, v in instance_ips.items()
if v['type'] == 'fixed')
logger.debug('Try to connect to vm {name} '
'with {ip} ({ip_type})'.format(name=vm.name,
ip=vm_ip,
ip_type=ip_type))
proxies = []
# retrieve proxy nodes
if ip_type == 'fixed':
vm_mac = instance_ips[vm_ip]['mac']
net_id = self.neutron.list_ports(
mac_address=vm_mac)['ports'][0]['network_id']
dhcp_namespace = "qdhcp-{0}".format(net_id)
if proxy_node is None:
proxy_nodes = wait(
lambda: self.get_node_with_dhcp_for_network(net_id),
expected_exceptions=NeutronClientException,
timeout_seconds=60 * 3,
sleep_seconds=10,
waiting_for="any alive DHCP agent for instance network",
log=False)
else:
proxy_nodes = [proxy_node]
for node in proxy_nodes:
for pkey in env.admin_ssh_keys:
ip = env.find_node_by_fqdn(node).data['ip']
proxy = NetNsProxy(ip=ip, pkey=pkey, ns=dhcp_namespace,
proxy_to_ip=vm_ip)
proxies.append(proxy)
instance_keys = []
if vm_keypair is not None:
instance_keys.append(paramiko.RSAKey.from_private_key(six.StringIO(
vm_keypair.private_key)))
return SSHClient(vm_ip,
port=22,
username=username,
password=password,
private_keys=instance_keys,
proxies=proxies)
def run_on_vm(self,
env,
vm,
vm_keypair=None,
command='uname',
vm_login="******",
timeout=3 * 60,
vm_password='******',
vm_ip=None):
"""Execute command on vm and return CommandResult instance
:param vm: server to execute command on
:param vm_keypair: keypair used during vm creating
:param command: command (or commands list) to execute
:param vm_login: username to login to vm via ssh
:param vm_password: password to login to vm via ssh
:param timeout: type - int or None
- if None - execute command and return results
- if int - wait `timeout` seconds until command exit_code will be 0
:rtype: mos_tests.environment.ssh.CommandResult
:returns: result of command executing
"""
results = []
def execute(expected_exceptions=None):
if not self.server_status_is(vm, 'ACTIVE'):
return False
expected_exceptions = expected_exceptions or ()
with suppress(*expected_exceptions):
with self.ssh_to_instance(env,
vm,
vm_keypair,
username=vm_login,
password=vm_password,
vm_ip=vm_ip) as remote:
result = remote.execute(command)
results.append(result)
return result.is_ok
if timeout is None:
logger.info('Executing `{cmd}` on {vm}'.format(cmds=command,
vm=vm.name))
execute()
else:
wait(lambda: execute(expected_exceptions=(Exception, )),
sleep_seconds=(1, 20, 2),
timeout_seconds=timeout,
waiting_for=("SSH commands: `{cmd}` completed "
"with 0 exit code").format(cmd=command))
return results[-1]
def wait_agents_alive(self, agt_ids_to_check):
wait(lambda: all(agt['alive'] for agt in
self.neutron.list_agents()['agents']
if agt['id'] in agt_ids_to_check),
timeout_seconds=5 * 60,
waiting_for='agents is alive')
def wait_agents_down(self, agt_ids_to_check):
wait(lambda: all(not agt['alive'] for agt in
self.neutron.list_agents()['agents']
if agt['id'] in agt_ids_to_check),
timeout_seconds=5 * 60,
waiting_for='agents go down')
def add_net(self, router_id):
i = len(self.neutron.list_networks()['networks']) + 1
network = self.create_network(name='net%02d' % i)['network']
logger.info('network {name}({id}) is created'.format(**network))
subnet = self.create_subnet(
network_id=network['id'],
name='net%02d__subnet' % i,
cidr="192.168.%d.0/24" % i)
logger.info('subnet {name}({id}) is created'.format(
**subnet['subnet']))
self.router_interface_add(
router_id=router_id,
subnet_id=subnet['subnet']['id'])
return network['id']
def add_server(self, network_id, key_name, hostname, sg_id):
i = len(self.nova.servers.list()) + 1
zone = self.nova.availability_zones.find(zoneName="nova")
srv = self.create_server(
name='server%02d' % i,
availability_zone='{}:{}'.format(zone.zoneName, hostname),
key_name=key_name,
nics=[{'net-id': network_id}],
security_groups=[sg_id])
return srv
def reschedule_router_to_primary_host(self, router_id, primary_host):
agent_list = self.neutron.list_agents(
binary='neutron-l3-agent')['agents']
agt_id_to_move_on = [agt['id'] for agt in agent_list
if agt['host'] == primary_host][0]
self.force_l3_reschedule(router_id, agt_id_to_move_on)
def force_l3_reschedule(self, router_id, new_l3_agt_id=None,
current_l3_agt_id=None):
logger.info('going to reschedule the router on new agent')
if current_l3_agt_id is None:
l3_agents = self.neutron.list_l3_agent_hosting_routers(
router_id)['agents']
if len(l3_agents) != 1:
raise Exception("Can't determine l3 agent to move router from")
current_l3_agt_id = l3_agents[0]['id']
if new_l3_agt_id is None:
all_l3_agts = self.neutron.list_agents(
binary='neutron-l3-agent')['agents']
available_l3_agts = [agt for agt in all_l3_agts
if agt['id'] != current_l3_agt_id]
new_l3_agt_id = available_l3_agts[0]['id']
self.neutron.remove_router_from_l3_agent(current_l3_agt_id,
router_id)
self.neutron.add_router_to_l3_agent(new_l3_agt_id,
{"router_id": router_id})
wait(lambda: self.neutron.list_l3_agent_hosting_routers(router_id),
timeout_seconds=5 * 60, waiting_for="router moved to new agent")
def reschedule_dhcp_agent(self, net_id, controller_fqdn):
agent_list = self.neutron.list_agents(
binary='neutron-dhcp-agent')['agents']
agt_id_to_move_on = [agt['id'] for agt in agent_list
if agt['host'] == controller_fqdn][0]
self.force_dhcp_reschedule(net_id, agt_id_to_move_on)
def force_dhcp_reschedule(self, net_id, new_dhcp_agt_id):
logger.info('going to reschedule network to specified '
'controller dhcp agent')
current_dhcp_agt_id = self.neutron.list_dhcp_agent_hosting_networks(
net_id)['agents'][0]['id']
self.neutron.remove_network_from_dhcp_agent(current_dhcp_agt_id,
net_id)
self.neutron.add_network_to_dhcp_agent(new_dhcp_agt_id,
{'network_id': net_id})
wait(lambda: self.neutron.list_dhcp_agent_hosting_networks(net_id),
timeout_seconds=5 * 60,
waiting_for="network reschedule to new dhcp agent")
def _get_controller(self):
# TODO(gdyuldin) remove this methods after moving to functions.os_cli
return self.env.get_nodes_by_role('controller')[0]
def tenant_create(self, name):
# TODO(gdyuldin) remove this methods after moving to functions.os_cli
with self._get_controller().ssh() as remote:
return os_cli.OpenStack(remote).tenant_create(name=name)
def tenant_delete(self, name):
# TODO(gdyuldin) remove this methods after moving to functions.os_cli
with self._get_controller().ssh() as remote:
return os_cli.OpenStack(remote).tenant_delete(name=name)
def user_create(self, name, password, tenant=None):
# TODO(gdyuldin) remove this methods after moving to functions.os_cli
with self._get_controller().ssh() as remote:
return os_cli.OpenStack(remote).user_create(name=name,
password=password,
tenant=tenant)
def user_delete(self, name):
# TODO(gdyuldin) remove this methods after moving to functions.os_cli
with self._get_controller().ssh() as remote:
return os_cli.OpenStack(remote).user_delete(name=name)
def server_hard_reboot(self, server):
try:
self.nova.servers.reboot(server.id, reboot_type='HARD')
except nova_exceptions.ClientException:
logger.info("nova server {} can't be rebooted".format(server))
def server_start(self, server):
try:
self.nova.servers.start(server.id)
except nova_exceptions.ClientException:
logger.info("nova server {} can't be started".format(server))
def server_stop(self, server):
try:
self.nova.servers.stop(server.id)
except nova_exceptions.ClientException:
logger.info("nova server {} can't be stopped".format(server))
def rebuild_server(self, server, image):
srv = server.rebuild(image)
wait(lambda: self.nova.servers.get(srv).status == 'REBUILD',
timeout_seconds=60, waiting_for='start of instance rebuild')
return srv
def delete_volume(self, volume):
"""Delete volume and check that it is absent in the list
:param volume: volume object
"""
self.delete_volumes([volume])
def delete_volumes(self, volumes):
ids = ', '.join([x.id for x in volumes])
# Exclude non-existing volumes
all_volumes = self.cinder.volumes.findall()
provided_volumes_ids = [x.id for x in volumes]
volumes = [x for x in all_volumes
if x.id in provided_volumes_ids]
del all_volumes, provided_volumes_ids
# Detach volume from instances; delete snapshots and backups from VOL.
for volume in volumes:
# if volume attached to any instance
for attach in volume.attachments:
serv_id = attach['server_id']
self.nova.volumes.delete_server_volume(serv_id, volume.id)
# if volume have snapshots
snapshots = self.cinder.volume_snapshots.findall(
volume_id=volume.id)
for snapshot in snapshots:
self.cinder.volume_snapshots.delete(snapshot)
# if volume have backups
backups = self.cinder.backups.findall(volume_id=volume.id)
for backup in backups:
self.cinder.backups.delete(backup)
# Wait till volume will be detached and all connections will be removed
wait(lambda: not any([self.cinder.backups.findall(volume_id=x.id)
for x in volumes]),
timeout_seconds=60 * 10,
waiting_for=('backups from volumes [{ids}] '
'to be deleted').format(ids=ids))
wait(lambda: not any([self.cinder.volume_snapshots.findall(
volume_id=x.id) for x in volumes]),
timeout_seconds=60 * 5,
sleep_seconds=10,
waiting_for=('snapshots from volumes [{ids}] '
'to be deleted').format(ids=ids))
wait(lambda: all([self.cinder.volumes.get(x.id).status == 'available'
for x in volumes]),
timeout_seconds=60 * 5,
sleep_seconds=10,
waiting_for=('volumes [{ids}] '
'to became available').format(ids=ids))
# Delete volumes
for volume in volumes:
self.cinder.volumes.delete(volume.id)
# Too fast deletion requests make deletion too long
time.sleep(2)
self.wait_volumes_deleted(volumes)
def wait_volumes_deleted(self, volumes):
ids = ', '.join([x.id for x in volumes])
wait(
lambda: not any([self.cinder.volumes.findall(id=x.id)
for x in volumes]),
timeout_seconds=60 * 2,
sleep_seconds=10,
waiting_for='volumes [{ids}] to be deleted'.format(ids=ids))
def is_server_cloud_init_finished(self, vm):
finish_mark = 'Cloud-init .* finished'
vm.get()
if re.findall(finish_mark, vm.get_console_output()):
return True
else:
return False
def wait_servers_cloud_init_finished(self, vms, timeout=5 * 60):
"""Wait till vm will be booted and ready"""
wait(lambda: all(self.is_server_cloud_init_finished(x) for x in vms),
timeout_seconds=timeout,
waiting_for='cloud init finish')
def __init__(self, controller_ip, keystone_version=2, domain='Default',
user='******', password='******', tenant='admin',
cert=None, env=None, proxy_session=None):
logger.debug('Init OpenStack clients on {0}'.format(controller_ip))
self.controller_ip = controller_ip
self.username = user
self.password = password
self.tenant = tenant
if cert is None:
if keystone_version == 2:
auth_url = 'http://{0}:5000/v2.0/'.format(self.controller_ip)
else:
auth_url = 'http://{0}:5000/v3/'.format(self.controller_ip)
self.path_to_cert = None
self.insecure = True
else:
if keystone_version == 2:
auth_url = 'https://{0}:5000/v2.0/'.format(self.controller_ip)
else:
auth_url = 'https://{0}:5000/v3/'.format(self.controller_ip)
with gen_temp_file(prefix="fuel_cert_", suffix=".pem") as f:
f.write(cert)
self.path_to_cert = f.name
self.insecure = False
logger.debug('Auth URL is {0}'.format(auth_url))
if keystone_version == 2:
self.auth = KeystonePassword(username=user,
password=password,
auth_url=auth_url,
tenant_name=tenant)
self.session = session.Session(auth=self.auth,
verify=self.path_to_cert)
self.keystone = KeystoneClient(session=self.session)
else:
self.auth = v3.Password(auth_url=auth_url,
user_domain_name=domain,
username=user,
password=password,
project_domain_name=domain,
project_name=tenant)
self.session = sessionV3.Session(auth=self.auth,
verify=self.path_to_cert)
self.keystone = KeystoneClientV3(session=self.session)
self.keystone.management_url = auth_url
self.nova = nova_client.Client(version=2, session=self.session)
self.cinder = cinderclient.Client(version=2, session=self.session)
if keystone_version == 2:
self.neutron = neutron_client.Client(session=self.session)
else:
self.neutron = neutron_client.Client(session=self.session,
project_name=tenant)
self.glance = GlanceClient(session=self.session)
# TODO(akuznetsova): Need to refactor initialization of heatclient
self.endpoint_url = self.session.get_endpoint(
service_type='orchestration',
endpoint_type='publicURL'
)
token = self.session.get_token()
self.heat = HeatClient(endpoint=self.endpoint_url, token=token)
self.env = env
def init_by_creds(cls, tenant_id, os_username, os_password, auth_url) -> 'OpenstackClientBase':
plugin = Password(auth_url=auth_url, username=os_username, password=os_password, tenant_id=tenant_id)
return cls.init_by_plugin(plugin)
