def discover_services(client, opts): user, password, key = parse_creds(opts.pop('auth')) services = [] if opts['service'] == "all": services = client.services.list() else: if isinstance(opts['service'], basestring): opts['service'] = [opts['service']] for s in opts['service']: services.extend(client.services.list(binary=s)) host_services_mapping = {} for service in services: ip = socket.gethostbyname(service.host) host_services_mapping[ip].append(service.binary) logger.debug("Found %s openstack service nodes" % len(host_services_mapping)) return [ Node(host, services, username=user, password=password, key_path=key) for host, services in host_services_mapping.items() ]
def discover_vms(client, search_opts): user, password, key = parse_creds(search_opts.pop('auth')) servers = client.servers.list(search_opts=search_opts) logger.debug("Found %s openstack vms" % len(servers)) return [ Node(get_floating_ip(server), ["test_vm"], username=user, password=password, key_path=key) for server in servers if get_floating_ip(server) ]
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True): nodes_to_run = [] clean_data = None ctx.fuel_openstack_creds = None for cluster in discover: if cluster == "openstack" and not discover_nodes: logger.warning("Skip openstack cluster discovery") elif cluster == "openstack" and discover_nodes: cluster_info = clusters_info["openstack"] conn = cluster_info['connection'] user, passwd, tenant = parse_creds(conn['creds']) auth_data = dict( auth_url=conn['auth_url'], username=user, api_key=passwd, project_id=tenant) if not conn: logger.error("No connection provided for %s. Skipping" % cluster) continue logger.debug("Discovering openstack nodes " "with connection details: %r" % conn) os_nodes = openstack.discover_openstack_nodes(auth_data, cluster_info) nodes_to_run.extend(os_nodes) elif cluster == "fuel" or cluster == "fuel_openrc_only": if cluster == "fuel_openrc_only": discover_nodes = False res = fuel.discover_fuel_nodes(clusters_info['fuel'], var_dir, discover_nodes) nodes, clean_data, openrc_dict, version = res if openrc_dict is None: ctx.fuel_openstack_creds = None else: if version >= [8, 0] and openrc_dict['os_auth_url'].startswith("https://"): logger.warning("Fixing FUEL 8.0 AUTH url - replace https://->http://") openrc_dict['os_auth_url'] = "http" + openrc_dict['os_auth_url'][5:] ctx.fuel_openstack_creds = { 'name': openrc_dict['username'], 'passwd': openrc_dict['password'], 'tenant': openrc_dict['tenant_name'], 'auth_url': openrc_dict['os_auth_url'], 'insecure': openrc_dict['insecure']} env_name = clusters_info['fuel']['openstack_env'] env_f_name = env_name for char in "-+ {}()[]": env_f_name = env_f_name.replace(char, '_') fuel_openrc_fname = os.path.join(var_dir, env_f_name + "_openrc") if ctx.fuel_openstack_creds is not None: with open(fuel_openrc_fname, "w") as fd: fd.write(openrc_templ.format(**ctx.fuel_openstack_creds)) msg = "Openrc for cluster {0} saves into {1}" logger.info(msg.format(env_name, fuel_openrc_fname)) nodes_to_run.extend(nodes) elif cluster == "ceph": if discover_nodes: cluster_info = clusters_info["ceph"] nodes_to_run.extend(ceph.discover_ceph_nodes(cluster_info)) else: logger.warning("Skip ceph cluster discovery") else: msg_templ = "Unknown cluster type in 'discover' parameter: {0!r}" raise ValueError(msg_templ.format(cluster)) return nodes_to_run
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True): nodes_to_run = [] clean_data = None ctx.fuel_openstack_creds = None for cluster in discover: if cluster == "openstack" and not discover_nodes: logger.warning("Skip openstack cluster discovery") elif cluster == "openstack" and discover_nodes: cluster_info = clusters_info["openstack"] conn = cluster_info["connection"] user, passwd, tenant = parse_creds(conn["creds"]) auth_data = dict(auth_url=conn["auth_url"], username=user, api_key=passwd, project_id=tenant) if not conn: logger.error("No connection provided for %s. Skipping" % cluster) continue logger.debug("Discovering openstack nodes " "with connection details: %r" % conn) os_nodes = openstack.discover_openstack_nodes(auth_data, cluster_info) nodes_to_run.extend(os_nodes) elif cluster == "fuel" or cluster == "fuel_openrc_only": if cluster == "fuel_openrc_only": discover_nodes = False res = fuel.discover_fuel_nodes(clusters_info["fuel"], var_dir, discover_nodes) nodes, clean_data, openrc_dict = res if openrc_dict is None: ctx.fuel_openstack_creds = None else: ctx.fuel_openstack_creds = { "name": openrc_dict["username"], "passwd": openrc_dict["password"], "tenant": openrc_dict["tenant_name"], "auth_url": openrc_dict["os_auth_url"], "insecure": openrc_dict.get("insecure", False), } env_name = clusters_info["fuel"]["openstack_env"] env_f_name = env_name for char in "-+ {}()[]": env_f_name = env_f_name.replace(char, "_") fuel_openrc_fname = os.path.join(var_dir, env_f_name + "_openrc") if ctx.fuel_openstack_creds is not None: with open(fuel_openrc_fname, "w") as fd: fd.write(openrc_templ.format(**ctx.fuel_openstack_creds)) msg = "Openrc for cluster {0} saves into {1}" logger.info(msg.format(env_name, fuel_openrc_fname)) nodes_to_run.extend(nodes) elif cluster == "ceph": if discover_nodes: cluster_info = clusters_info["ceph"] nodes_to_run.extend(ceph.discover_ceph_nodes(cluster_info)) else: logger.warning("Skip ceph cluster discovery") else: msg_templ = "Unknown cluster type in 'discover' parameter: {0!r}" raise ValueError(msg_templ.format(cluster)) return nodes_to_run
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True): nodes_to_run = [] clean_data = None ctx.fuel_openstack_creds = None for cluster in discover: if cluster == "openstack" and not discover_nodes: logger.warning("Skip openstack cluster discovery") elif cluster == "openstack" and discover_nodes: cluster_info = clusters_info["openstack"] conn = cluster_info['connection'] user, passwd, tenant = parse_creds(conn['creds']) auth_data = dict(auth_url=conn['auth_url'], username=user, api_key=passwd, project_id=tenant) if not conn: logger.error("No connection provided for %s. Skipping" % cluster) continue logger.debug("Discovering openstack nodes " "with connection details: %r" % conn) os_nodes = openstack.discover_openstack_nodes( auth_data, cluster_info) nodes_to_run.extend(os_nodes) elif cluster == "fuel" or cluster == "fuel_openrc_only": if cluster == "fuel_openrc_only": discover_nodes = False res = fuel.discover_fuel_nodes(clusters_info['fuel'], var_dir, discover_nodes) nodes, clean_data, openrc_dict, version = res if openrc_dict is None: ctx.fuel_openstack_creds = None else: if version >= [8, 0] and openrc_dict['os_auth_url'].startswith( "https://"): logger.warning( "Fixing FUEL 8.0 AUTH url - replace https://->http://") openrc_dict['os_auth_url'] = "http" + openrc_dict[ 'os_auth_url'][5:] ctx.fuel_openstack_creds = { 'name': openrc_dict['username'], 'passwd': openrc_dict['password'], 'tenant': openrc_dict['tenant_name'], 'auth_url': openrc_dict['os_auth_url'], 'insecure': openrc_dict['insecure'] } env_name = clusters_info['fuel']['openstack_env'] env_f_name = env_name for char in "-+ {}()[]": env_f_name = env_f_name.replace(char, '_') fuel_openrc_fname = os.path.join(var_dir, env_f_name + "_openrc") if ctx.fuel_openstack_creds is not None: with open(fuel_openrc_fname, "w") as fd: fd.write(openrc_templ.format(**ctx.fuel_openstack_creds)) msg = "Openrc for cluster {0} saves into {1}" logger.info(msg.format(env_name, fuel_openrc_fname)) nodes_to_run.extend(nodes) elif cluster == "ceph": if discover_nodes: cluster_info = clusters_info["ceph"] nodes_to_run.extend(ceph.discover_ceph_nodes(cluster_info)) else: logger.warning("Skip ceph cluster discovery") else: msg_templ = "Unknown cluster type in 'discover' parameter: {0!r}" raise ValueError(msg_templ.format(cluster)) return nodes_to_run
def discover_fuel_nodes(fuel_data, var_dir, discover_nodes=True): username, tenant_name, password = parse_creds(fuel_data["creds"]) creds = {"username": username, "tenant_name": tenant_name, "password": password} conn = KeystoneAuth(fuel_data["url"], creds, headers=None) msg = "openstack_env should be provided in fuel config" check_input_param("openstack_env" in fuel_data, msg) cluster_id = get_cluster_id(conn, fuel_data["openstack_env"]) cluster = reflect_cluster(conn, cluster_id) if not discover_nodes: logger.warning("Skip fuel cluster discovery") return ([], None, cluster.get_openrc()) version = FuelInfo(conn).get_version() fuel_nodes = list(cluster.get_nodes()) logger.info("Found FUEL {0}".format(".".join(map(str, version)))) network = "fuelweb_admin" if version >= [6, 0] else "admin" ssh_creds = fuel_data["ssh_creds"] fuel_host = urlparse(fuel_data["url"]).hostname fuel_ip = socket.gethostbyname(fuel_host) try: ssh_conn = connect("{0}@{1}".format(ssh_creds, fuel_host)) except AuthenticationException: raise StopTestError("Wrong fuel credentials") except Exception: logger.exception("While connection to FUEL") raise StopTestError("Failed to connect to FUEL") fuel_ext_iface = get_external_interface(ssh_conn, fuel_ip) logger.debug("Downloading fuel master key") fuel_key = download_master_key(ssh_conn) nodes = [] ips_ports = [] logger.info("Forwarding ssh ports from FUEL nodes to localhost") fuel_usr, fuel_passwd = ssh_creds.split(":", 1) ips = [str(fuel_node.get_ip(network)) for fuel_node in fuel_nodes] port_fw = forward_ssh_ports(fuel_host, fuel_usr, fuel_passwd, ips) listen_ip = get_ip_for_target(fuel_host) for port, fuel_node, ip in zip(port_fw, fuel_nodes, ips): logger.debug("SSH port forwarding {0} => localhost:{1}".format(ip, port)) conn_url = "ssh://[email protected]:{0}".format(port) set_key_for_node(("127.0.0.1", port), fuel_key) node = Node(conn_url, fuel_node["roles"]) node.monitor_ip = listen_ip nodes.append(node) ips_ports.append((ip, port)) logger.debug("Found %s fuel nodes for env %r" % (len(nodes), fuel_data["openstack_env"])) if version > [6, 0]: openrc = cluster.get_openrc() else: logger.warning("Getting openrc on fuel 6.0 is broken, skip") openrc = None return (nodes, (ssh_conn, fuel_ext_iface, ips_ports), openrc)
def discover_fuel_nodes(fuel_data, var_dir, discover_nodes=True): username, tenant_name, password = parse_creds(fuel_data['creds']) creds = {"username": username, "tenant_name": tenant_name, "password": password} conn = KeystoneAuth(fuel_data['url'], creds, headers=None) msg = "openstack_env should be provided in fuel config" check_input_param('openstack_env' in fuel_data, msg) cluster_id = get_cluster_id(conn, fuel_data['openstack_env']) cluster = reflect_cluster(conn, cluster_id) version = FuelInfo(conn).get_version() if not discover_nodes: logger.warning("Skip fuel cluster discovery") return ([], None, cluster.get_openrc(), version) fuel_nodes = list(cluster.get_nodes()) logger.info("Found FUEL {0}".format(".".join(map(str, version)))) network = 'fuelweb_admin' if version >= [6, 0] else 'admin' ssh_creds = fuel_data['ssh_creds'] fuel_host = urlparse(fuel_data['url']).hostname fuel_ip = socket.gethostbyname(fuel_host) try: ssh_conn = connect("{0}@{1}".format(ssh_creds, fuel_host)) except AuthenticationException: raise StopTestError("Wrong fuel credentials") except Exception: logger.exception("While connection to FUEL") raise StopTestError("Failed to connect to FUEL") fuel_ext_iface = get_external_interface(ssh_conn, fuel_ip) logger.debug("Downloading fuel master key") fuel_key = download_master_key(ssh_conn) nodes = [] ips_ports = [] logger.info("Forwarding ssh ports from FUEL nodes to localhost") fuel_usr, fuel_passwd = ssh_creds.split(":", 1) ips = [str(fuel_node.get_ip(network)) for fuel_node in fuel_nodes] port_fw = forward_ssh_ports(fuel_host, fuel_usr, fuel_passwd, ips) listen_ip = get_ip_for_target(fuel_host) for port, fuel_node, ip in zip(port_fw, fuel_nodes, ips): logger.debug( "SSH port forwarding {0} => localhost:{1}".format(ip, port)) conn_url = "ssh://[email protected]:{0}".format(port) set_key_for_node(('127.0.0.1', port), fuel_key) node = Node(conn_url, fuel_node['roles']) node.monitor_ip = listen_ip nodes.append(node) ips_ports.append((ip, port)) logger.debug("Found %s fuel nodes for env %r" % (len(nodes), fuel_data['openstack_env'])) return (nodes, (ssh_conn, fuel_ext_iface, ips_ports), cluster.get_openrc(), version)