Example #1
0
def discover_services(client, opts):
    user, password, key = parse_creds(opts.pop('auth'))

    services = []
    if opts['service'] == "all":
        services = client.services.list()
    else:
        if isinstance(opts['service'], basestring):
            opts['service'] = [opts['service']]

        for s in opts['service']:
            services.extend(client.services.list(binary=s))

    host_services_mapping = {}

    for service in services:
        ip = socket.gethostbyname(service.host)
        host_services_mapping[ip].append(service.binary)

    logger.debug("Found %s openstack service nodes" %
                 len(host_services_mapping))
    return [
        Node(host, services, username=user, password=password, key_path=key)
        for host, services in host_services_mapping.items()
    ]
Example #2
0
def discover_vms(client, search_opts):
    user, password, key = parse_creds(search_opts.pop('auth'))

    servers = client.servers.list(search_opts=search_opts)
    logger.debug("Found %s openstack vms" % len(servers))
    return [
        Node(get_floating_ip(server), ["test_vm"],
             username=user,
             password=password,
             key_path=key) for server in servers if get_floating_ip(server)
    ]
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True):
    nodes_to_run = []
    clean_data = None
    ctx.fuel_openstack_creds = None

    for cluster in discover:
        if cluster == "openstack" and not discover_nodes:
            logger.warning("Skip openstack cluster discovery")
        elif cluster == "openstack" and discover_nodes:
            cluster_info = clusters_info["openstack"]
            conn = cluster_info['connection']
            user, passwd, tenant = parse_creds(conn['creds'])

            auth_data = dict(
                auth_url=conn['auth_url'],
                username=user,
                api_key=passwd,
                project_id=tenant)

            if not conn:
                logger.error("No connection provided for %s. Skipping"
                             % cluster)
                continue

            logger.debug("Discovering openstack nodes "
                         "with connection details: %r" %
                         conn)

            os_nodes = openstack.discover_openstack_nodes(auth_data,
                                                          cluster_info)
            nodes_to_run.extend(os_nodes)

        elif cluster == "fuel" or cluster == "fuel_openrc_only":
            if cluster == "fuel_openrc_only":
                discover_nodes = False

            res = fuel.discover_fuel_nodes(clusters_info['fuel'],
                                           var_dir,
                                           discover_nodes)
            nodes, clean_data, openrc_dict, version = res

            if openrc_dict is None:
                ctx.fuel_openstack_creds = None
            else:
                if version >= [8, 0] and openrc_dict['os_auth_url'].startswith("https://"):
                    logger.warning("Fixing FUEL 8.0 AUTH url - replace https://->http://")
                    openrc_dict['os_auth_url'] = "http" + openrc_dict['os_auth_url'][5:]

                ctx.fuel_openstack_creds = {
                    'name': openrc_dict['username'],
                    'passwd': openrc_dict['password'],
                    'tenant': openrc_dict['tenant_name'],
                    'auth_url': openrc_dict['os_auth_url'],
                    'insecure': openrc_dict['insecure']}

            env_name = clusters_info['fuel']['openstack_env']
            env_f_name = env_name
            for char in "-+ {}()[]":
                env_f_name = env_f_name.replace(char, '_')

            fuel_openrc_fname = os.path.join(var_dir,
                                             env_f_name + "_openrc")

            if ctx.fuel_openstack_creds is not None:
                with open(fuel_openrc_fname, "w") as fd:
                    fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
                    msg = "Openrc for cluster {0} saves into {1}"
                    logger.info(msg.format(env_name, fuel_openrc_fname))
            nodes_to_run.extend(nodes)

        elif cluster == "ceph":
            if discover_nodes:
                cluster_info = clusters_info["ceph"]
                nodes_to_run.extend(ceph.discover_ceph_nodes(cluster_info))
            else:
                logger.warning("Skip ceph cluster discovery")
        else:
            msg_templ = "Unknown cluster type in 'discover' parameter: {0!r}"
            raise ValueError(msg_templ.format(cluster))

    return nodes_to_run
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True):
    nodes_to_run = []
    clean_data = None
    ctx.fuel_openstack_creds = None

    for cluster in discover:
        if cluster == "openstack" and not discover_nodes:
            logger.warning("Skip openstack cluster discovery")
        elif cluster == "openstack" and discover_nodes:
            cluster_info = clusters_info["openstack"]
            conn = cluster_info["connection"]
            user, passwd, tenant = parse_creds(conn["creds"])

            auth_data = dict(auth_url=conn["auth_url"], username=user, api_key=passwd, project_id=tenant)

            if not conn:
                logger.error("No connection provided for %s. Skipping" % cluster)
                continue

            logger.debug("Discovering openstack nodes " "with connection details: %r" % conn)

            os_nodes = openstack.discover_openstack_nodes(auth_data, cluster_info)
            nodes_to_run.extend(os_nodes)

        elif cluster == "fuel" or cluster == "fuel_openrc_only":
            if cluster == "fuel_openrc_only":
                discover_nodes = False

            res = fuel.discover_fuel_nodes(clusters_info["fuel"], var_dir, discover_nodes)
            nodes, clean_data, openrc_dict = res

            if openrc_dict is None:
                ctx.fuel_openstack_creds = None
            else:
                ctx.fuel_openstack_creds = {
                    "name": openrc_dict["username"],
                    "passwd": openrc_dict["password"],
                    "tenant": openrc_dict["tenant_name"],
                    "auth_url": openrc_dict["os_auth_url"],
                    "insecure": openrc_dict.get("insecure", False),
                }

            env_name = clusters_info["fuel"]["openstack_env"]
            env_f_name = env_name
            for char in "-+ {}()[]":
                env_f_name = env_f_name.replace(char, "_")

            fuel_openrc_fname = os.path.join(var_dir, env_f_name + "_openrc")

            if ctx.fuel_openstack_creds is not None:
                with open(fuel_openrc_fname, "w") as fd:
                    fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
                    msg = "Openrc for cluster {0} saves into {1}"
                    logger.info(msg.format(env_name, fuel_openrc_fname))
            nodes_to_run.extend(nodes)

        elif cluster == "ceph":
            if discover_nodes:
                cluster_info = clusters_info["ceph"]
                nodes_to_run.extend(ceph.discover_ceph_nodes(cluster_info))
            else:
                logger.warning("Skip ceph cluster discovery")
        else:
            msg_templ = "Unknown cluster type in 'discover' parameter: {0!r}"
            raise ValueError(msg_templ.format(cluster))

    return nodes_to_run
def discover(ctx, discover, clusters_info, var_dir, discover_nodes=True):
    nodes_to_run = []
    clean_data = None
    ctx.fuel_openstack_creds = None

    for cluster in discover:
        if cluster == "openstack" and not discover_nodes:
            logger.warning("Skip openstack cluster discovery")
        elif cluster == "openstack" and discover_nodes:
            cluster_info = clusters_info["openstack"]
            conn = cluster_info['connection']
            user, passwd, tenant = parse_creds(conn['creds'])

            auth_data = dict(auth_url=conn['auth_url'],
                             username=user,
                             api_key=passwd,
                             project_id=tenant)

            if not conn:
                logger.error("No connection provided for %s. Skipping" %
                             cluster)
                continue

            logger.debug("Discovering openstack nodes "
                         "with connection details: %r" % conn)

            os_nodes = openstack.discover_openstack_nodes(
                auth_data, cluster_info)
            nodes_to_run.extend(os_nodes)

        elif cluster == "fuel" or cluster == "fuel_openrc_only":
            if cluster == "fuel_openrc_only":
                discover_nodes = False

            res = fuel.discover_fuel_nodes(clusters_info['fuel'], var_dir,
                                           discover_nodes)
            nodes, clean_data, openrc_dict, version = res

            if openrc_dict is None:
                ctx.fuel_openstack_creds = None
            else:
                if version >= [8, 0] and openrc_dict['os_auth_url'].startswith(
                        "https://"):
                    logger.warning(
                        "Fixing FUEL 8.0 AUTH url - replace https://->http://")
                    openrc_dict['os_auth_url'] = "http" + openrc_dict[
                        'os_auth_url'][5:]

                ctx.fuel_openstack_creds = {
                    'name': openrc_dict['username'],
                    'passwd': openrc_dict['password'],
                    'tenant': openrc_dict['tenant_name'],
                    'auth_url': openrc_dict['os_auth_url'],
                    'insecure': openrc_dict['insecure']
                }

            env_name = clusters_info['fuel']['openstack_env']
            env_f_name = env_name
            for char in "-+ {}()[]":
                env_f_name = env_f_name.replace(char, '_')

            fuel_openrc_fname = os.path.join(var_dir, env_f_name + "_openrc")

            if ctx.fuel_openstack_creds is not None:
                with open(fuel_openrc_fname, "w") as fd:
                    fd.write(openrc_templ.format(**ctx.fuel_openstack_creds))
                    msg = "Openrc for cluster {0} saves into {1}"
                    logger.info(msg.format(env_name, fuel_openrc_fname))
            nodes_to_run.extend(nodes)

        elif cluster == "ceph":
            if discover_nodes:
                cluster_info = clusters_info["ceph"]
                nodes_to_run.extend(ceph.discover_ceph_nodes(cluster_info))
            else:
                logger.warning("Skip ceph cluster discovery")
        else:
            msg_templ = "Unknown cluster type in 'discover' parameter: {0!r}"
            raise ValueError(msg_templ.format(cluster))

    return nodes_to_run
Example #6
0
def discover_fuel_nodes(fuel_data, var_dir, discover_nodes=True):
    username, tenant_name, password = parse_creds(fuel_data["creds"])
    creds = {"username": username, "tenant_name": tenant_name, "password": password}

    conn = KeystoneAuth(fuel_data["url"], creds, headers=None)

    msg = "openstack_env should be provided in fuel config"
    check_input_param("openstack_env" in fuel_data, msg)

    cluster_id = get_cluster_id(conn, fuel_data["openstack_env"])
    cluster = reflect_cluster(conn, cluster_id)

    if not discover_nodes:
        logger.warning("Skip fuel cluster discovery")
        return ([], None, cluster.get_openrc())

    version = FuelInfo(conn).get_version()

    fuel_nodes = list(cluster.get_nodes())

    logger.info("Found FUEL {0}".format(".".join(map(str, version))))

    network = "fuelweb_admin" if version >= [6, 0] else "admin"

    ssh_creds = fuel_data["ssh_creds"]

    fuel_host = urlparse(fuel_data["url"]).hostname
    fuel_ip = socket.gethostbyname(fuel_host)

    try:
        ssh_conn = connect("{0}@{1}".format(ssh_creds, fuel_host))
    except AuthenticationException:
        raise StopTestError("Wrong fuel credentials")
    except Exception:
        logger.exception("While connection to FUEL")
        raise StopTestError("Failed to connect to FUEL")

    fuel_ext_iface = get_external_interface(ssh_conn, fuel_ip)

    logger.debug("Downloading fuel master key")
    fuel_key = download_master_key(ssh_conn)

    nodes = []
    ips_ports = []

    logger.info("Forwarding ssh ports from FUEL nodes to localhost")
    fuel_usr, fuel_passwd = ssh_creds.split(":", 1)
    ips = [str(fuel_node.get_ip(network)) for fuel_node in fuel_nodes]
    port_fw = forward_ssh_ports(fuel_host, fuel_usr, fuel_passwd, ips)
    listen_ip = get_ip_for_target(fuel_host)

    for port, fuel_node, ip in zip(port_fw, fuel_nodes, ips):
        logger.debug("SSH port forwarding {0} => localhost:{1}".format(ip, port))

        conn_url = "ssh://[email protected]:{0}".format(port)
        set_key_for_node(("127.0.0.1", port), fuel_key)

        node = Node(conn_url, fuel_node["roles"])
        node.monitor_ip = listen_ip
        nodes.append(node)
        ips_ports.append((ip, port))

    logger.debug("Found %s fuel nodes for env %r" % (len(nodes), fuel_data["openstack_env"]))

    if version > [6, 0]:
        openrc = cluster.get_openrc()
    else:
        logger.warning("Getting openrc on fuel 6.0 is broken, skip")
        openrc = None

    return (nodes, (ssh_conn, fuel_ext_iface, ips_ports), openrc)
Example #7
0
def discover_fuel_nodes(fuel_data, var_dir, discover_nodes=True):
    username, tenant_name, password = parse_creds(fuel_data['creds'])
    creds = {"username": username,
             "tenant_name": tenant_name,
             "password": password}

    conn = KeystoneAuth(fuel_data['url'], creds, headers=None)

    msg = "openstack_env should be provided in fuel config"
    check_input_param('openstack_env' in fuel_data, msg)

    cluster_id = get_cluster_id(conn, fuel_data['openstack_env'])
    cluster = reflect_cluster(conn, cluster_id)
    version = FuelInfo(conn).get_version()

    if not discover_nodes:
        logger.warning("Skip fuel cluster discovery")
        return ([], None, cluster.get_openrc(), version)

    fuel_nodes = list(cluster.get_nodes())

    logger.info("Found FUEL {0}".format(".".join(map(str, version))))

    network = 'fuelweb_admin' if version >= [6, 0] else 'admin'

    ssh_creds = fuel_data['ssh_creds']

    fuel_host = urlparse(fuel_data['url']).hostname
    fuel_ip = socket.gethostbyname(fuel_host)

    try:
        ssh_conn = connect("{0}@{1}".format(ssh_creds, fuel_host))
    except AuthenticationException:
        raise StopTestError("Wrong fuel credentials")
    except Exception:
        logger.exception("While connection to FUEL")
        raise StopTestError("Failed to connect to FUEL")

    fuel_ext_iface = get_external_interface(ssh_conn, fuel_ip)

    logger.debug("Downloading fuel master key")
    fuel_key = download_master_key(ssh_conn)

    nodes = []
    ips_ports = []

    logger.info("Forwarding ssh ports from FUEL nodes to localhost")
    fuel_usr, fuel_passwd = ssh_creds.split(":", 1)
    ips = [str(fuel_node.get_ip(network)) for fuel_node in fuel_nodes]
    port_fw = forward_ssh_ports(fuel_host, fuel_usr, fuel_passwd, ips)
    listen_ip = get_ip_for_target(fuel_host)

    for port, fuel_node, ip in zip(port_fw, fuel_nodes, ips):
        logger.debug(
            "SSH port forwarding {0} => localhost:{1}".format(ip, port))

        conn_url = "ssh://[email protected]:{0}".format(port)
        set_key_for_node(('127.0.0.1', port), fuel_key)

        node = Node(conn_url, fuel_node['roles'])
        node.monitor_ip = listen_ip
        nodes.append(node)
        ips_ports.append((ip, port))

    logger.debug("Found %s fuel nodes for env %r" %
                 (len(nodes), fuel_data['openstack_env']))

    return (nodes,
            (ssh_conn, fuel_ext_iface, ips_ports),
            cluster.get_openrc(),
            version)