def install_devstack(fab_settings, string_descriptor, hostname, download_conf=False,
                     ipversion="4", patch="", proxy="", quiet=False):
    verbose = []
    if quiet:
        verbose = ['output', 'running', 'warnings']
    with settings(**fab_settings), hide(*verbose):
        if exists("/etc/gai.conf"):
            append("/etc/gai.conf", "precedence ::ffff:0:0/96  100", use_sudo=True)
        if proxy:
            warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
                             "/etc/apt/apt.conf.d/00proxy", use_sudo=True))
            warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
                             "/etc/apt/apt.conf.d/00no_pipelining", use_sudo=True))
        update_time(sudo)
        if ipversion != "4":
            sudo("/sbin/sysctl -w net.ipv6.conf.all.forwarding=1")
            append("/etc/sysctl.conf", "net.ipv6.conf.all.forwarding=1", use_sudo=True)
        warn_if_fail(sudo("apt-get update"))
        warn_if_fail(sudo("apt-get install -y git python-pip"))
        warn_if_fail(run("git config --global user.email '*****@*****.**';"
                         "git config --global user.name 'Test Node'"))
        run("rm -rf ~/devstack")
        quit_if_fail(run("git clone https://github.com/openstack-dev/devstack.git"))
        if patch:
            warn_if_fail(run("git fetch https://review.openstack.org/openstack-dev/devstack {patch} "
                             "&& git cherry-pick FETCH_HEAD".format(patch)))
        warn_if_fail(put(string_descriptor, "devstack/local.conf", use_sudo=False))
        with cd("devstack"):
            warn_if_fail(run("./stack.sh"))
        if download_conf:
            get('~/devstack/openrc', "./openrc")
            get('/opt/stack/tempest/etc/tempest.conf', "./tempest.conf")
        collect_logs(run, hostname)
def prepare_for_install(settings_dict,
                     envs=None,
                     verbose=None,
                     proxy=None,
                     key=None):
    envs = envs or {}
    verbose = verbose or []
    if settings_dict['user'] != 'root':
        use_sudo_flag = True
        run_func = sudo
    else:
        use_sudo_flag = False
        run_func = run
    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        if exists("/etc/gai.conf"):
            append("/etc/gai.conf", "precedence ::ffff:0:0/96  100", use_sudo=use_sudo_flag)
        warn_if_fail(run_func("yum -y update"))
        warn_if_fail(run_func("yum -y install -y git python-pip vim ntpdate patch"))
        update_time(run_func)
        warn_if_fail(run_func("ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''"))
        warn_if_fail(run_func("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys"))
        append("/etc/ssh/ssh_config",
               "\nStrictHostKeyChecking no\nUserKnownHostsFile=/dev/null",
               use_sudo=use_sudo_flag)
        #warn_if_fail(run_func("/bin/systemctl stop  NetworkManager.service"))
        #warn_if_fail(run_func("/bin/systemctl disable NetworkManager.service"))
        append("~/.bashrc", "export TERM=xterm")
        if key:
            warn_if_fail(run_func("echo '%s' >> ~/.ssh/authorized_keys" % key))
        return run_func("cat ~/.ssh/id_rsa.pub")
Esempio n. 3
0
def run_services(host, settings_dict, envs=None, verbose=None, config=None):
    """
        Install OS with COI on other servers

    :param host: configuration of current lab box
    :param settings_dict: settings dictionary for Fabric
    :param envs: environment variables to inject when executing job
    :param verbose: if to hide all output or print everything
    :param config: configurations of all boxes for /etc/hosts
    """
    envs = envs or {}
    verbose = verbose or []
    if settings_dict['user'] != 'root':
        run_func = sudo
        use_sudo_flag = True
    else:
        run_func = run
        use_sudo_flag = False
    print >> sys.stderr, "FABRIC connecting to", settings_dict[
        "host_string"], host["hostname"]
    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        with cd("/root/"):
            update_time(run_func)
            run_func("apt-get update")
            run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
                     '-o Dpkg::Options::="--force-confdef" -o '
                     'Dpkg::Options::="--force-confold" dist-upgrade')
            # prepare /etc/hosts
            if config:
                append("/etc/hosts", prepare_hosts(config))
            run_func("apt-get install -y git")
            run_func(
                "git clone -b icehouse https://github.com/CiscoSystems/puppet_openstack_builder"
            )
            # use latest, not i.0 release
            #with cd("/root/puppet_openstack_builder"):
            #        run_func('git checkout i.0')
            sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
                "icehouse/snapshots/i.0",
                "icehouse-proposed",
                use_sudo=use_sudo_flag)
            sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
                "/snapshots/i.0",
                "-proposed",
                use_sudo=use_sudo_flag)
            with cd("/root/puppet_openstack_builder/install-scripts"):
                warn_if_fail(run_func("./setup.sh"))
                warn_if_fail(run_func('puppet agent --enable'))
                warn_if_fail(
                    run_func(
                        "puppet agent -td --server=build-server.domain.name --pluginsync"
                    ))
                collect_logs(run_func=run_func, hostname=host["hostname"])
Esempio n. 4
0
def run_services(host,
                 settings_dict,
                 envs=None,
                 verbose=None,
                 config=None):
    """
        Install OS with COI on other servers

    :param host: configuration of current lab box
    :param settings_dict: settings dictionary for Fabric
    :param envs: environment variables to inject when executing job
    :param verbose: if to hide all output or print everything
    :param config: configurations of all boxes for /etc/hosts
    """
    envs = envs or {}
    verbose = verbose or []
    if settings_dict['user'] != 'root':
        run_func = sudo
        use_sudo_flag = True
    else:
        run_func = run
        use_sudo_flag = False
    print >> sys.stderr, "FABRIC connecting to", settings_dict["host_string"], host["hostname"]
    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        with cd("/root/"):
            update_time(run_func)
            run_func("apt-get update")
            run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
                     '-o Dpkg::Options::="--force-confdef" -o '
                     'Dpkg::Options::="--force-confold" dist-upgrade')
            # prepare /etc/hosts
            if config:
                append("/etc/hosts", prepare_hosts(config))
            run_func("apt-get install -y git")
            run_func("git clone -b icehouse https://github.com/CiscoSystems/puppet_openstack_builder")
            # use latest, not i.0 release
            #with cd("/root/puppet_openstack_builder"):
            #        run_func('git checkout i.0')
            sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
                            "icehouse/snapshots/i.0",
                            "icehouse-proposed", use_sudo=use_sudo_flag)
            sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
                            "/snapshots/i.0",
                            "-proposed", use_sudo=use_sudo_flag)
            with cd("/root/puppet_openstack_builder/install-scripts"):
                warn_if_fail(run_func("./setup.sh"))
                warn_if_fail(run_func('puppet agent --enable'))
                warn_if_fail(run_func("puppet agent -td --server=build-server.domain.name --pluginsync"))
                collect_logs(run_func=run_func, hostname=host["hostname"])
def install_devstack(settings_dict,
                     envs=None,
                     verbose=None,
                     proxy=None,
                     patch=False,
                     opts=None):
    envs = envs or {}
    verbose = verbose or []
    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        if exists("/etc/gai.conf"):
            append("/etc/gai.conf", "precedence ::ffff:0:0/96  100", use_sudo=True)
        if proxy:
            warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
                             "/etc/apt/apt.conf.d/00proxy",
                             use_sudo=True))
            warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
                             "/etc/apt/apt.conf.d/00no_pipelining",
                             use_sudo=True))
        update_time(sudo)
        if opts.ipversion != 4:
            sudo("/sbin/sysctl -w net.ipv6.conf.all.forwarding=1")
            append("/etc/sysctl.conf", "net.ipv6.conf.all.forwarding=1", use_sudo=True)
        warn_if_fail(sudo("apt-get update"))
        warn_if_fail(sudo("apt-get install -y git python-pip"))
        warn_if_fail(run("git config --global user.email '*****@*****.**';"
                         "git config --global user.name 'Test Node'"))
        run("rm -rf ~/devstack")
        quit_if_fail(run("git clone https://github.com/openstack-dev/devstack.git"))
        make_local("devstack/local.conf", sudo_flag=False, opts=opts)
        with cd("devstack"):
            warn_if_fail(run("./stack.sh"))
            if patch:
                apply_changes()
        if exists('~/devstack/openrc'):
            get('~/devstack/openrc', "./openrc")
        else:
            print (red("No openrc file, something went wrong! :("))
        if exists('/opt/stack/tempest/etc/tempest.conf'):
            get('/opt/stack/tempest/etc/tempest.conf', "./tempest.conf")
        else:
            print (red("No openrc file, something went wrong! :("))
        print (green("Finished!"))
        return True
Esempio n. 6
0
def install_devstack(settings_dict,
                     envs=None,
                     verbose=None,
                     proxy=None,
                     patch=False,
                     opts=None):
    envs = envs or {}
    verbose = verbose or []
    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        if exists("/etc/gai.conf"):
            append("/etc/gai.conf", "precedence ::ffff:0:0/96  100", use_sudo=True)
        if proxy:
            warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
                             "/etc/apt/apt.conf.d/00proxy",
                             use_sudo=True))
            warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
                             "/etc/apt/apt.conf.d/00no_pipelining",
                             use_sudo=True))
        update_time(sudo)
        if opts.ipversion != 4:
            sudo("/sbin/sysctl -w net.ipv6.conf.all.forwarding=1")
            append("/etc/sysctl.conf", "net.ipv6.conf.all.forwarding=1", use_sudo=True)
        warn_if_fail(sudo("apt-get update"))
        warn_if_fail(sudo("apt-get install -y git python-pip"))
        warn_if_fail(run("git config --global user.email '*****@*****.**';"
                         "git config --global user.name 'Test Node'"))
        run("rm -rf ~/devstack")
        quit_if_fail(run("git clone https://github.com/openstack-dev/devstack.git"))
        make_local("devstack/local.conf", sudo_flag=False, opts=opts)
        with cd("devstack"):
            warn_if_fail(run("./stack.sh"))
            if patch:
                apply_changes()
        if exists('~/devstack/openrc'):
            get('~/devstack/openrc', "./openrc")
        else:
            print (red("No openrc file, something went wrong! :("))
        if exists('/opt/stack/tempest/etc/tempest.conf'):
            get('/opt/stack/tempest/etc/tempest.conf', "./tempest.conf")
        else:
            print (red("No openrc file, something went wrong! :("))
        print (green("Finished!"))
        return True
Esempio n. 7
0
def install_openstack(settings_dict,
                      envs=None,
                      verbose=None,
                      url_script=None,
                      prepare=False,
                      force=False,
                      config=None,
                      use_cobbler=False,
                      proxy=None):
    """
        Install OS with COI on build server

    :param settings_dict: settings dictionary for Fabric
    :param envs: environment variables to inject when executing job
    :param verbose: if to hide all output or print everything
    :param url_script: URl of Cisco installer script from Chris
    :param force: Use if you don't connect via interface you gonna bridge later
    :return: always true
    """
    envs = envs or {}
    verbose = verbose or []
    if settings_dict['user'] != 'root':
        use_sudo_flag = True
        run_func = sudo
    else:
        use_sudo_flag = False
        run_func = run

    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        with cd("/root/"):
            if proxy:
                warn_if_fail(
                    put(StringIO(
                        'Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'
                    ),
                        "/etc/apt/apt.conf.d/00proxy",
                        use_sudo=use_sudo_flag))
                warn_if_fail(
                    put(StringIO('Acquire::http::Pipeline-Depth "0";'),
                        "/etc/apt/apt.conf.d/00no_pipelining",
                        use_sudo=use_sudo_flag))
            run_func("apt-get update")
            run_func("apt-get install -y git")
            run_func("git config --global user.email '*****@*****.**';"
                     "git config --global user.name 'Test Node'")
            if not force and not prepare:
                update_time(run_func)
                # avoid grub and other prompts
                warn_if_fail(
                    run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
                             '-o Dpkg::Options::="--force-confdef" -o '
                             'Dpkg::Options::="--force-confold" dist-upgrade'))
                # prepare /etc/hosts file
                append("/etc/hosts", prepare_hosts(config))
                with cd("/root"):
                    warn_if_fail(
                        run_func(
                            "git clone -b icehouse "
                            "https://github.com/CiscoSystems/puppet_openstack_builder"
                        ))
                    with cd("puppet_openstack_builder"):
                        ## run the latest, not i.0 release
                        #run_func('git checkout i.0')
                        sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
                            "icehouse/snapshots/i.0",
                            "icehouse-proposed",
                            use_sudo=use_sudo_flag)
                        sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
                            "/snapshots/i.0",
                            "-proposed",
                            use_sudo=use_sudo_flag)
                        with cd("install-scripts"):
                            warn_if_fail(run_func("./install.sh"))
                prepare_files(
                    config,
                    paths=(
                        "/etc/puppet/data/hiera_data/user.common.yaml",
                        "/etc/puppet/data/hiera_data/user.full_ha.yaml",
                        "/etc/puppet/data/cobbler/cobbler.yaml",
                        "/etc/puppet/data/role_mappings.yaml",
                        "/etc/puppet/data/hiera_data/hostname/build_server.yaml"
                    ),
                    use_sudo_flag=use_sudo_flag)
                prepare_new_files(config,
                                  path="/etc/puppet/data/hiera_data/hostname",
                                  use_sudo_flag=use_sudo_flag)
                resolve_names(run_func, use_sudo_flag)
                result = run_func(
                    'puppet apply -v /etc/puppet/manifests/site.pp')
                tries = 1
                if use_cobbler:
                    cobbler_error = "[cobbler-sync]/returns: unable to connect to cobbler on localhost using cobbler"
                    while cobbler_error in result and tries <= APPLY_LIMIT:
                        time.sleep(60)
                        print >> sys.stderr, "Cobbler is not installed properly, running apply again"
                        result = run_func(
                            'puppet apply -v /etc/puppet/manifests/site.pp',
                            pty=False)
                        tries += 1
                error = "Error:"
                while error in result and tries <= APPLY_LIMIT:
                    time.sleep(60)
                    print >> sys.stderr, "Some errors found, running apply again"
                    result = run_func(
                        'puppet apply -v /etc/puppet/manifests/site.pp',
                        pty=False)
                    tries += 1
                if exists('/root/openrc'):
                    get('/root/openrc', "./openrc")
                else:
                    print(red("No openrc file, something went wrong! :("))
                print(green("Copying logs and configs"))
                collect_logs(
                    run_func=run_func,
                    hostname=config["servers"]["build-server"][0]["hostname"],
                    clean=True)
                print(green("Finished!"))
                return True
            elif not force and prepare:
                return True
    print(green("Finished!"))
    return True
Esempio n. 8
0
def install_openstack(settings_dict,
                      envs=None,
                      verbose=None,
                      url_script=None,
                      prepare=False,
                      force=False,
                      config=None,
                      use_cobbler=False,
                      proxy=None):
    """
        Install OS with COI on build server

    :param settings_dict: settings dictionary for Fabric
    :param envs: environment variables to inject when executing job
    :param verbose: if to hide all output or print everything
    :param url_script: URl of Cisco installer script from Chris
    :param force: Use if you don't connect via interface you gonna bridge later
    :return: always true
    """
    envs = envs or {}
    verbose = verbose or []
    if settings_dict['user'] != 'root':
        use_sudo_flag = True
        run_func = sudo
    else:
        use_sudo_flag = False
        run_func = run

    with settings(**settings_dict), hide(*verbose), shell_env(**envs):
        with cd("/root/"):
            if proxy:
                warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
                                 "/etc/apt/apt.conf.d/00proxy",
                                 use_sudo=use_sudo_flag))
                warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
                                 "/etc/apt/apt.conf.d/00no_pipelining",
                                 use_sudo=use_sudo_flag))
            run_func("apt-get update")
            run_func("apt-get install -y git")
            run_func("git config --global user.email '*****@*****.**';"
                     "git config --global user.name 'Test Node'")
            if not force and not prepare:
                update_time(run_func)
                # avoid grub and other prompts
                warn_if_fail(run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
                                      '-o Dpkg::Options::="--force-confdef" -o '
                                      'Dpkg::Options::="--force-confold" dist-upgrade'))
                # prepare /etc/hosts file
                append("/etc/hosts", prepare_hosts(config))
                with cd("/root"):
                    warn_if_fail(run_func("git clone -b icehouse "
                                          "https://github.com/CiscoSystems/puppet_openstack_builder"))
                    with cd("puppet_openstack_builder"):
                        ## run the latest, not i.0 release
                        #run_func('git checkout i.0')
                        sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
                            "icehouse/snapshots/i.0",
                            "icehouse-proposed", use_sudo=use_sudo_flag)
                        with cd("install-scripts"):
                            warn_if_fail(run_func("./install.sh"))
                prepare_files(config,
                              paths=(
                                  "/etc/puppet/data/hiera_data/user.common.yaml",
                                  "/etc/puppet/data/hiera_data/user.full_ha.yaml",
                                  "/etc/puppet/data/cobbler/cobbler.yaml",
                                  "/etc/puppet/data/role_mappings.yaml",
                                  "/etc/puppet/data/hiera_data/hostname/build_server.yaml"
                              ),
                              use_sudo_flag=use_sudo_flag)
                prepare_new_files(
                    config,
                    path="/etc/puppet/data/hiera_data/hostname",
                    use_sudo_flag=use_sudo_flag
                )
                resolve_names(run_func, use_sudo_flag)
                result = run_func('puppet apply -v /etc/puppet/manifests/site.pp')
                tries = 1
                if use_cobbler:
                    cobbler_error = "[cobbler-sync]/returns: unable to connect to cobbler on localhost using cobbler"
                    while cobbler_error in result and tries <= APPLY_LIMIT:
                        time.sleep(60)
                        print >> sys.stderr, "Cobbler is not installed properly, running apply again"
                        result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False)
                        tries += 1
                error = "Error:"
                while error in result and tries <= APPLY_LIMIT:
                    time.sleep(60)
                    print >> sys.stderr, "Some errors found, running apply again"
                    result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False)
                    tries += 1
                if exists('/root/openrc'):
                    get('/root/openrc', "./openrc")
                else:
                    print (red("No openrc file, something went wrong! :("))
                print (green("Copying logs and configs"))
                collect_logs(run_func=run_func, hostname=config["servers"]["build-server"]["hostname"], clean=True)
                print (green("Finished!"))
                return True
            elif not force and prepare:
                return True
    print (green("Finished!"))
    return True
Esempio n. 9
0
def install(aggregated_configs, apt_cacher_proxy=None,
            patch="", proxy="", devstack_repo="", devstack_br="", quiet=False):
    verbose = []
    fab_settings = aggregated_configs[env.host]["fab_settings"]
    local_conf = aggregated_configs[env.host]["devstack_config"]
    config_files = aggregated_configs[env.host]["config_files"]
    hostname = aggregated_configs[env.host]["hostname"]
    exec_command_before = aggregated_configs[env.host]["exec_commands_before"]
    exec_command_after = aggregated_configs[env.host]["exec_commands_after"]
    if quiet:
        verbose = ['output', 'running', 'warnings']
    with settings(**fab_settings), hide(*verbose):
        if exists("/etc/gai.conf"):
            append("/etc/gai.conf", "precedence ::ffff:0:0/96  100",
                   use_sudo=True)
        if proxy:
            warn_if_fail(put(StringIO(
                'Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
                             "/etc/apt/apt.conf.d/00proxy", use_sudo=True))
            warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
                             "/etc/apt/apt.conf.d/00no_pipelining",
                             use_sudo=True))
        if apt_cacher_proxy:
            warn_if_fail(
                put(StringIO('Acquire::http { Proxy "{url}"; };'.format(
                    url=apt_cacher_proxy)),
                    "/etc/apt/apt.conf.d/02proxy", use_sudo=True))
        update_time(sudo)
        warn_if_fail(sudo("apt-get update"))
        warn_if_fail(sudo("apt-get install -y git python-pip"))
        warn_if_fail(
            run("git config --global user.email '*****@*****.**';"
                "git config --global user.name 'Test Node'"))
        if exists("devstack"):
            with cd("devstack"):
                warn_if_fail(run("./unstack.sh"))
        run("rm -rf ~/devstack")
        quit_if_fail(run(
            "git clone -b {branch} {devstack}".format(devstack=devstack_repo,
                                                      branch=devstack_br)))
        if patch:
            with cd("devstack"):
                warn_if_fail(
                    run(
                        "git fetch https://review.openstack.org/openstack-dev/devstack {patch} "
                        "&& git cherry-pick FETCH_HEAD".format(patch=patch)))
        warn_if_fail(put(StringIO(local_conf), "devstack/local.conf"))
        if exec_command_before:
            run(exec_command_before)
        if config_files:
            for path, conf_file in config_files.iteritems():
                warn_if_fail(put(StringIO(conf_file), path))
        with cd("devstack"):
            warn_if_fail(run("./stack.sh"))
        path = os.environ.get("WORKSPACE", os.getcwd())
        if os.path.exists(
                "{path}/id_rsa_{key}".format(path=path, key=hostname)):
            path = os.environ.get("WORKSPACE", os.getcwd())
            put("{path}/id_rsa_{key}".format(path=path, key=hostname),
                "~/.ssh/id_rsa")
            put("{path}/id_rsa_{key}.pub".format(path=path, key=hostname),
                "~/.ssh/id_rsa.pub")
            put("{0}/id_rsa_all.pub".format(path), "/tmp/all_authorized")
            warn_if_fail(run("chmod 500 ~/.ssh/id_rsa"))
            warn_if_fail(
                run("cat /tmp/all_authorized >> ~/.ssh/authorized_keys"))
            append("/etc/ssh/ssh_config",
                   "\nStrictHostKeyChecking no\nUserKnownHostsFile=/dev/null",
                   use_sudo=True)
        if exec_command_after:
            run(exec_command_after)
        collect_logs(run, hostname)
        collect_logs_devstack("install_{host}".format(host=hostname))