def install_devstack(fab_settings, string_descriptor, hostname, download_conf=False,
ipversion="4", patch="", proxy="", quiet=False):
verbose = []
if quiet:
verbose = ['output', 'running', 'warnings']
with settings(**fab_settings), hide(*verbose):
if exists("/etc/gai.conf"):
append("/etc/gai.conf", "precedence ::ffff:0:0/96 100", use_sudo=True)
if proxy:
warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
"/etc/apt/apt.conf.d/00proxy", use_sudo=True))
warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining", use_sudo=True))
update_time(sudo)
if ipversion != "4":
sudo("/sbin/sysctl -w net.ipv6.conf.all.forwarding=1")
append("/etc/sysctl.conf", "net.ipv6.conf.all.forwarding=1", use_sudo=True)
warn_if_fail(sudo("apt-get update"))
warn_if_fail(sudo("apt-get install -y git python-pip"))
warn_if_fail(run("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'"))
run("rm -rf ~/devstack")
quit_if_fail(run("git clone https://github.com/openstack-dev/devstack.git"))
if patch:
warn_if_fail(run("git fetch https://review.openstack.org/openstack-dev/devstack {patch} "
"&& git cherry-pick FETCH_HEAD".format(patch)))
warn_if_fail(put(string_descriptor, "devstack/local.conf", use_sudo=False))
with cd("devstack"):
warn_if_fail(run("./stack.sh"))
if download_conf:
get('~/devstack/openrc', "./openrc")
get('/opt/stack/tempest/etc/tempest.conf', "./tempest.conf")
collect_logs(run, hostname)
def prepare_for_install(settings_dict,
envs=None,
verbose=None,
proxy=None,
key=None):
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
use_sudo_flag = True
run_func = sudo
else:
use_sudo_flag = False
run_func = run
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
if exists("/etc/gai.conf"):
append("/etc/gai.conf", "precedence ::ffff:0:0/96 100", use_sudo=use_sudo_flag)
warn_if_fail(run_func("yum -y update"))
warn_if_fail(run_func("yum -y install -y git python-pip vim ntpdate patch"))
update_time(run_func)
warn_if_fail(run_func("ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''"))
warn_if_fail(run_func("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys"))
append("/etc/ssh/ssh_config",
"\nStrictHostKeyChecking no\nUserKnownHostsFile=/dev/null",
use_sudo=use_sudo_flag)
#warn_if_fail(run_func("/bin/systemctl stop NetworkManager.service"))
#warn_if_fail(run_func("/bin/systemctl disable NetworkManager.service"))
append("~/.bashrc", "export TERM=xterm")
if key:
warn_if_fail(run_func("echo '%s' >> ~/.ssh/authorized_keys" % key))
return run_func("cat ~/.ssh/id_rsa.pub")
def run_services(host, settings_dict, envs=None, verbose=None, config=None):
"""
Install OS with COI on other servers
:param host: configuration of current lab box
:param settings_dict: settings dictionary for Fabric
:param envs: environment variables to inject when executing job
:param verbose: if to hide all output or print everything
:param config: configurations of all boxes for /etc/hosts
"""
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
run_func = sudo
use_sudo_flag = True
else:
run_func = run
use_sudo_flag = False
print >> sys.stderr, "FABRIC connecting to", settings_dict[
"host_string"], host["hostname"]
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
with cd("/root/"):
update_time(run_func)
run_func("apt-get update")
run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
'-o Dpkg::Options::="--force-confdef" -o '
'Dpkg::Options::="--force-confold" dist-upgrade')
# prepare /etc/hosts
if config:
append("/etc/hosts", prepare_hosts(config))
run_func("apt-get install -y git")
run_func(
"git clone -b icehouse https://github.com/CiscoSystems/puppet_openstack_builder"
)
# use latest, not i.0 release
#with cd("/root/puppet_openstack_builder"):
# run_func('git checkout i.0')
sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
"icehouse/snapshots/i.0",
"icehouse-proposed",
use_sudo=use_sudo_flag)
sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
"/snapshots/i.0",
"-proposed",
use_sudo=use_sudo_flag)
with cd("/root/puppet_openstack_builder/install-scripts"):
warn_if_fail(run_func("./setup.sh"))
warn_if_fail(run_func('puppet agent --enable'))
warn_if_fail(
run_func(
"puppet agent -td --server=build-server.domain.name --pluginsync"
))
collect_logs(run_func=run_func, hostname=host["hostname"])
def run_services(host,
settings_dict,
envs=None,
verbose=None,
config=None):
"""
Install OS with COI on other servers
:param host: configuration of current lab box
:param settings_dict: settings dictionary for Fabric
:param envs: environment variables to inject when executing job
:param verbose: if to hide all output or print everything
:param config: configurations of all boxes for /etc/hosts
"""
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
run_func = sudo
use_sudo_flag = True
else:
run_func = run
use_sudo_flag = False
print >> sys.stderr, "FABRIC connecting to", settings_dict["host_string"], host["hostname"]
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
with cd("/root/"):
update_time(run_func)
run_func("apt-get update")
run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
'-o Dpkg::Options::="--force-confdef" -o '
'Dpkg::Options::="--force-confold" dist-upgrade')
# prepare /etc/hosts
if config:
append("/etc/hosts", prepare_hosts(config))
run_func("apt-get install -y git")
run_func("git clone -b icehouse https://github.com/CiscoSystems/puppet_openstack_builder")
# use latest, not i.0 release
#with cd("/root/puppet_openstack_builder"):
# run_func('git checkout i.0')
sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
"icehouse/snapshots/i.0",
"icehouse-proposed", use_sudo=use_sudo_flag)
sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
"/snapshots/i.0",
"-proposed", use_sudo=use_sudo_flag)
with cd("/root/puppet_openstack_builder/install-scripts"):
warn_if_fail(run_func("./setup.sh"))
warn_if_fail(run_func('puppet agent --enable'))
warn_if_fail(run_func("puppet agent -td --server=build-server.domain.name --pluginsync"))
collect_logs(run_func=run_func, hostname=host["hostname"])
def install_devstack(settings_dict,
envs=None,
verbose=None,
proxy=None,
patch=False,
opts=None):
envs = envs or {}
verbose = verbose or []
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
if exists("/etc/gai.conf"):
append("/etc/gai.conf", "precedence ::ffff:0:0/96 100", use_sudo=True)
if proxy:
warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
"/etc/apt/apt.conf.d/00proxy",
use_sudo=True))
warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=True))
update_time(sudo)
if opts.ipversion != 4:
sudo("/sbin/sysctl -w net.ipv6.conf.all.forwarding=1")
append("/etc/sysctl.conf", "net.ipv6.conf.all.forwarding=1", use_sudo=True)
warn_if_fail(sudo("apt-get update"))
warn_if_fail(sudo("apt-get install -y git python-pip"))
warn_if_fail(run("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'"))
run("rm -rf ~/devstack")
quit_if_fail(run("git clone https://github.com/openstack-dev/devstack.git"))
make_local("devstack/local.conf", sudo_flag=False, opts=opts)
with cd("devstack"):
warn_if_fail(run("./stack.sh"))
if patch:
apply_changes()
if exists('~/devstack/openrc'):
get('~/devstack/openrc', "./openrc")
else:
print (red("No openrc file, something went wrong! :("))
if exists('/opt/stack/tempest/etc/tempest.conf'):
get('/opt/stack/tempest/etc/tempest.conf', "./tempest.conf")
else:
print (red("No openrc file, something went wrong! :("))
print (green("Finished!"))
return True
def install_devstack(settings_dict,
envs=None,
verbose=None,
proxy=None,
patch=False,
opts=None):
envs = envs or {}
verbose = verbose or []
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
if exists("/etc/gai.conf"):
append("/etc/gai.conf", "precedence ::ffff:0:0/96 100", use_sudo=True)
if proxy:
warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
"/etc/apt/apt.conf.d/00proxy",
use_sudo=True))
warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=True))
update_time(sudo)
if opts.ipversion != 4:
sudo("/sbin/sysctl -w net.ipv6.conf.all.forwarding=1")
append("/etc/sysctl.conf", "net.ipv6.conf.all.forwarding=1", use_sudo=True)
warn_if_fail(sudo("apt-get update"))
warn_if_fail(sudo("apt-get install -y git python-pip"))
warn_if_fail(run("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'"))
run("rm -rf ~/devstack")
quit_if_fail(run("git clone https://github.com/openstack-dev/devstack.git"))
make_local("devstack/local.conf", sudo_flag=False, opts=opts)
with cd("devstack"):
warn_if_fail(run("./stack.sh"))
if patch:
apply_changes()
if exists('~/devstack/openrc'):
get('~/devstack/openrc', "./openrc")
else:
print (red("No openrc file, something went wrong! :("))
if exists('/opt/stack/tempest/etc/tempest.conf'):
get('/opt/stack/tempest/etc/tempest.conf', "./tempest.conf")
else:
print (red("No openrc file, something went wrong! :("))
print (green("Finished!"))
return True
def install_openstack(settings_dict,
envs=None,
verbose=None,
url_script=None,
prepare=False,
force=False,
config=None,
use_cobbler=False,
proxy=None):
"""
Install OS with COI on build server
:param settings_dict: settings dictionary for Fabric
:param envs: environment variables to inject when executing job
:param verbose: if to hide all output or print everything
:param url_script: URl of Cisco installer script from Chris
:param force: Use if you don't connect via interface you gonna bridge later
:return: always true
"""
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
use_sudo_flag = True
run_func = sudo
else:
use_sudo_flag = False
run_func = run
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
with cd("/root/"):
if proxy:
warn_if_fail(
put(StringIO(
'Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'
),
"/etc/apt/apt.conf.d/00proxy",
use_sudo=use_sudo_flag))
warn_if_fail(
put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=use_sudo_flag))
run_func("apt-get update")
run_func("apt-get install -y git")
run_func("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'")
if not force and not prepare:
update_time(run_func)
# avoid grub and other prompts
warn_if_fail(
run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
'-o Dpkg::Options::="--force-confdef" -o '
'Dpkg::Options::="--force-confold" dist-upgrade'))
# prepare /etc/hosts file
append("/etc/hosts", prepare_hosts(config))
with cd("/root"):
warn_if_fail(
run_func(
"git clone -b icehouse "
"https://github.com/CiscoSystems/puppet_openstack_builder"
))
with cd("puppet_openstack_builder"):
## run the latest, not i.0 release
#run_func('git checkout i.0')
sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
"icehouse/snapshots/i.0",
"icehouse-proposed",
use_sudo=use_sudo_flag)
sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
"/snapshots/i.0",
"-proposed",
use_sudo=use_sudo_flag)
with cd("install-scripts"):
warn_if_fail(run_func("./install.sh"))
prepare_files(
config,
paths=(
"/etc/puppet/data/hiera_data/user.common.yaml",
"/etc/puppet/data/hiera_data/user.full_ha.yaml",
"/etc/puppet/data/cobbler/cobbler.yaml",
"/etc/puppet/data/role_mappings.yaml",
"/etc/puppet/data/hiera_data/hostname/build_server.yaml"
),
use_sudo_flag=use_sudo_flag)
prepare_new_files(config,
path="/etc/puppet/data/hiera_data/hostname",
use_sudo_flag=use_sudo_flag)
resolve_names(run_func, use_sudo_flag)
result = run_func(
'puppet apply -v /etc/puppet/manifests/site.pp')
tries = 1
if use_cobbler:
cobbler_error = "[cobbler-sync]/returns: unable to connect to cobbler on localhost using cobbler"
while cobbler_error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Cobbler is not installed properly, running apply again"
result = run_func(
'puppet apply -v /etc/puppet/manifests/site.pp',
pty=False)
tries += 1
error = "Error:"
while error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Some errors found, running apply again"
result = run_func(
'puppet apply -v /etc/puppet/manifests/site.pp',
pty=False)
tries += 1
if exists('/root/openrc'):
get('/root/openrc', "./openrc")
else:
print(red("No openrc file, something went wrong! :("))
print(green("Copying logs and configs"))
collect_logs(
run_func=run_func,
hostname=config["servers"]["build-server"][0]["hostname"],
clean=True)
print(green("Finished!"))
return True
elif not force and prepare:
return True
print(green("Finished!"))
return True
def install_openstack(settings_dict,
envs=None,
verbose=None,
url_script=None,
prepare=False,
force=False,
config=None,
use_cobbler=False,
proxy=None):
"""
Install OS with COI on build server
:param settings_dict: settings dictionary for Fabric
:param envs: environment variables to inject when executing job
:param verbose: if to hide all output or print everything
:param url_script: URl of Cisco installer script from Chris
:param force: Use if you don't connect via interface you gonna bridge later
:return: always true
"""
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
use_sudo_flag = True
run_func = sudo
else:
use_sudo_flag = False
run_func = run
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
with cd("/root/"):
if proxy:
warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
"/etc/apt/apt.conf.d/00proxy",
use_sudo=use_sudo_flag))
warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=use_sudo_flag))
run_func("apt-get update")
run_func("apt-get install -y git")
run_func("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'")
if not force and not prepare:
update_time(run_func)
# avoid grub and other prompts
warn_if_fail(run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
'-o Dpkg::Options::="--force-confdef" -o '
'Dpkg::Options::="--force-confold" dist-upgrade'))
# prepare /etc/hosts file
append("/etc/hosts", prepare_hosts(config))
with cd("/root"):
warn_if_fail(run_func("git clone -b icehouse "
"https://github.com/CiscoSystems/puppet_openstack_builder"))
with cd("puppet_openstack_builder"):
## run the latest, not i.0 release
#run_func('git checkout i.0')
sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
"icehouse/snapshots/i.0",
"icehouse-proposed", use_sudo=use_sudo_flag)
with cd("install-scripts"):
warn_if_fail(run_func("./install.sh"))
prepare_files(config,
paths=(
"/etc/puppet/data/hiera_data/user.common.yaml",
"/etc/puppet/data/hiera_data/user.full_ha.yaml",
"/etc/puppet/data/cobbler/cobbler.yaml",
"/etc/puppet/data/role_mappings.yaml",
"/etc/puppet/data/hiera_data/hostname/build_server.yaml"
),
use_sudo_flag=use_sudo_flag)
prepare_new_files(
config,
path="/etc/puppet/data/hiera_data/hostname",
use_sudo_flag=use_sudo_flag
)
resolve_names(run_func, use_sudo_flag)
result = run_func('puppet apply -v /etc/puppet/manifests/site.pp')
tries = 1
if use_cobbler:
cobbler_error = "[cobbler-sync]/returns: unable to connect to cobbler on localhost using cobbler"
while cobbler_error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Cobbler is not installed properly, running apply again"
result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False)
tries += 1
error = "Error:"
while error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Some errors found, running apply again"
result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False)
tries += 1
if exists('/root/openrc'):
get('/root/openrc', "./openrc")
else:
print (red("No openrc file, something went wrong! :("))
print (green("Copying logs and configs"))
collect_logs(run_func=run_func, hostname=config["servers"]["build-server"]["hostname"], clean=True)
print (green("Finished!"))
return True
elif not force and prepare:
return True
print (green("Finished!"))
return True
def install(aggregated_configs, apt_cacher_proxy=None,
patch="", proxy="", devstack_repo="", devstack_br="", quiet=False):
verbose = []
fab_settings = aggregated_configs[env.host]["fab_settings"]
local_conf = aggregated_configs[env.host]["devstack_config"]
config_files = aggregated_configs[env.host]["config_files"]
hostname = aggregated_configs[env.host]["hostname"]
exec_command_before = aggregated_configs[env.host]["exec_commands_before"]
exec_command_after = aggregated_configs[env.host]["exec_commands_after"]
if quiet:
verbose = ['output', 'running', 'warnings']
with settings(**fab_settings), hide(*verbose):
if exists("/etc/gai.conf"):
append("/etc/gai.conf", "precedence ::ffff:0:0/96 100",
use_sudo=True)
if proxy:
warn_if_fail(put(StringIO(
'Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
"/etc/apt/apt.conf.d/00proxy", use_sudo=True))
warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=True))
if apt_cacher_proxy:
warn_if_fail(
put(StringIO('Acquire::http { Proxy "{url}"; };'.format(
url=apt_cacher_proxy)),
"/etc/apt/apt.conf.d/02proxy", use_sudo=True))
update_time(sudo)
warn_if_fail(sudo("apt-get update"))
warn_if_fail(sudo("apt-get install -y git python-pip"))
warn_if_fail(
run("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'"))
if exists("devstack"):
with cd("devstack"):
warn_if_fail(run("./unstack.sh"))
run("rm -rf ~/devstack")
quit_if_fail(run(
"git clone -b {branch} {devstack}".format(devstack=devstack_repo,
branch=devstack_br)))
if patch:
with cd("devstack"):
warn_if_fail(
run(
"git fetch https://review.openstack.org/openstack-dev/devstack {patch} "
"&& git cherry-pick FETCH_HEAD".format(patch=patch)))
warn_if_fail(put(StringIO(local_conf), "devstack/local.conf"))
if exec_command_before:
run(exec_command_before)
if config_files:
for path, conf_file in config_files.iteritems():
warn_if_fail(put(StringIO(conf_file), path))
with cd("devstack"):
warn_if_fail(run("./stack.sh"))
path = os.environ.get("WORKSPACE", os.getcwd())
if os.path.exists(
"{path}/id_rsa_{key}".format(path=path, key=hostname)):
path = os.environ.get("WORKSPACE", os.getcwd())
put("{path}/id_rsa_{key}".format(path=path, key=hostname),
"~/.ssh/id_rsa")
put("{path}/id_rsa_{key}.pub".format(path=path, key=hostname),
"~/.ssh/id_rsa.pub")
put("{0}/id_rsa_all.pub".format(path), "/tmp/all_authorized")
warn_if_fail(run("chmod 500 ~/.ssh/id_rsa"))
warn_if_fail(
run("cat /tmp/all_authorized >> ~/.ssh/authorized_keys"))
append("/etc/ssh/ssh_config",
"\nStrictHostKeyChecking no\nUserKnownHostsFile=/dev/null",
use_sudo=True)
if exec_command_after:
run(exec_command_after)
collect_logs(run, hostname)
collect_logs_devstack("install_{host}".format(host=hostname))
