def configure_second_admin_firewall(ip, network, netmask, interface,
master_ip):
# Allow input/forwarding for nodes from the second admin network and
# enable source NAT for UDP (tftp) and HTTP (proxy server) traffic
# on master node
rules = [
('-I INPUT -i {0} -m comment --comment "input from 2nd admin network" '
'-j ACCEPT').format(interface),
('-t nat -I POSTROUTING -s {0}/{1} -o e+ -m comment --comment '
'"004 forward_admin_net2" -j MASQUERADE').format(network, netmask),
("-t nat -I POSTROUTING -o {0} -d {1}/{2} -p udp -m addrtype "
"--src-type LOCAL -j SNAT --to-source {3}").format(
interface, network, netmask, master_ip),
("-t nat -I POSTROUTING -d {0}/{1} -p tcp --dport 8888 -j SNAT "
"--to-source {2}").format(network, netmask, master_ip),
('-I FORWARD -i {0} -m comment --comment '
'"forward custom admin net" -j ACCEPT').format(interface)
]
for rule in rules:
cmd = 'iptables {0}'.format(rule)
result = SSHManager().execute(ip=ip, cmd=cmd)
assert_equal(result['exit_code'], 0,
('Failed to add firewall rule for second admin net '
'on master node: {0}, {1}').format(rule, result))
# Save new firewall configuration
cmd = 'service iptables save'
result = SSHManager().execute(ip=ip, cmd=cmd)
assert_equal(result['exit_code'], 0,
('Failed to save firewall configuration on master node:'
' {0}').format(result))
def get_oswl_services_names():
cmd = "systemctl list-units| grep oswl_ | awk '{print $1}'"
result = SSHManager().execute_on_remote(
SSHManager().admin_ip, cmd)['stdout_str'].strip()
logger.info('list of statistic services {0}'.format(
result.split('\n')))
return result.split('\n')
def get_nodes_tasks(node_id):
"""
:param node_id: an integer number of node id
:return: a set of deployment tasks for corresponding node
"""
tasks = set()
ssh = SSHManager()
result = ssh.execute_on_remote(ssh.admin_ip, "ls /var/log/astute")
filenames = [filename.strip() for filename in result['stdout']]
for filename in filenames:
ssh.download_from_remote(
ssh.admin_ip,
destination="/var/log/astute/{0}".format(filename),
target="/tmp/{0}".format(filename))
data = fileinput.FileInput(
files=["/tmp/{0}".format(filename) for filename in filenames],
openhook=fileinput.hook_compressed)
for line in data:
if "Task time summary" in line \
and "node {}".format(node_id) in line:
# FIXME: define an exact search of task
task_name = line.split("Task time summary: ")[1].split()[0]
check = any([excluded_task in task_name
for excluded_task in TASKS_BLACKLIST])
if check:
continue
tasks.add(task_name)
return tasks
def replace_rpm_package(package):
"""Replaced rpm package.rpm on master node with package.rpm
from review
"""
ssh = SSHManager()
logger.info("Patching {}".format(package))
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
try:
# Upload package
target_path = '/var/www/nailgun/{}/'.format(package)
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=target_path)
package_name = package
package_ext = '*.noarch.rpm'
pkg_path = os.path.join(target_path,
'{}{}'.format(package_name, package_ext))
full_package_name = get_full_filename(wildcard_name=pkg_path)
logger.debug('Package name is {0}'.format(full_package_name))
full_package_path = os.path.join(os.path.dirname(pkg_path),
full_package_name)
# Update package on master node
if not does_new_pkg_equal_to_installed_pkg(
installed_package=package_name,
new_package=full_package_path):
update_rpm(path=full_package_path)
except Exception:
logger.error("Could not upload package")
raise
def get_sha_sum(file_path):
logger.debug('Get md5 fo file {0}'.format(file_path))
md5_sum = SSHManager().execute_on_remote(
SSHManager().admin_ip,
cmd='md5sum {0}'.format(file_path))['stdout_str'].strip()
logger.info('MD5 is {0}'.format(md5_sum))
return md5_sum
def store_astute_yaml_for_one_node(nailgun_node):
ssh_manager = SSHManager()
if 'roles' not in nailgun_node:
return None
errmsg = 'Downloading "{0}.yaml" from the {1} failed'
msg = 'File "{0}.yaml" was downloaded from the {1}'
nodename = nailgun_node['name']
ip = nailgun_node['ip']
for role in nailgun_node['roles']:
filename = '{0}/{1}-{2}-{3}.yaml'.format(settings.LOGS_DIR,
func_name,
nodename,
role)
if not ssh_manager.isfile_on_remote(ip,
'/etc/{0}.yaml'.format(role)):
role = 'primary-' + role
if ssh_manager.download_from_remote(ip,
'/etc/{0}.yaml'.format(role),
filename):
logger.info(msg.format(role, nodename))
else:
logger.error(errmsg.format(role, nodename))
if settings.DOWNLOAD_FACTS:
fact_filename = re.sub(r'-\w*\.', '-facts.', filename)
generate_facts(ip)
if ssh_manager.download_from_remote(ip,
'/tmp/facts.yaml',
fact_filename):
logger.info(msg.format('facts', nodename))
else:
logger.error(errmsg.format('facts', nodename))
def replace_rpm_package(package):
"""Replaced rpm package.rpm on master node with package.rpm
from review
"""
ssh = SSHManager()
logger.info("Patching {}".format(package))
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
try:
# Upload package
target_path = '/var/www/nailgun/{}/'.format(package)
ssh.upload_to_remote(ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=target_path)
package_name = package
package_ext = '*.noarch.rpm'
pkg_path = os.path.join(target_path,
'{}{}'.format(package_name, package_ext))
full_package_name = get_full_filename(wildcard_name=pkg_path)
logger.debug('Package name is {0}'.format(full_package_name))
full_package_path = os.path.join(os.path.dirname(pkg_path),
full_package_name)
# Update package on master node
if not does_new_pkg_equal_to_installed_pkg(
installed_package=package_name, new_package=full_package_path):
update_rpm(path=full_package_path)
except Exception:
logger.error("Could not upload package")
raise
def install_pkg_2(ip, pkg_name, port=22):
"""Install a package <pkg_name> on node
:param ip: ip of node
:param pkg_name: name of a package
:param port: ssh port
:return: exit code of installation
"""
ssh_manager = SSHManager()
remote_status = ssh_manager.execute(
ip=ip,
port=port,
cmd="rpm -q '{0}'".format(pkg_name)
)
if remote_status['exit_code'] == 0:
logger.info("Package '{0}' already installed.".format(pkg_name))
else:
logger.info("Installing package '{0}' ...".format(pkg_name))
remote_status = ssh_manager.execute(
ip=ip,
port=port,
cmd="yum -y install {0}".format(pkg_name)
)
logger.info("Installation of the package '{0}' has been"
" completed with exit code {1}"
.format(pkg_name, remote_status['exit_code']))
return remote_status['exit_code']
def get_nodes_tasks(node_id):
"""
:param node_id: an integer number of node id
:return: a set of deployment tasks for corresponding node
"""
tasks = set()
ssh = SSHManager()
result = ssh.execute_on_remote(ssh.admin_ip, "ls /var/log/astute")
filenames = [filename.strip() for filename in result['stdout']]
for filename in filenames:
ssh.download_from_remote(
ssh.admin_ip,
destination="/var/log/astute/{0}".format(filename),
target="/tmp/{0}".format(filename))
data = fileinput.FileInput(
files=["/tmp/{0}".format(filename) for filename in filenames],
openhook=fileinput.hook_compressed)
for line in data:
if "Task time summary" in line \
and "node {}".format(node_id) in line:
# FIXME: define an exact search of task
task_name = line.split("Task time summary: ")[1].split()[0]
check = any([excluded_task in task_name
for excluded_task in TASKS_BLACKLIST])
if check:
continue
tasks.add(task_name)
return tasks
def __init__(self):
self.ssh_manager = SSHManager()
self.ip = self.ssh_manager.admin_ip
self.path_scripts = ('{0}/fuelweb_test/helpers/'
.format(os.environ.get("WORKSPACE", "./")))
self.remote_path_scripts = '/tmp/'
self.ubuntu_script = 'regenerate_ubuntu_repo'
self.centos_script = 'regenerate_centos_repo'
self.local_mirror_ubuntu = settings.LOCAL_MIRROR_UBUNTU
self.local_mirror_centos = settings.LOCAL_MIRROR_CENTOS
self.ubuntu_release = settings.UBUNTU_RELEASE
self.centos_supported_archs = ['noarch', 'x86_64']
self.pkgs_list = []
self.custom_pkgs_mirror_path = ''
if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE:
# Trying to determine the root of Ubuntu repository
pkgs_path = settings.CUSTOM_PKGS_MIRROR.split('/dists/')
if len(pkgs_path) == 2:
self.custom_pkgs_mirror = pkgs_path[0]
self.custom_pkgs_mirror_path = '/dists/{}'.format(pkgs_path[1])
else:
self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR
else:
self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR
def store_astute_yaml_for_one_node(nailgun_node):
ssh_manager = SSHManager()
if 'roles' not in nailgun_node:
return None
errmsg = 'Downloading "{0}.yaml" from the {1} failed'
msg = 'File "{0}.yaml" was downloaded from the {1}'
nodename = nailgun_node['name']
ip = nailgun_node['ip']
for role in nailgun_node['roles']:
filename = '{0}/{1}-{2}-{3}.yaml'.format(settings.LOGS_DIR,
func_name, nodename, role)
if not ssh_manager.isfile_on_remote(ip,
'/etc/{0}.yaml'.format(role)):
role = 'primary-' + role
if ssh_manager.download_from_remote(ip,
'/etc/{0}.yaml'.format(role),
filename):
logger.info(msg.format(role, nodename))
else:
logger.error(errmsg.format(role, nodename))
if settings.DOWNLOAD_FACTS:
fact_filename = re.sub(r'-\w*\.', '-facts.', filename)
generate_facts(ip)
if ssh_manager.download_from_remote(ip, '/tmp/facts.yaml',
fact_filename):
logger.info(msg.format('facts', nodename))
else:
logger.error(errmsg.format('facts', nodename))
def install_plugin_check_code(ip, plugin, exit_code=0):
# Moved from checkers.py for improvement of code
cmd = "cd /var && fuel plugins --install {0} ".format(plugin)
chan, _, stderr, _ = SSHManager().execute_async_on_remote(ip=ip, cmd=cmd)
logger.debug('Try to read status code from chain...')
assert_equal(
chan.recv_exit_status(), exit_code,
'Install script fails with next message {0}'.format(''.join(stderr)))
def _turnoff_executable_ruby(node):
"""Set mode -x for /usr/bin/ruby
:param node: dict, node attributes
"""
ssh = SSHManager()
cmd = 'chmod -x /usr/bin/ruby'
ssh.execute_on_remote(node['ip'], cmd)
def get_full_filename(wildcard_name):
cmd = 'ls {}'.format(wildcard_name)
logger.info("Getting full file name for: {}".format(wildcard_name))
full_pkg_name = SSHManager().execute_on_remote(ip=SSHManager().admin_ip,
cmd=cmd)['stdout_str']
return full_pkg_name
def _turnon_executable_ruby(node):
"""Set mode +x for /usr/bin/ruby
:param node: dict, node attributes
"""
ssh = SSHManager()
cmd = 'chmod +x /usr/bin/ruby'
ssh.execute_on_remote(node['ip'], cmd)
def _turnon_executable_ruby(node):
"""Set mode +x for /usr/bin/ruby
:param node: dict, node attributes
"""
ssh = SSHManager()
cmd = "chmod +x /usr/bin/ruby"
ssh.execute_on_remote(node["ip"], cmd)
def get_os_packages(ip, packages_pattern=None):
"""Pick names of some OS packages from node"""
if not packages_pattern:
packages_pattern = "neutron|nova|cinder|keystone|" "ceilometer|ironic|glance"
packages = SSHManager().execute_on_remote(
ip, "dpkg-query -W -f '${{package}}\\n' | grep -E '{}'".format(packages_pattern)
)["stdout_str"]
return packages.split("\n")
def get_os_packages(ip, packages_pattern=None):
"""Pick names of some OS packages from node"""
if not packages_pattern:
packages_pattern = "neutron|nova|cinder|keystone|" \
"ceilometer|ironic|glance"
packages = SSHManager().execute_on_remote(
ip, "dpkg-query -W -f '${{package}}\\n' | grep -E '{}'".format(
packages_pattern))['stdout_str']
return packages.split('\n')
def install_plugin_check_code(ip, plugin, exit_code=0):
# Moved from checkers.py for improvement of code
cmd = "cd /var && fuel plugins --install {0} ".format(plugin)
chan, _, stderr, _ = SSHManager().execute_async_on_remote(
ip=ip,
cmd=cmd
)
logger.debug('Try to read status code from chain...')
assert_equal(
chan.recv_exit_status(), exit_code,
'Install script fails with next message {0}'.format(''.join(stderr)))
def update_rpm(path, rpm_cmd='/bin/rpm -Uvh --force'):
cmd = '{rpm_cmd} {rpm_path}'\
.format(rpm_cmd=rpm_cmd, rpm_path=path)
logger.info("Updating rpm '{0}'".format(path))
try:
SSHManager().execute(SSHManager().admin_ip, cmd)
logger.info("Rpm '{0}' has been updated successfully ".format(path))
except Exception as ex:
logger.error("Could not update rpm '{0}' in the '{1}'".format(
path, ex))
raise
def delete_astute_log():
"""Delete astute.log file(s) on master node.
This is to ensure that no unwanted tasks are used by tests (e.g. from
previous deployments).
:return: None
"""
ssh = SSHManager()
ssh.execute_on_remote(ssh.admin_ip, "rm /var/log/astute/astute*")
ssh.execute_on_remote(ssh.admin_ip, "systemctl restart astute.service")
def hiera_json_out(node_ip, parameter):
hiera_cmd = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \
"Hiera.logger = 'noop'; " \
"puts JSON.dump(h.lookup(\'{0}\', " \
"[], {{}}, nil, nil))\"".format(parameter)
ssh_manager = SSHManager()
config = ssh_manager.execute_on_remote(
ip=node_ip,
cmd=hiera_cmd,
jsonify=True,
err_msg='Cannot get floating ranges')['stdout_json']
return config
def hiera_json_out(node_ip, parameter):
hiera_cmd = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \
"Hiera.logger = 'noop'; " \
"puts JSON.dump(h.lookup(\'{0}\', " \
"[], {{}}, nil, nil))\"".format(parameter)
ssh_manager = SSHManager()
config = ssh_manager.execute_on_remote(
ip=node_ip,
cmd=hiera_cmd,
jsonify=True,
err_msg='Cannot get floating ranges')['stdout_json']
return config
def check_package_version_injected_in_bootstraps(package,
cluster_id=None,
ironic=None):
ssh = SSHManager()
try:
pack_path = '/var/www/nailgun/{}/'.format(package)
ssh.upload_to_remote(ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=pack_path)
except Exception:
logger.exception("Could not upload package")
raise
# Step 1 - unpack active bootstrap
logger.info("unpack active bootstrap")
if ironic:
bootstrap = "/var/www/nailgun/bootstrap/ironic/{}".format(cluster_id)
else:
bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap"
bootstrap_var = "/var/root.squashfs"
cmd = "unsquashfs -d {} {}/root.squashfs".format(bootstrap_var, bootstrap)
ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
# Step 2 - check package version
logger.info("check package {} version injected in ubuntu bootstrap".format(
package))
cmd = "ls {}|grep {} |grep deb |cut -f 2 -d '_'".format(pack_path, package)
package_from_review = ssh.execute_on_remote(ip=ssh.admin_ip,
cmd=cmd)['stdout_str']
logger.info("package from review is {}".format(package_from_review))
awk_pattern = "awk '{print $2}'"
cmd = "chroot {}/ /bin/bash -c \"dpkg -s {}\"|grep Version|{}".format(
bootstrap_var, package, awk_pattern)
installed_package = ssh.execute_on_remote(ip=ssh.admin_ip,
cmd=cmd)['stdout_str']
logger.info("injected package is {}".format(installed_package))
assert_equal(
installed_package, package_from_review,
"The new package {0} wasn't injected in bootstrap".format(
package_from_review))
# Step 3 - remove unpacked bootstrap
cmd = "rm -rf {}".format(bootstrap_var)
ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
def get_mongo_partitions(ip, device):
# Moved from checkers.py for improvement of code
ret = SSHManager().check_call(
ip=ip,
cmd="lsblk | grep {device} | awk {size}".format(
device=device, size=re.escape('{print $4}')))['stdout']
if not ret:
logger.error("Partition not present! {partitions}: ".format(
partitions=SSHManager().check_call(ip=ip,
cmd="parted {device} print")))
raise Exception()
logger.debug("Partitions: {part}".format(part=ret))
return ret
def get_ceph_partitions(ip, device, fs_type="xfs"):
# Moved from checkers.py for improvement of code
ret = SSHManager().check_call(
ip=ip,
cmd="parted {device} print | grep {type}".format(
device=device, type=fs_type))['stdout']
if not ret:
logger.error("Partition not present! {partitions}: ".format(
partitions=SSHManager().check_call(ip=ip,
cmd="parted {device} print")))
raise Exception()
logger.debug("Partitions: {part}".format(part=ret))
return ret
class DockerActions(object):
"""DockerActions.""" # TODO documentation
def __init__(self):
self.ssh_manager = SSHManager()
def list_containers(self):
result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd='dockerctl list'
)
return result['stdout']
def wait_for_ready_containers(self, timeout=300):
if MASTER_IS_CENTOS7:
return
cont_actions = []
for container in self.list_containers():
cont_action = BaseActions()
cont_action.container = container
cont_actions.append(cont_action)
try:
wait(lambda: all([cont_action.is_container_ready
for cont_action in cont_actions]),
timeout=timeout)
except TimeoutError:
failed_containers = [x.container for x in cont_actions
if not x.is_container_ready]
raise TimeoutError(
"Container(s) {0} failed to start in {1} seconds."
.format(failed_containers, timeout))
def restart_container(self, container):
self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd='dockerctl restart {0}'.format(container)
)
cont_action = BaseActions()
cont_action.container = container
cont_action.wait_for_ready_container()
def restart_containers(self):
for container in self.list_containers():
self.restart_container(container)
def execute_in_containers(self, cmd):
for container in self.list_containers():
self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="dockerctl shell {0} bash -c '{1}'".format(container, cmd)
)
def check_service(ip, commands):
"""Check that required nova services are running on controller.
:param ip: ip address of node
:param commands: type list, nova commands to execute on controller,
example of commands:
['nova-manage service list | grep vcenter-vmcluster1']
"""
ssh_manager = SSHManager()
ssh_manager.check_call(ip=ip, command='source openrc')
for cmd in commands:
wait(lambda:
':-)' in ssh_manager.check_call(ip=ip, command=cmd).stdout[-1],
timeout=200)
def restart_service(service_name, timeout=30):
restart_cmd = 'service {} restart'.format(service_name)
get_status_cmd = 'service {} status'.format(service_name)
logger.info("Restarting service '{0}'".format(service_name))
try:
SSHManager().execute_on_remote(SSHManager().admin_ip, restart_cmd)
helpers.wait(lambda: 'running' in SSHManager().execute_on_remote(
SSHManager().admin_ip, get_status_cmd)['stdout_str'],
timeout=timeout)
logger.info("Service '{0}' has been restarted successfully ".format(
service_name))
except Exception as ex:
logger.error("Could not restart '{0}' service "
"in the '{1}'".format(service_name, ex))
raise
def mcollective_nodes_online(self):
nodes_uids = set([
str(node['id']) for node in
self.fuel_web.client.list_cluster_nodes(self.cluster_id)
])
ssh_manager = SSHManager()
out = ssh_manager.execute_on_remote(ip=ssh_manager.admin_ip,
cmd='mco find',
assert_ec_equal=[0,
1])['stdout_str']
ready_nodes_uids = set(out.split('\n'))
unavailable_nodes = nodes_uids - ready_nodes_uids
logger.debug('Nodes {0} are not reacheable via'
' mcollective'.format(unavailable_nodes))
return not unavailable_nodes
def check_service(ip, commands):
"""Check that required nova services are running on controller.
:param ip: ip address of node
:param commands: type list, nova commands to execute on controller,
example of commands:
['nova-manage service list | grep vcenter-vmcluster1'
"""
ssh_manager = SSHManager()
ssh_manager.execute_on_remote(ip=ip, cmd='source openrc')
for cmd in commands:
wait(lambda: ':-)' in ssh_manager.execute_on_remote(ip=ip, cmd=cmd)[
'stdout'][-1],
timeout=200)
class DockerActions(object):
"""DockerActions.""" # TODO documentation
def __init__(self):
self.ssh_manager = SSHManager()
def list_containers(self):
result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip,
cmd='dockerctl list')
return result['stdout']
def wait_for_ready_containers(self, timeout=300):
if MASTER_IS_CENTOS7:
return
cont_actions = []
for container in self.list_containers():
cont_action = BaseActions()
cont_action.container = container
cont_actions.append(cont_action)
try:
wait(lambda: all([
cont_action.is_container_ready for cont_action in cont_actions
]),
timeout=timeout)
except TimeoutError:
failed_containers = [
x.container for x in cont_actions if not x.is_container_ready
]
raise TimeoutError(
"Container(s) {0} failed to start in {1} seconds.".format(
failed_containers, timeout))
def restart_container(self, container):
self.ssh_manager.execute(ip=self.ssh_manager.admin_ip,
cmd='dockerctl restart {0}'.format(container))
cont_action = BaseActions()
cont_action.container = container
cont_action.wait_for_ready_container()
def restart_containers(self):
for container in self.list_containers():
self.restart_container(container)
def execute_in_containers(self, cmd):
for container in self.list_containers():
self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="dockerctl shell {0} bash -c '{1}'".format(container, cmd))
def check_emc_cinder_config(cls, ip, path):
command = 'cat {0}'.format(path)
conf_data = SSHManager().execute_on_remote(ip, command)['stdout_str']
conf_data = cStringIO(conf_data)
cinder_conf = configparser.ConfigParser()
cinder_conf.readfp(conf_data)
asserts.assert_equal(
cinder_conf.get('DEFAULT', 'volume_driver'),
'cinder.volume.drivers.emc.emc_cli_iscsi.EMCCLIISCSIDriver')
asserts.assert_equal(
cinder_conf.get('DEFAULT', 'storage_vnx_authentication_type'),
'global')
asserts.assert_false(
cinder_conf.getboolean('DEFAULT', 'destroy_empty_storage_group'))
asserts.assert_true(
cinder_conf.getboolean('DEFAULT', 'initiator_auto_registration'))
asserts.assert_equal(
cinder_conf.getint('DEFAULT', 'attach_detach_batch_interval'), -1)
asserts.assert_equal(cinder_conf.getint('DEFAULT', 'default_timeout'),
10)
asserts.assert_equal(cinder_conf.get('DEFAULT', 'naviseccli_path'),
'/opt/Navisphere/bin/naviseccli')
asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_ip'))
asserts.assert_true(
cinder_conf.has_option('DEFAULT', 'san_secondary_ip'))
asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_login'))
asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_password'))
def mcollective_nodes_online(self):
nodes_uids = set(
[str(node['id']) for node in
self.fuel_web.client.list_cluster_nodes(self.cluster_id)]
)
ssh_manager = SSHManager()
out = ssh_manager.execute_on_remote(
ip=ssh_manager.admin_ip,
cmd='mco find',
assert_ec_equal=[0, 1]
)['stdout_str']
ready_nodes_uids = set(out.split('\n'))
unavailable_nodes = nodes_uids - ready_nodes_uids
logger.debug('Nodes {0} are not reacheable via'
' mcollective'.format(unavailable_nodes))
return not unavailable_nodes
def upload_nailgun_agent_rpm():
"""Upload nailgun_agent.rpm on master node
"""
ssh = SSHManager()
logger.info("Upload nailgun-agent")
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
pack_path = '/var/www/nailgun/nailgun-agent-review/'
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=pack_path)
# Extract rpm context
cmd = 'cd {0}; rpm2cpio {1} | cpio -idmv'.format(
pack_path, 'nailgun-agent-*.noarch.rpm ')
ssh.execute_on_remote(ssh.admin_ip, cmd)
def get_puppet_report(node):
"""Get puppet run report from corresponding node
:param node: a dictionary with node description
:return: a dictionary with puppet report data
"""
ssh = SSHManager()
ip = node['ip']
report_file = "/var/lib/puppet/state/last_run_report.yaml"
asserts.assert_true(ssh.isfile_on_remote(ip, report_file),
'File {!r} not found on node {!r}'
.format(report_file, node['id']))
with ssh.open_on_remote(ip, report_file) as f:
data = yaml.load(f)
ssh.rm_rf_on_remote(ip, report_file)
return data
def check_emc_cinder_config(cls, ip, path):
with SSHManager().open_on_remote(ip=ip, path=path) as f:
cinder_conf = configparser.ConfigParser()
cinder_conf.readfp(f)
asserts.assert_equal(
cinder_conf.get('DEFAULT', 'volume_driver'),
'cinder.volume.drivers.emc.emc_cli_iscsi.EMCCLIISCSIDriver')
asserts.assert_equal(
cinder_conf.get('DEFAULT', 'storage_vnx_authentication_type'),
'global')
asserts.assert_false(
cinder_conf.getboolean('DEFAULT', 'destroy_empty_storage_group'))
asserts.assert_true(
cinder_conf.getboolean('DEFAULT', 'initiator_auto_registration'))
asserts.assert_equal(
cinder_conf.getint('DEFAULT', 'attach_detach_batch_interval'), -1)
asserts.assert_equal(cinder_conf.getint('DEFAULT', 'default_timeout'),
10)
asserts.assert_equal(cinder_conf.get('DEFAULT', 'naviseccli_path'),
'/opt/Navisphere/bin/naviseccli')
asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_ip'))
asserts.assert_true(
cinder_conf.has_option('DEFAULT', 'san_secondary_ip'))
asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_login'))
asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_password'))
def connect_slaves_to_repo(environment, nodes, repo_name):
repo_ip = environment.get_admin_node_ip()
repo_port = '8080'
repourl = 'http://{master_ip}:{repo_port}/{repo_name}/'.format(
master_ip=repo_ip, repo_name=repo_name, repo_port=repo_port)
if settings.OPENSTACK_RELEASE == settings.OPENSTACK_RELEASE_UBUNTU:
cmds = [
"echo -e '\ndeb {repourl} /' > /etc/apt/sources.list.d/{repo_name}"
".list".format(repourl=repourl, repo_name=repo_name),
"apt-key add <(curl -s '{repourl}/Release.key') || :".format(
repourl=repourl),
# Set highest priority to all repositories located on master node
"echo -e 'Package: *\nPin: origin {0}\nPin-Priority: 1060' > "
"/etc/apt/preferences.d/custom_repo".format(
environment.get_admin_node_ip()),
"apt-get update"
]
else:
cmds = [
"yum-config-manager --add-repo {url}".format(url=repourl),
"echo -e 'gpgcheck=0\npriority=20' >>/etc/yum.repos.d/{ip}_{port}_"
"{repo}_.repo".format(ip=repo_ip, repo=repo_name, port=repo_port),
"yum -y clean all",
]
for slave in nodes:
for cmd in cmds:
SSHManager().execute_on_remote(ip=slave['ip'], cmd=cmd)
def __init__(self):
self.ssh_manager = SSHManager()
self.ip = self.ssh_manager.admin_ip
self.path_scripts = ('{0}/fuelweb_test/helpers/'
.format(os.environ.get("WORKSPACE", "./")))
self.remote_path_scripts = '/tmp/'
self.ubuntu_script = 'regenerate_ubuntu_repo'
self.centos_script = 'regenerate_centos_repo'
self.local_mirror_ubuntu = settings.LOCAL_MIRROR_UBUNTU
self.local_mirror_centos = settings.LOCAL_MIRROR_CENTOS
self.ubuntu_release = settings.UBUNTU_RELEASE
self.centos_supported_archs = ['noarch', 'x86_64']
self.pkgs_list = []
self.custom_pkgs_mirror_path = ''
if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE:
# Trying to determine the root of Ubuntu repository
pkgs_path = settings.CUSTOM_PKGS_MIRROR.split('/dists/')
if len(pkgs_path) == 2:
self.custom_pkgs_mirror = pkgs_path[0]
self.custom_pkgs_mirror_path = '/dists/{}'.format(pkgs_path[1])
else:
self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR
else:
self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR
def configure_second_admin_dhcp(ip, interface):
dhcp_conf_file = '/etc/cobbler/dnsmasq.template'
cmd = ("sed '0,/^interface.*/s//\\0\\ninterface={0}/' -i {1};"
"cobbler sync").format(interface, dhcp_conf_file)
result = SSHManager().execute(ip=ip, cmd=cmd)
assert_equal(result['exit_code'], 0,
('Failed to add second admin '
'network to DHCP server: {0}').format(result))
def check_cinder_vmware_srv(self):
"""Verify cinder-vmware service."""
ctrl_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
self.cluster_id, ["controller"])
cmd = '. openrc; cinder-manage service list | grep vcenter | ' \
'grep ":-)"'
logger.debug('CMD: {}'.format(cmd))
SSHManager().execute_on_remote(ctrl_nodes[0]['ip'], cmd)
def get_file_size(ip, file_name, file_path):
# Moved from checkers.py for improvement of code
file_size = SSHManager().execute(
ip, 'stat -c "%s" {0}/{1}'.format(file_path, file_name))
assert_equal(
int(file_size['exit_code']), 0, "Failed to get '{0}/{1}' file stats on"
" remote node".format(file_path, file_name))
return int(file_size['stdout'][0].rstrip())
def __init__(self, config=None):
if not hasattr(self, "_virt_env"):
self._virt_env = None
if not hasattr(self, "_fuel_web"):
self._fuel_web = None
self._config = config
self.ssh_manager = SSHManager()
self.ssh_manager.initialize(
self.get_admin_node_ip(),
login=settings.SSH_CREDENTIALS['login'],
password=settings.SSH_CREDENTIALS['password'])
self.admin_actions = AdminActions()
self.base_actions = BaseActions()
self.cobbler_actions = CobblerActions()
self.nailgun_actions = NailgunActions()
self.postgres_actions = PostgresActions()
self.fuel_bootstrap_actions = FuelBootstrapCliActions()
def parse_pcs_status_xml(remote_ip):
"""Parse 'pcs status xml'. <Nodes> section
:param remote_ip: remote IP address
:return: nested dictionary with node-fqdn and attribute name as keys
"""
pcs_status_dict = SSHManager().execute_on_remote(
remote_ip, 'pcs status xml')['stdout_str']
return pcs_status_dict
def patch_centos_bootstrap():
"""Replaced initramfs.img in /var/www/nailgun/
with newly_builded from review
environment - Environment Model object - self.env
"""
logger.info("Update fuel-agent code and assemble new bootstrap")
ssh = SSHManager()
if not settings.UPDATE_FUEL:
raise Exception("{} variable don't exist"
.format(settings.UPDATE_FUEL))
try:
pack_path = '/var/www/nailgun/fuel-agent-review/'
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.FUEL_AGENT_REPO_PATH.rstrip('/'),
target=pack_path)
# Step 1 - unpack bootstrap
bootstrap_var = "/var/initramfs"
bootstrap = "/var/www/nailgun/bootstrap"
cmd = ("mkdir {0}; cp /{1}/initramfs.img {0}/; cd {0}; "
"cat initramfs.img | gunzip | cpio -imudv;").format(
bootstrap_var, bootstrap)
result = ssh.execute_on_remote(
ip=ssh.admin_ip, cmd=cmd)['stdout_str']
logger.debug("Patching bootsrap finishes with {0}".format(result))
# Step 2 - replace fuel-agent code in unpacked bootstrap
agent_path = "/usr/lib/python2.7/site-packages/fuel_agent"
image_rebuild = "{} | {} | {}".format(
"find . -xdev",
"cpio --create --format='newc'",
"gzip -9 > /var/initramfs.img.updated")
cmd = ("rm -rf {0}/initramfs.img; "
"rsync -r {2}fuel_agent/* {0}{1}/;"
"cd {0}/;"
"{3};").format(bootstrap_var, agent_path, pack_path,
image_rebuild)
result = ssh.execute_on_remote(
ip=ssh.admin_ip, cmd=cmd)['stdout_str']
logger.debug("Failed to rebuild image with {0}".format(result))
except Exception as e:
logger.error("Could not upload package {e}".format(e=e))
raise
def check_package_origin(ip, package, origin):
"""Check that given package was installed from given repository"""
version_cmd = ("apt-cache policy {package} | "
"awk '$1 == \"Installed:\" {{print $2}}'").format(
package=package)
version = SSHManager().execute_on_remote(ip, version_cmd)['stdout_str']
origin_cmd = ("apt-cache madison {package} | "
"grep '{version}'").format(package=package,
version=version)
result = SSHManager().execute_on_remote(ip, origin_cmd)['stdout']
# we only want to check for the UCA uri because it might be in main
# or proposed
repos = [str.strip(line.split("|")[2]) for line in result]
assert_true(
any([origin in repo for repo in repos]),
"Package {!r}: repository {!r} not found in {!r}".format(
package, origin, repos)
)
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600):
"""Check Swift ring and rebalance it if needed.
Replication should be performed on primary controller node.
Retry check several times. Wait for replication due to LP1498368.
"""
ssh = SSHManager()
cmd = "/usr/local/bin/swift-rings-rebalance.sh"
logger.debug('Check swift ring and rebalance it.')
for _ in xrange(retry_count):
try:
checkers.check_swift_ring(controller_ip)
break
except AssertionError:
result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd)
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(controller_ip)
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600):
"""Check Swift ring and rebalance it if needed.
Replication should be performed on primary controller node.
Retry check several times. Wait for replication due to LP1498368.
"""
ssh = SSHManager()
cmd = "/usr/local/bin/swift-rings-rebalance.sh"
logger.debug('Check swift ring and rebalance it.')
for _ in xrange(retry_count):
try:
checkers.check_swift_ring(controller_ip)
break
except AssertionError:
result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd)
logger.debug("command execution result is {0}".format(result))
else:
checkers.check_swift_ring(controller_ip)
class BaseActions(object):
"""BaseActions.""" # TODO documentation
def __init__(self):
self.ssh_manager = SSHManager()
self.admin_ip = self.ssh_manager.admin_ip
def __repr__(self):
klass, obj_id = type(self), hex(id(self))
return "[{klass}({obj_id})]".format(
klass=klass,
obj_id=obj_id)
def restart_service(self, service):
self.ssh_manager.execute_on_remote(
ip=self.admin_ip,
cmd="systemctl restart {0}".format(service),
err_msg="Failed to restart service {!r}, please inspect logs for "
"details".format(service))
def check_ping(ip, host, deadline=10, size=56, timeout=1, interval=1):
"""Check network connectivity from remote to host using ICMP (ping)
:param ip: remote ip
:param host: string IP address or host/domain name
:param deadline: time in seconds before ping exits
:param size: size of data to be sent
:param timeout: time to wait for a response, in seconds
:param interval: wait interval seconds between sending each packet
:return: bool: True if ping command
"""
ssh_manager = SSHManager()
cmd = ("ping -W {timeout} -i {interval} -s {size} -c 1 -w {deadline} "
"{host}".format(host=host,
size=size,
timeout=timeout,
interval=interval,
deadline=deadline))
res = ssh_manager.execute(ip, cmd)
return int(res['exit_code']) == 0
class BaseActions(object):
"""BaseActions.""" # TODO documentation
def __init__(self):
self.ssh_manager = SSHManager()
self.admin_ip = self.ssh_manager.admin_ip
def __repr__(self):
klass, obj_id = type(self), hex(id(self))
return "[{klass}({obj_id})]".format(
klass=klass,
obj_id=obj_id)
def restart_service(self, service):
self.ssh_manager.execute_on_remote(
ip=self.admin_ip,
cmd="systemctl restart {0}".format(service),
err_msg="Failed to restart service {!r}, please inspect logs for "
"details".format(service))
def replace_centos_bootstrap(environment):
"""Replaced initramfs.img in /var/www/nailgun/
with re-builded with review code
environment - Environment Model object - self.env
"""
logger.info("Updating bootstrap")
ssh = SSHManager()
if not settings.UPDATE_FUEL:
raise Exception("{} variable don't exist"
.format(settings.UPDATE_FUEL))
rebuilded_bootstrap = '/var/initramfs.img.updated'
with environment.d_env.get_admin_remote() as remote:
checkers.check_file_exists(
remote,
'{0}'.format(rebuilded_bootstrap))
logger.info("Assigning new bootstrap from {}".format(rebuilded_bootstrap))
bootstrap = "/var/www/nailgun/bootstrap"
cmd = ("mv {0}/initramfs.img /var/initramfs.img;"
"cp /var/initramfs.img.updated {0}/initramfs.img;"
"chmod +r {0}/initramfs.img;").format(bootstrap)
ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
cmd = "cobbler sync"
ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
def __init__(self, config=None):
if not hasattr(self, "_virt_env"):
self._virt_env = None
if not hasattr(self, "_fuel_web"):
self._fuel_web = None
self._config = config
self.ssh_manager = SSHManager()
self.ssh_manager.initialize(
self.get_admin_node_ip(),
login=settings.SSH_CREDENTIALS["login"],
password=settings.SSH_CREDENTIALS["password"],
)
self.admin_actions = AdminActions()
self.base_actions = BaseActions()
self.cobbler_actions = CobblerActions()
self.nailgun_actions = NailgunActions()
self.postgres_actions = PostgresActions()
self.fuel_bootstrap_actions = FuelBootstrapCliActions()
def replace_fuel_agent_rpm():
"""Replaced fuel_agent.rpm on master node with fuel_agent.rpm
from review
"""
ssh = SSHManager()
logger.info("Patching fuel-agent")
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
try:
pack_path = '/var/www/nailgun/fuel-agent/'
full_pack_path = os.path.join(pack_path, 'fuel-agent*.noarch.rpm')
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=pack_path)
# Update fuel-agent on master node
cmd = "rpm -q fuel-agent"
old_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str']
cmd = "rpm -qp {0}".format(full_pack_path)
new_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str']
logger.info("Updating package {0} with {1}"
.format(old_package, new_package))
if old_package != new_package:
logger.info("Updating fuel-agent package on master node")
logger.info('Try to install package {0}'.format(
new_package))
cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path)
ssh.execute_on_remote(ssh.admin_ip, cmd)
cmd = "rpm -q fuel-agent"
installed_package = ssh.execute_on_remote(
ssh.admin_ip, cmd)['stdout_str']
assert_equal(installed_package, new_package,
"The new package {0} was not installed".
format(new_package))
except Exception as e:
logger.error("Could not upload package {e}".format(e=e))
raise
class EnvironmentModel(object):
"""EnvironmentModel.""" # TODO documentation
def __init__(self, config=None):
if not hasattr(self, "_virt_env"):
self._virt_env = None
if not hasattr(self, "_fuel_web"):
self._fuel_web = None
self._config = config
self.ssh_manager = SSHManager()
self.ssh_manager.initialize(
self.get_admin_node_ip(),
admin_login=settings.SSH_FUEL_CREDENTIALS['login'],
admin_password=settings.SSH_FUEL_CREDENTIALS['password'],
slave_login=settings.SSH_SLAVE_CREDENTIALS['login'],
slave_password=settings.SSH_SLAVE_CREDENTIALS['password']
)
self.admin_actions = AdminActions()
self.base_actions = BaseActions()
self.cobbler_actions = CobblerActions()
self.nailgun_actions = NailgunActions()
self.postgres_actions = PostgresActions()
self.fuel_bootstrap_actions = FuelBootstrapCliActions()
@property
def fuel_web(self):
if self._fuel_web is None:
self._fuel_web = FuelWebClient(self)
return self._fuel_web
def __repr__(self):
klass, obj_id = type(self), hex(id(self))
if getattr(self, '_fuel_web'):
ip = self.fuel_web.admin_node_ip
else:
ip = None
return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass,
obj_id=obj_id,
ip=ip)
@property
def admin_node_ip(self):
return self.fuel_web.admin_node_ip
@property
def collector(self):
return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json')
@logwrap
def add_syslog_server(self, cluster_id, port=5514):
self.fuel_web.add_syslog_server(
cluster_id, self.d_env.router(), port)
def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT,
skip_timesync=False):
"""Lists registered nailgun nodes
Start vms and wait until they are registered on nailgun.
:rtype : List of registered nailgun nodes
"""
# self.dhcrelay_check()
for node in devops_nodes:
logger.info("Bootstrapping node: {}".format(node.name))
node.start()
# TODO(aglarendil): LP#1317213 temporary sleep
# remove after better fix is applied
time.sleep(5)
with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"):
wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout)
if not skip_timesync:
self.sync_time()
return self.nailgun_nodes(devops_nodes)
def sync_time(self, nodes_names=None, skip_sync=False):
if nodes_names is None:
roles = ['fuel_master', 'fuel_slave']
nodes_names = [node.name for node in self.d_env.get_nodes()
if node.role in roles and
node.driver.node_active(node)]
logger.info("Please wait while time on nodes: {0} "
"will be synchronized"
.format(', '.join(sorted(nodes_names))))
new_time = sync_time(self.d_env, nodes_names, skip_sync)
for name in sorted(new_time):
logger.info("New time on '{0}' = {1}".format(name, new_time[name]))
@logwrap
def get_admin_node_ip(self):
return str(
self.d_env.nodes(
).admin.get_ip_address_by_network_name(
self.d_env.admin_net))
@logwrap
def get_ebtables(self, cluster_id, devops_nodes):
return Ebtables(self.get_target_devs(devops_nodes),
self.fuel_web.client.get_cluster_vlans(cluster_id))
def get_keys(self, node, custom=None, build_images=None,
iso_connect_as='cdrom'):
params = {
'device_label': settings.ISO_LABEL,
'iface': iface_alias('eth0'),
'ip': node.get_ip_address_by_network_name(
self.d_env.admin_net),
'mask': self.d_env.get_network(
name=self.d_env.admin_net).ip.netmask,
'gw': self.d_env.router(),
'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME,
settings.DNS_SUFFIX)),
'nat_interface': self.d_env.nat_interface,
'nameserver': settings.DNS,
'showmenu': 'yes' if settings.SHOW_FUELMENU else 'no',
'wait_for_external_config': 'yes',
'build_images': '1' if build_images else '0',
'MASTER_NODE_EXTRA_PACKAGES': settings.MASTER_NODE_EXTRA_PACKAGES
}
# TODO(akostrikov) add tests for menu items/kernel parameters
# TODO(akostrikov) refactor it.
if iso_connect_as == 'usb':
keys = (
"<Wait>\n" # USB boot uses boot_menu=yes for master node
"<F12>\n"
"2\n"
)
else: # cdrom is default
keys = (
"<Wait>\n"
"<Wait>\n"
"<Wait>\n"
)
keys += (
"<Esc>\n"
"<Wait>\n"
"vmlinuz initrd=initrd.img"
" inst.ks=cdrom:LABEL=%(device_label)s:/ks.cfg"
" inst.repo=cdrom:LABEL=%(device_label)s:/"
" ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s"
":%(iface)s:off::: nameserver=%(nameserver)s"
" showmenu=%(showmenu)s\n"
" wait_for_external_config=%(wait_for_external_config)s"
" build_images=%(build_images)s\n"
" MASTER_NODE_EXTRA_PACKAGES='%(MASTER_NODE_EXTRA_PACKAGES)s'\n"
" <Enter>\n"
) % params
return keys
@staticmethod
def get_target_devs(devops_nodes):
return [
interface.target_dev for interface in [
val for var in map(lambda node: node.interfaces, devops_nodes)
for val in var]]
@property
def d_env(self):
if self._virt_env is None:
if not self._config:
try:
return Environment.get(name=settings.ENV_NAME)
except Exception:
self._virt_env = Environment.describe_environment(
boot_from=settings.ADMIN_BOOT_DEVICE)
self._virt_env.define()
else:
try:
return Environment.get(name=self._config[
'template']['devops_settings']['env_name'])
except Exception:
self._virt_env = Environment.create_environment(
full_config=self._config)
self._virt_env.define()
return self._virt_env
def resume_environment(self):
self.d_env.resume()
admin = self.d_env.nodes().admin
self.ssh_manager.clean_all_connections()
try:
admin.await(self.d_env.admin_net, timeout=30, by_port=8000)
except Exception as e:
logger.warning("From first time admin isn't reverted: "
"{0}".format(e))
admin.destroy()
logger.info('Admin node was destroyed. Wait 10 sec.')
time.sleep(10)
admin.start()
logger.info('Admin node started second time.')
self.d_env.nodes().admin.await(self.d_env.admin_net)
self.set_admin_ssh_password()
self.admin_actions.wait_for_fuel_ready(timeout=600)
# set collector address in case of admin node destroy
if settings.FUEL_STATS_ENABLED:
self.nailgun_actions.set_collector_address(
settings.FUEL_STATS_HOST,
settings.FUEL_STATS_PORT,
settings.FUEL_STATS_SSL)
# Restart statsenderd in order to apply new collector address
self.nailgun_actions.force_fuel_stats_sending()
self.fuel_web.client.send_fuel_stats(enabled=True)
logger.info('Enabled sending of statistics to {0}:{1}'.format(
settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT
))
self.set_admin_ssh_password()
self.admin_actions.wait_for_fuel_ready()
def make_snapshot(self, snapshot_name, description="", is_make=False):
if settings.MAKE_SNAPSHOT or is_make:
self.d_env.suspend()
time.sleep(10)
self.d_env.snapshot(snapshot_name, force=True,
description=description)
revert_info(snapshot_name, self.get_admin_node_ip(), description)
if settings.FUEL_STATS_CHECK:
self.resume_environment()
def nailgun_nodes(self, devops_nodes):
return [self.fuel_web.get_nailgun_node_by_devops_node(node)
for node in devops_nodes]
def check_slaves_are_ready(self):
devops_nodes = [node for node in self.d_env.nodes().slaves
if node.driver.node_active(node)]
# Bug: 1455753
time.sleep(30)
for node in devops_nodes:
try:
wait(lambda:
self.fuel_web.get_nailgun_node_by_devops_node(
node)['online'], timeout=60 * 6)
except TimeoutError:
raise TimeoutError(
"Node {0} does not become online".format(node.name))
return True
def revert_snapshot(self, name, skip_timesync=False,
skip_slaves_check=False):
if not self.d_env.has_snapshot(name):
return False
logger.info('We have snapshot with such name: {:s}'.format(name))
logger.info("Reverting the snapshot '{0}' ....".format(name))
self.d_env.revert(name)
logger.info("Resuming the snapshot '{0}' ....".format(name))
self.resume_environment()
if not skip_timesync:
self.sync_time()
try:
_wait(self.fuel_web.client.get_releases,
expected=EnvironmentError, timeout=300)
except exceptions.Unauthorized:
self.set_admin_keystone_password()
self.fuel_web.get_nailgun_version()
if not skip_slaves_check:
_wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6)
return True
def set_admin_ssh_password(self):
new_login = settings.SSH_FUEL_CREDENTIALS['login']
new_password = settings.SSH_FUEL_CREDENTIALS['password']
try:
self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd='date'
)
logger.debug('Accessing admin node using SSH: SUCCESS')
except Exception:
logger.debug('Accessing admin node using SSH credentials:'
' FAIL, trying to change password from default')
self.ssh_manager.initialize(
admin_ip=self.ssh_manager.admin_ip,
admin_login='******',
admin_password='******',
slave_login=settings.SSH_SLAVE_CREDENTIALS['login'],
slave_password=settings.SSH_SLAVE_CREDENTIALS['password']
)
self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login,
new_password)
)
self.ssh_manager.initialize(
admin_ip=self.ssh_manager.admin_ip,
admin_login=new_login,
admin_password=new_password,
slave_login=settings.SSH_SLAVE_CREDENTIALS['login'],
slave_password=settings.SSH_SLAVE_CREDENTIALS['password']
)
self.ssh_manager.update_connection(
ip=self.ssh_manager.admin_ip,
login=new_login,
password=new_password
)
logger.debug("Admin node password has changed.")
logger.info("Admin node login name: '{0}' , password: '******'".
format(new_login, new_password))
def set_admin_keystone_password(self):
try:
self.fuel_web.client.get_releases()
# TODO(akostrikov) CENTOS7 except exceptions.Unauthorized:
except:
self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd='fuel user --newpass {0} --change-password'.format(
settings.KEYSTONE_CREDS['password'])
)
logger.info(
'New Fuel UI (keystone) username: "******", password: "******"'
.format(settings.KEYSTONE_CREDS['username'],
settings.KEYSTONE_CREDS['password']))
def insert_cdrom_tray(self):
# This is very rude implementation and it SHOULD be changes after
# implementation this feature in fuel-devops
name = "{}_{}".format(settings.ENV_NAME, self.d_env.nodes().admin.name)
name_size = 80
if len(name) > name_size:
hash_str = str(hash(name))
name = (hash_str + name)[:name_size]
cmd = """EDITOR="sed -i s/tray=\\'open\\'//" virsh edit {}""".format(
name)
subprocess.check_call(cmd, shell=True)
def setup_environment(self, custom=settings.CUSTOM_ENV,
build_images=settings.BUILD_IMAGES,
iso_connect_as=settings.ADMIN_BOOT_DEVICE,
security=settings.SECURITY_TEST):
# Create environment and start the Fuel master node
admin = self.d_env.nodes().admin
self.d_env.start([admin])
logger.info("Waiting for admin node to start up")
wait(lambda: admin.driver.node_active(admin), 60)
logger.info("Proceed with installation")
# update network parameters at boot screen
admin.send_keys(self.get_keys(admin, custom=custom,
build_images=build_images,
iso_connect_as=iso_connect_as))
if settings.SHOW_FUELMENU:
self.wait_for_fuelmenu()
else:
self.wait_for_provisioning()
self.set_admin_ssh_password()
self.wait_for_external_config()
if custom:
self.setup_customisation()
if security:
nessus_node = NessusActions(self.d_env)
nessus_node.add_nessus_node()
# wait while installation complete
self.admin_actions.modify_configs(self.d_env.router())
self.kill_wait_for_external_config()
self.wait_bootstrap()
self.admin_actions.wait_for_fuel_ready()
@logwrap
def enable_force_https(self, admin_node_ip):
cmd = """
echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml
"""
self.ssh_manager.execute_on_remote(admin_node_ip, cmd)
cmd = "find / -name \"nginx_services.pp\""
puppet_manifest = \
self.ssh_manager.execute_on_remote(
admin_node_ip, cmd)['stdout'][0].strip()
cmd = 'puppet apply {0}'.format(puppet_manifest)
self.ssh_manager.execute_on_remote(admin_node_ip, cmd)
cmd = """
systemctl status nginx.service |
awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}'
"""
wait(lambda: (
self.ssh_manager.execute_on_remote(
admin_node_ip, cmd)['stdout'][0] != 'dead'), interval=10,
timeout=30)
# pylint: disable=no-self-use
@update_rpm_packages
@upload_manifests
def setup_customisation(self):
logger.info('Installing custom packages/manifests '
'before master node bootstrap...')
# pylint: enable=no-self-use
@logwrap
def wait_for_provisioning(self,
timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT):
_wait(lambda: _tcp_ping(
self.d_env.nodes(
).admin.get_ip_address_by_network_name
(self.d_env.admin_net), 22), timeout=timeout)
@logwrap
def wait_for_fuelmenu(self,
timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT):
def check_ssh_connection():
"""Try to close fuelmenu and check ssh connection"""
try:
_tcp_ping(
self.d_env.nodes(
).admin.get_ip_address_by_network_name
(self.d_env.admin_net), 22)
except Exception:
# send F8 trying to exit fuelmenu
self.d_env.nodes().admin.send_keys("<F8>\n")
return False
return True
wait(check_ssh_connection, interval=30, timeout=timeout,
timeout_msg="Fuelmenu hasn't appeared during allocated timeout")
@logwrap
def wait_for_external_config(self, timeout=120):
wait(lambda: self.ssh_manager.exists_on_remote(
self.ssh_manager.admin_ip,
'/var/lock/wait_for_external_config'),
timeout=600)
check_cmd = 'pkill -0 -f wait_for_external_config'
wait(
lambda: self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=check_cmd)['exit_code'] == 0, timeout=timeout)
@logwrap
def kill_wait_for_external_config(self):
kill_cmd = 'pkill -f "^wait_for_external_config"'
check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]'
self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd=kill_cmd
)
self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd=check_cmd
)
def wait_bootstrap(self):
logger.info("Waiting while bootstrapping is in progress")
log_path = "/var/log/puppet/bootstrap_admin_node.log"
logger.info("Running bootstrap (timeout: {0})".format(
float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT)))
with TimeStat("admin_node_bootsrap_time", is_uniq=True):
wait(
lambda: self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="grep 'Fuel node deployment' '{:s}'".format(log_path)
)['exit_code'] == 0,
timeout=(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))
)
result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="grep 'Fuel node deployment "
"complete' '{:s}'".format(log_path))['exit_code']
if result != 0:
raise Exception('Fuel node deployment failed.')
self.bootstrap_image_check()
def dhcrelay_check(self):
# CentOS 7 is pretty stable with admin iface.
# TODO(akostrikov) refactor it.
iface = iface_alias('eth0')
command = "dhcpcheck discover " \
"--ifaces {iface} " \
"--repeat 3 " \
"--timeout 10".format(iface=iface)
out = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=command
)['stdout']
assert_true(self.get_admin_node_ip() in "".join(out),
"dhcpcheck doesn't discover master ip")
def bootstrap_image_check(self):
fuel_settings = self.admin_actions.get_fuel_settings()
if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu':
logger.warning('Default image for bootstrap '
'is not based on Ubuntu!')
return
bootstrap_images = self.ssh_manager.execute_on_remote(
ip=self.ssh_manager.admin_ip,
cmd='fuel-bootstrap --quiet list'
)['stdout']
assert_true(any('active' in line for line in bootstrap_images),
'Ubuntu bootstrap image wasn\'t built and activated! '
'See logs in /var/log/fuel-bootstrap-image-build.log '
'for details.')
def admin_install_pkg(self, pkg_name):
"""Install a package <pkg_name> on the admin node"""
remote_status = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="rpm -q {0}'".format(pkg_name)
)
if remote_status['exit_code'] == 0:
logger.info("Package '{0}' already installed.".format(pkg_name))
else:
logger.info("Installing package '{0}' ...".format(pkg_name))
remote_status = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="yum -y install {0}".format(pkg_name)
)
logger.info("Installation of the package '{0}' has been"
" completed with exit code {1}"
.format(pkg_name, remote_status['exit_code']))
return remote_status['exit_code']
def admin_run_service(self, service_name):
"""Start a service <service_name> on the admin node"""
self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="service {0} start".format(service_name)
)
remote_status = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="service {0} status".format(service_name)
)
if any('running...' in status for status in remote_status['stdout']):
logger.info("Service '{0}' is running".format(service_name))
else:
logger.info("Service '{0}' failed to start"
" with exit code {1} :\n{2}"
.format(service_name,
remote_status['exit_code'],
remote_status['stdout']))
# Execute yum updates
# If updates installed,
# then `bootstrap_admin_node.sh;`
def admin_install_updates(self):
logger.info('Searching for updates..')
update_command = 'yum clean expire-cache; yum update -y'
update_result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=update_command
)
logger.info('Result of "{1}" command on master node: '
'{0}'.format(update_result, update_command))
assert_equal(int(update_result['exit_code']), 0,
'Packages update failed, '
'inspect logs for details')
# Check if any packets were updated and update was successful
for str_line in update_result['stdout']:
match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package",
str_line)
if match_updated_count:
updates_count = match_updated_count.group(1)
match_complete_message = re.search("(Complete!)", str_line)
match_no_updates = re.search("No Packages marked for Update",
str_line)
if (not match_updated_count or match_no_updates)\
and not match_complete_message:
logger.warning('No updates were found or update was incomplete.')
return
logger.info('{0} packet(s) were updated'.format(updates_count))
cmd = 'bootstrap_admin_node.sh;'
result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=cmd
)
logger.info('Result of "{1}" command on master node: '
'{0}'.format(result, cmd))
assert_equal(int(result['exit_code']), 0,
'bootstrap failed, '
'inspect logs for details')
# Modifies a resolv.conf on the Fuel master node and returns
# its original content.
# * adds 'nameservers' at start of resolv.conf if merge=True
# * replaces resolv.conf with 'nameservers' if merge=False
def modify_resolv_conf(self, nameservers=None, merge=True):
if nameservers is None:
nameservers = []
resolv_conf = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd='cat /etc/resolv.conf'
)
assert_equal(0, resolv_conf['exit_code'],
'Executing "{0}" on the admin node has failed with: {1}'
.format('cat /etc/resolv.conf', resolv_conf['stderr']))
if merge:
nameservers.extend(resolv_conf['stdout'])
resolv_keys = ['search', 'domain', 'nameserver']
resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
if any(x in ns for x in resolv_keys))
logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
echo_result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=echo_cmd
)
assert_equal(0, echo_result['exit_code'],
'Executing "{0}" on the admin node has failed with: {1}'
.format(echo_cmd, echo_result['stderr']))
return resolv_conf['stdout']
@staticmethod
@logwrap
def execute_remote_cmd(remote, cmd, exit_code=0):
result = remote.execute(cmd)
assert_equal(result['exit_code'], exit_code,
'Failed to execute "{0}" on remote host: {1}'.
format(cmd, result))
return result['stdout']
@logwrap
def describe_other_admin_interfaces(self, admin):
admin_networks = [iface.network.name for iface in admin.interfaces]
iface_name = None
for i, network_name in enumerate(admin_networks):
if 'admin' in network_name and 'admin' != network_name:
# This will be replaced with actual interface labels
# form fuel-devops
iface_name = 'enp0s' + str(i + 3)
logger.info("Describe Fuel admin node interface {0} for "
"network {1}".format(iface_name, network_name))
self.describe_admin_interface(iface_name, network_name)
if iface_name:
return self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd="cobbler sync")
@logwrap
def describe_admin_interface(self, admin_if, network_name):
admin_net_object = self.d_env.get_network(name=network_name)
admin_network = admin_net_object.ip.network
admin_netmask = admin_net_object.ip.netmask
admin_ip = str(self.d_env.nodes(
).admin.get_ip_address_by_network_name(network_name))
logger.info(('Parameters for admin interface configuration: '
'Network - {0}, Netmask - {1}, Interface - {2}, '
'IP Address - {3}').format(admin_network,
admin_netmask,
admin_if,
admin_ip))
add_admin_ip = ('DEVICE={0}\\n'
'ONBOOT=yes\\n'
'NM_CONTROLLED=no\\n'
'USERCTL=no\\n'
'PEERDNS=no\\n'
'BOOTPROTO=static\\n'
'IPADDR={1}\\n'
'NETMASK={2}\\n').format(admin_if,
admin_ip,
admin_netmask)
cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};'
'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format(
add_admin_ip, admin_if, admin_ip)
logger.debug('Trying to assign {0} IP to the {1} on master node...'.
format(admin_ip, admin_if))
result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=cmd
)
assert_equal(result['exit_code'], 0, ('Failed to assign second admin '
'IP address on master node: {0}').format(result))
logger.debug('Done: {0}'.format(result['stdout']))
# TODO for ssh manager
multiple_networks_hacks.configure_second_admin_dhcp(
self.ssh_manager.admin_ip,
admin_if
)
multiple_networks_hacks.configure_second_admin_firewall(
self.ssh_manager.admin_ip,
admin_network,
admin_netmask,
admin_if,
self.get_admin_node_ip()
)
@logwrap
def get_masternode_uuid(self):
return self.postgres_actions.run_query(
db='nailgun',
query="select master_node_uid from master_node_settings limit 1;")
def __init__(self):
self.ssh_manager = SSHManager()
self.admin_ip = self.ssh_manager.admin_ip
class BaseActions(object):
"""BaseActions.""" # TODO documentation
def __init__(self):
self.ssh_manager = SSHManager()
self.admin_ip = self.ssh_manager.admin_ip
def __repr__(self):
klass, obj_id = type(self), hex(id(self))
return "[{klass}({obj_id})]".format(
klass=klass,
obj_id=obj_id)
# TODO(kozhukalov): This method seems not needed and
# can easily be replaced by using execute_on_remote
# available in SSHManager (up to the type of return value)
def execute(self, cmd, exit_code=None, stdin=None):
if stdin is not None:
cmd = 'echo "{0}" | {1}'.format(stdin, cmd)
result = self.ssh_manager.execute(
ip=self.admin_ip,
cmd=cmd
)
if exit_code is not None:
assert_equal(exit_code,
result['exit_code'],
('Command {cmd} returned exit code "{e}", but '
'expected "{c}". Output: {out}; {err} ').format(
cmd=cmd,
e=result['exit_code'],
c=exit_code,
out=result['stdout'],
err=result['stderr']
))
return ''.join(result['stdout']).strip()
def restart_service(self, service):
result = self.ssh_manager(
ip=self.admin_ip,
cmd="systemctl restart {0}".format(service))
return result['exit_code'] == 0
def put_value_to_local_yaml(self, old_file, new_file, element, value):
"""Changes content in old_file at element is given to the new value
and creates new file with changed content
:param old_file: a path to the file content from to be changed
:param new_file: a path to the new file to ve created with new content
:param element: tuple with path to element to be changed
for example: ['root_elem', 'first_elem', 'target_elem']
if there are a few elements with equal names use integer
to identify which element should be used
:return: nothing
"""
with open(old_file, 'r') as f_old:
yaml_dict = yaml.load(f_old)
origin_yaml = yaml_dict
for k in element[:-1]:
yaml_dict = yaml_dict[k]
yaml_dict[element[-1]] = value
with open(new_file, 'w') as f_new:
yaml.dump(origin_yaml, f_new, default_flow_style=False,
default_style='"')
def get_value_from_local_yaml(self, yaml_file, element):
"""Get a value of the element from the local yaml file
:param str yaml_file: a path to the yaml file
:param list element:
list with path to element to be read
for example: ['root_elem', 'first_elem', 'target_elem']
if there are a few elements with equal names use integer
to identify which element should be used
:return obj: value
"""
with open(yaml_file, 'r') as f_old:
yaml_dict = yaml.load(f_old)
for i, k in enumerate(element):
try:
yaml_dict = yaml_dict[k]
except IndexError:
raise IndexError("Element {0} not found in the file {1}"
.format(element[: i + 1], f_old))
except KeyError:
raise KeyError("Element {0} not found in the file {1}"
.format(element[: i + 1], f_old))
return yaml_dict
def change_remote_yaml(self, path_to_file, element, value):
"""Changes values in the yaml file stored
There is no need to copy file manually
:param path_to_file: absolute path to the file
:param element: list with path to the element be changed
:param value: new value for element
:return: Nothing
"""
old_file = '/tmp/temp_file_{0}.old.yaml'.format(str(os.getpid()))
new_file = '/tmp/temp_file_{0}.new.yaml'.format(str(os.getpid()))
self.ssh_manager.download_from_remote(
ip=self.admin_ip,
destination=path_to_file,
target=old_file
)
self.put_value_to_local_yaml(old_file, new_file, element, value)
self.ssh_manager.upload_to_remote(
ip=self.admin_ip,
source=new_file,
target=path_to_file
)
os.remove(old_file)
os.remove(new_file)
def get_value_from_remote_yaml(self, path_to_file, element):
"""Get a value from the yaml file stored
on the master node
:param str path_to_file: absolute path to the file
:param list element: list with path to the element
:return obj: value
"""
host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
self.ssh_manager.download_from_remote(
ip=self.admin_ip,
destination=path_to_file,
target=host_tmp_file
)
value = self.get_value_from_local_yaml(host_tmp_file, element)
os.remove(host_tmp_file)
return value
def put_value_to_remote_yaml(self, path_to_file, element, value):
"""Put a value to the yaml file stored
on the master node
:param str path_to_file: absolute path to the file
:param list element: list with path to the element be changed
:param value: new value for element
:return: None
"""
host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid()))
self.ssh_manager.download_from_remote(
ip=self.admin_ip,
destination=path_to_file,
target=host_tmp_file
)
self.put_value_to_local_yaml(host_tmp_file, host_tmp_file,
element, value)
self.ssh_manager.upload_to_remote(
ip=self.admin_ip,
source=host_tmp_file,
target=path_to_file
)
os.remove(host_tmp_file)
def generate_facts(ip):
ssh_manager = SSHManager()
facter_dir = '/var/lib/puppet/lib/facter'
exluded_facts = ['naily.rb']
if not ssh_manager.isdir_on_remote(ip, facter_dir):
ssh_manager.mkdir_on_remote(ip, facter_dir)
logger.debug('Directory {0} was created'.format(facter_dir))
ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir))
logger.debug('rb files were removed from {0}'.format(facter_dir))
facts_files = ssh_manager.execute_on_remote(
ip,
'find /etc/puppet/modules/ -wholename "*/lib/facter/*.rb"')['stdout']
facts_files = [i.strip() for i in facts_files]
logger.debug('The following facts {0} will'
' be copied to {1}'.format(facts_files, facter_dir))
for fact in facts_files:
if not fact or re.sub(r'.*/', '', fact) in exluded_facts:
continue
ssh_manager.execute_on_remote(ip,
'cp {0} {1}/'.format(fact, facter_dir))
logger.debug('Facts were copied')
ssh_manager.execute_on_remote(ip, 'facter -p -y > /tmp/facts.yaml')
logger.info('Facts yaml was created')
ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir))
logger.debug('rb files were removed from {0}'.format(facter_dir))
def centos_setup_fuel(self, hostname):
logger.info("upload fuel-release packet")
if not settings.FUEL_RELEASE_PATH:
raise exceptions.FuelQAVariableNotSet('FUEL_RELEASE_PATH', '/path')
try:
ssh = SSHManager()
pack_path = '/tmp/'
full_pack_path = os.path.join(pack_path,
'fuel-release*.noarch.rpm')
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.FUEL_RELEASE_PATH.rstrip('/'),
target=pack_path)
except Exception:
logger.exception("Could not upload package")
logger.debug("Update host information")
cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "echo {0} {1} {2} >> /etc/hosts".format(
ssh.admin_ip,
hostname,
settings.FUEL_MASTER_HOSTNAME)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "hostname {}".format(hostname)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
logger.debug("setup MOS repositories")
cmd = "rpm -ivh {}".format(full_pack_path)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "yum install -y fuel-setup"
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "yum install -y screen"
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
logger.info("Install Fuel services")
cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \
"bootstrap_admin_node.sh'"
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
self.env.wait_for_external_config()
self.env.admin_actions.modify_configs(self.env.d_env.router())
self.env.kill_wait_for_external_config()
self.env.wait_bootstrap()
logger.debug("Check Fuel services")
self.env.admin_actions.wait_for_fuel_ready()
logger.debug("post-installation configuration of Fuel services")
self.fuel_post_install_actions()
def centos_setup_fuel(self, hostname):
with TimeStat("bootstrap_centos_node", is_uniq=True):
admin = list(self.env.d_env.get_nodes(role__contains='master'))[0]
self.env.d_env.start([admin])
logger.info("Waiting for Centos node to start up")
wait(lambda: admin.driver.node_active(admin), 60,
timeout_msg='Centos node failed to start')
logger.info("Waiting for Centos node ssh ready")
self.env.wait_for_provisioning()
ssh = SSHManager()
logger.debug("Update host information")
cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "echo {0} {1} {2} >> /etc/hosts".format(
ssh.admin_ip,
hostname,
settings.FUEL_MASTER_HOSTNAME)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "hostname {}".format(hostname)
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
cmd = "yum install -y screen"
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
install_mos_repos()
logger.info("Install Fuel services")
cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \
"bootstrap_admin_node.sh'"
ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
self.env.wait_for_external_config()
self.env.admin_actions.modify_configs(self.env.d_env.router())
if CUSTOM_FUEL_SETTING_YAML:
self.env.admin_actions.update_fuel_setting_yaml(
CUSTOM_FUEL_SETTING_YAML)
self.env.kill_wait_for_external_config()
self.env.wait_bootstrap()
logger.debug("Check Fuel services")
self.env.admin_actions.wait_for_fuel_ready()
logger.debug("post-installation configuration of Fuel services")
self.fuel_post_install_actions()
