def configure_second_admin_firewall(ip, network, netmask, interface, master_ip): # Allow input/forwarding for nodes from the second admin network and # enable source NAT for UDP (tftp) and HTTP (proxy server) traffic # on master node rules = [ ('-I INPUT -i {0} -m comment --comment "input from 2nd admin network" ' '-j ACCEPT').format(interface), ('-t nat -I POSTROUTING -s {0}/{1} -o e+ -m comment --comment ' '"004 forward_admin_net2" -j MASQUERADE').format(network, netmask), ("-t nat -I POSTROUTING -o {0} -d {1}/{2} -p udp -m addrtype " "--src-type LOCAL -j SNAT --to-source {3}").format( interface, network, netmask, master_ip), ("-t nat -I POSTROUTING -d {0}/{1} -p tcp --dport 8888 -j SNAT " "--to-source {2}").format(network, netmask, master_ip), ('-I FORWARD -i {0} -m comment --comment ' '"forward custom admin net" -j ACCEPT').format(interface) ] for rule in rules: cmd = 'iptables {0}'.format(rule) result = SSHManager().execute(ip=ip, cmd=cmd) assert_equal(result['exit_code'], 0, ('Failed to add firewall rule for second admin net ' 'on master node: {0}, {1}').format(rule, result)) # Save new firewall configuration cmd = 'service iptables save' result = SSHManager().execute(ip=ip, cmd=cmd) assert_equal(result['exit_code'], 0, ('Failed to save firewall configuration on master node:' ' {0}').format(result))
def get_oswl_services_names(): cmd = "systemctl list-units| grep oswl_ | awk '{print $1}'" result = SSHManager().execute_on_remote( SSHManager().admin_ip, cmd)['stdout_str'].strip() logger.info('list of statistic services {0}'.format( result.split('\n'))) return result.split('\n')
def get_nodes_tasks(node_id): """ :param node_id: an integer number of node id :return: a set of deployment tasks for corresponding node """ tasks = set() ssh = SSHManager() result = ssh.execute_on_remote(ssh.admin_ip, "ls /var/log/astute") filenames = [filename.strip() for filename in result['stdout']] for filename in filenames: ssh.download_from_remote( ssh.admin_ip, destination="/var/log/astute/{0}".format(filename), target="/tmp/{0}".format(filename)) data = fileinput.FileInput( files=["/tmp/{0}".format(filename) for filename in filenames], openhook=fileinput.hook_compressed) for line in data: if "Task time summary" in line \ and "node {}".format(node_id) in line: # FIXME: define an exact search of task task_name = line.split("Task time summary: ")[1].split()[0] check = any([excluded_task in task_name for excluded_task in TASKS_BLACKLIST]) if check: continue tasks.add(task_name) return tasks
def replace_rpm_package(package): """Replaced rpm package.rpm on master node with package.rpm from review """ ssh = SSHManager() logger.info("Patching {}".format(package)) if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') try: # Upload package target_path = '/var/www/nailgun/{}/'.format(package) ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=target_path) package_name = package package_ext = '*.noarch.rpm' pkg_path = os.path.join(target_path, '{}{}'.format(package_name, package_ext)) full_package_name = get_full_filename(wildcard_name=pkg_path) logger.debug('Package name is {0}'.format(full_package_name)) full_package_path = os.path.join(os.path.dirname(pkg_path), full_package_name) # Update package on master node if not does_new_pkg_equal_to_installed_pkg( installed_package=package_name, new_package=full_package_path): update_rpm(path=full_package_path) except Exception: logger.error("Could not upload package") raise
def get_sha_sum(file_path): logger.debug('Get md5 fo file {0}'.format(file_path)) md5_sum = SSHManager().execute_on_remote( SSHManager().admin_ip, cmd='md5sum {0}'.format(file_path))['stdout_str'].strip() logger.info('MD5 is {0}'.format(md5_sum)) return md5_sum
def store_astute_yaml_for_one_node(nailgun_node): ssh_manager = SSHManager() if 'roles' not in nailgun_node: return None errmsg = 'Downloading "{0}.yaml" from the {1} failed' msg = 'File "{0}.yaml" was downloaded from the {1}' nodename = nailgun_node['name'] ip = nailgun_node['ip'] for role in nailgun_node['roles']: filename = '{0}/{1}-{2}-{3}.yaml'.format(settings.LOGS_DIR, func_name, nodename, role) if not ssh_manager.isfile_on_remote(ip, '/etc/{0}.yaml'.format(role)): role = 'primary-' + role if ssh_manager.download_from_remote(ip, '/etc/{0}.yaml'.format(role), filename): logger.info(msg.format(role, nodename)) else: logger.error(errmsg.format(role, nodename)) if settings.DOWNLOAD_FACTS: fact_filename = re.sub(r'-\w*\.', '-facts.', filename) generate_facts(ip) if ssh_manager.download_from_remote(ip, '/tmp/facts.yaml', fact_filename): logger.info(msg.format('facts', nodename)) else: logger.error(errmsg.format('facts', nodename))
def replace_rpm_package(package): """Replaced rpm package.rpm on master node with package.rpm from review """ ssh = SSHManager() logger.info("Patching {}".format(package)) if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') try: # Upload package target_path = '/var/www/nailgun/{}/'.format(package) ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=target_path) package_name = package package_ext = '*.noarch.rpm' pkg_path = os.path.join(target_path, '{}{}'.format(package_name, package_ext)) full_package_name = get_full_filename(wildcard_name=pkg_path) logger.debug('Package name is {0}'.format(full_package_name)) full_package_path = os.path.join(os.path.dirname(pkg_path), full_package_name) # Update package on master node if not does_new_pkg_equal_to_installed_pkg( installed_package=package_name, new_package=full_package_path): update_rpm(path=full_package_path) except Exception: logger.error("Could not upload package") raise
def install_pkg_2(ip, pkg_name, port=22): """Install a package <pkg_name> on node :param ip: ip of node :param pkg_name: name of a package :param port: ssh port :return: exit code of installation """ ssh_manager = SSHManager() remote_status = ssh_manager.execute( ip=ip, port=port, cmd="rpm -q '{0}'".format(pkg_name) ) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = ssh_manager.execute( ip=ip, port=port, cmd="yum -y install {0}".format(pkg_name) ) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code']
def __init__(self): self.ssh_manager = SSHManager() self.ip = self.ssh_manager.admin_ip self.path_scripts = ('{0}/fuelweb_test/helpers/' .format(os.environ.get("WORKSPACE", "./"))) self.remote_path_scripts = '/tmp/' self.ubuntu_script = 'regenerate_ubuntu_repo' self.centos_script = 'regenerate_centos_repo' self.local_mirror_ubuntu = settings.LOCAL_MIRROR_UBUNTU self.local_mirror_centos = settings.LOCAL_MIRROR_CENTOS self.ubuntu_release = settings.UBUNTU_RELEASE self.centos_supported_archs = ['noarch', 'x86_64'] self.pkgs_list = [] self.custom_pkgs_mirror_path = '' if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: # Trying to determine the root of Ubuntu repository pkgs_path = settings.CUSTOM_PKGS_MIRROR.split('/dists/') if len(pkgs_path) == 2: self.custom_pkgs_mirror = pkgs_path[0] self.custom_pkgs_mirror_path = '/dists/{}'.format(pkgs_path[1]) else: self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR else: self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR
def install_plugin_check_code(ip, plugin, exit_code=0): # Moved from checkers.py for improvement of code cmd = "cd /var && fuel plugins --install {0} ".format(plugin) chan, _, stderr, _ = SSHManager().execute_async_on_remote(ip=ip, cmd=cmd) logger.debug('Try to read status code from chain...') assert_equal( chan.recv_exit_status(), exit_code, 'Install script fails with next message {0}'.format(''.join(stderr)))
def _turnoff_executable_ruby(node): """Set mode -x for /usr/bin/ruby :param node: dict, node attributes """ ssh = SSHManager() cmd = 'chmod -x /usr/bin/ruby' ssh.execute_on_remote(node['ip'], cmd)
def get_full_filename(wildcard_name): cmd = 'ls {}'.format(wildcard_name) logger.info("Getting full file name for: {}".format(wildcard_name)) full_pkg_name = SSHManager().execute_on_remote(ip=SSHManager().admin_ip, cmd=cmd)['stdout_str'] return full_pkg_name
def _turnon_executable_ruby(node): """Set mode +x for /usr/bin/ruby :param node: dict, node attributes """ ssh = SSHManager() cmd = 'chmod +x /usr/bin/ruby' ssh.execute_on_remote(node['ip'], cmd)
def _turnon_executable_ruby(node): """Set mode +x for /usr/bin/ruby :param node: dict, node attributes """ ssh = SSHManager() cmd = "chmod +x /usr/bin/ruby" ssh.execute_on_remote(node["ip"], cmd)
def get_os_packages(ip, packages_pattern=None): """Pick names of some OS packages from node""" if not packages_pattern: packages_pattern = "neutron|nova|cinder|keystone|" "ceilometer|ironic|glance" packages = SSHManager().execute_on_remote( ip, "dpkg-query -W -f '${{package}}\\n' | grep -E '{}'".format(packages_pattern) )["stdout_str"] return packages.split("\n")
def get_os_packages(ip, packages_pattern=None): """Pick names of some OS packages from node""" if not packages_pattern: packages_pattern = "neutron|nova|cinder|keystone|" \ "ceilometer|ironic|glance" packages = SSHManager().execute_on_remote( ip, "dpkg-query -W -f '${{package}}\\n' | grep -E '{}'".format( packages_pattern))['stdout_str'] return packages.split('\n')
def install_plugin_check_code(ip, plugin, exit_code=0): # Moved from checkers.py for improvement of code cmd = "cd /var && fuel plugins --install {0} ".format(plugin) chan, _, stderr, _ = SSHManager().execute_async_on_remote( ip=ip, cmd=cmd ) logger.debug('Try to read status code from chain...') assert_equal( chan.recv_exit_status(), exit_code, 'Install script fails with next message {0}'.format(''.join(stderr)))
def update_rpm(path, rpm_cmd='/bin/rpm -Uvh --force'): cmd = '{rpm_cmd} {rpm_path}'\ .format(rpm_cmd=rpm_cmd, rpm_path=path) logger.info("Updating rpm '{0}'".format(path)) try: SSHManager().execute(SSHManager().admin_ip, cmd) logger.info("Rpm '{0}' has been updated successfully ".format(path)) except Exception as ex: logger.error("Could not update rpm '{0}' in the '{1}'".format( path, ex)) raise
def delete_astute_log(): """Delete astute.log file(s) on master node. This is to ensure that no unwanted tasks are used by tests (e.g. from previous deployments). :return: None """ ssh = SSHManager() ssh.execute_on_remote(ssh.admin_ip, "rm /var/log/astute/astute*") ssh.execute_on_remote(ssh.admin_ip, "systemctl restart astute.service")
def hiera_json_out(node_ip, parameter): hiera_cmd = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \ "Hiera.logger = 'noop'; " \ "puts JSON.dump(h.lookup(\'{0}\', " \ "[], {{}}, nil, nil))\"".format(parameter) ssh_manager = SSHManager() config = ssh_manager.execute_on_remote( ip=node_ip, cmd=hiera_cmd, jsonify=True, err_msg='Cannot get floating ranges')['stdout_json'] return config
def check_package_version_injected_in_bootstraps(package, cluster_id=None, ironic=None): ssh = SSHManager() try: pack_path = '/var/www/nailgun/{}/'.format(package) ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) except Exception: logger.exception("Could not upload package") raise # Step 1 - unpack active bootstrap logger.info("unpack active bootstrap") if ironic: bootstrap = "/var/www/nailgun/bootstrap/ironic/{}".format(cluster_id) else: bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d {} {}/root.squashfs".format(bootstrap_var, bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 2 - check package version logger.info("check package {} version injected in ubuntu bootstrap".format( package)) cmd = "ls {}|grep {} |grep deb |cut -f 2 -d '_'".format(pack_path, package) package_from_review = ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("package from review is {}".format(package_from_review)) awk_pattern = "awk '{print $2}'" cmd = "chroot {}/ /bin/bash -c \"dpkg -s {}\"|grep Version|{}".format( bootstrap_var, package, awk_pattern) installed_package = ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("injected package is {}".format(installed_package)) assert_equal( installed_package, package_from_review, "The new package {0} wasn't injected in bootstrap".format( package_from_review)) # Step 3 - remove unpacked bootstrap cmd = "rm -rf {}".format(bootstrap_var) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
def get_mongo_partitions(ip, device): # Moved from checkers.py for improvement of code ret = SSHManager().check_call( ip=ip, cmd="lsblk | grep {device} | awk {size}".format( device=device, size=re.escape('{print $4}')))['stdout'] if not ret: logger.error("Partition not present! {partitions}: ".format( partitions=SSHManager().check_call(ip=ip, cmd="parted {device} print"))) raise Exception() logger.debug("Partitions: {part}".format(part=ret)) return ret
def get_ceph_partitions(ip, device, fs_type="xfs"): # Moved from checkers.py for improvement of code ret = SSHManager().check_call( ip=ip, cmd="parted {device} print | grep {type}".format( device=device, type=fs_type))['stdout'] if not ret: logger.error("Partition not present! {partitions}: ".format( partitions=SSHManager().check_call(ip=ip, cmd="parted {device} print"))) raise Exception() logger.debug("Partitions: {part}".format(part=ret)) return ret
class DockerActions(object): """DockerActions.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() def list_containers(self): result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd='dockerctl list' ) return result['stdout'] def wait_for_ready_containers(self, timeout=300): if MASTER_IS_CENTOS7: return cont_actions = [] for container in self.list_containers(): cont_action = BaseActions() cont_action.container = container cont_actions.append(cont_action) try: wait(lambda: all([cont_action.is_container_ready for cont_action in cont_actions]), timeout=timeout) except TimeoutError: failed_containers = [x.container for x in cont_actions if not x.is_container_ready] raise TimeoutError( "Container(s) {0} failed to start in {1} seconds." .format(failed_containers, timeout)) def restart_container(self, container): self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd='dockerctl restart {0}'.format(container) ) cont_action = BaseActions() cont_action.container = container cont_action.wait_for_ready_container() def restart_containers(self): for container in self.list_containers(): self.restart_container(container) def execute_in_containers(self, cmd): for container in self.list_containers(): self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="dockerctl shell {0} bash -c '{1}'".format(container, cmd) )
def check_service(ip, commands): """Check that required nova services are running on controller. :param ip: ip address of node :param commands: type list, nova commands to execute on controller, example of commands: ['nova-manage service list | grep vcenter-vmcluster1'] """ ssh_manager = SSHManager() ssh_manager.check_call(ip=ip, command='source openrc') for cmd in commands: wait(lambda: ':-)' in ssh_manager.check_call(ip=ip, command=cmd).stdout[-1], timeout=200)
def restart_service(service_name, timeout=30): restart_cmd = 'service {} restart'.format(service_name) get_status_cmd = 'service {} status'.format(service_name) logger.info("Restarting service '{0}'".format(service_name)) try: SSHManager().execute_on_remote(SSHManager().admin_ip, restart_cmd) helpers.wait(lambda: 'running' in SSHManager().execute_on_remote( SSHManager().admin_ip, get_status_cmd)['stdout_str'], timeout=timeout) logger.info("Service '{0}' has been restarted successfully ".format( service_name)) except Exception as ex: logger.error("Could not restart '{0}' service " "in the '{1}'".format(service_name, ex)) raise
def mcollective_nodes_online(self): nodes_uids = set([ str(node['id']) for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id) ]) ssh_manager = SSHManager() out = ssh_manager.execute_on_remote(ip=ssh_manager.admin_ip, cmd='mco find', assert_ec_equal=[0, 1])['stdout_str'] ready_nodes_uids = set(out.split('\n')) unavailable_nodes = nodes_uids - ready_nodes_uids logger.debug('Nodes {0} are not reacheable via' ' mcollective'.format(unavailable_nodes)) return not unavailable_nodes
def check_service(ip, commands): """Check that required nova services are running on controller. :param ip: ip address of node :param commands: type list, nova commands to execute on controller, example of commands: ['nova-manage service list | grep vcenter-vmcluster1' """ ssh_manager = SSHManager() ssh_manager.execute_on_remote(ip=ip, cmd='source openrc') for cmd in commands: wait(lambda: ':-)' in ssh_manager.execute_on_remote(ip=ip, cmd=cmd)[ 'stdout'][-1], timeout=200)
class DockerActions(object): """DockerActions.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() def list_containers(self): result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd='dockerctl list') return result['stdout'] def wait_for_ready_containers(self, timeout=300): if MASTER_IS_CENTOS7: return cont_actions = [] for container in self.list_containers(): cont_action = BaseActions() cont_action.container = container cont_actions.append(cont_action) try: wait(lambda: all([ cont_action.is_container_ready for cont_action in cont_actions ]), timeout=timeout) except TimeoutError: failed_containers = [ x.container for x in cont_actions if not x.is_container_ready ] raise TimeoutError( "Container(s) {0} failed to start in {1} seconds.".format( failed_containers, timeout)) def restart_container(self, container): self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd='dockerctl restart {0}'.format(container)) cont_action = BaseActions() cont_action.container = container cont_action.wait_for_ready_container() def restart_containers(self): for container in self.list_containers(): self.restart_container(container) def execute_in_containers(self, cmd): for container in self.list_containers(): self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="dockerctl shell {0} bash -c '{1}'".format(container, cmd))
def check_emc_cinder_config(cls, ip, path): command = 'cat {0}'.format(path) conf_data = SSHManager().execute_on_remote(ip, command)['stdout_str'] conf_data = cStringIO(conf_data) cinder_conf = configparser.ConfigParser() cinder_conf.readfp(conf_data) asserts.assert_equal( cinder_conf.get('DEFAULT', 'volume_driver'), 'cinder.volume.drivers.emc.emc_cli_iscsi.EMCCLIISCSIDriver') asserts.assert_equal( cinder_conf.get('DEFAULT', 'storage_vnx_authentication_type'), 'global') asserts.assert_false( cinder_conf.getboolean('DEFAULT', 'destroy_empty_storage_group')) asserts.assert_true( cinder_conf.getboolean('DEFAULT', 'initiator_auto_registration')) asserts.assert_equal( cinder_conf.getint('DEFAULT', 'attach_detach_batch_interval'), -1) asserts.assert_equal(cinder_conf.getint('DEFAULT', 'default_timeout'), 10) asserts.assert_equal(cinder_conf.get('DEFAULT', 'naviseccli_path'), '/opt/Navisphere/bin/naviseccli') asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_ip')) asserts.assert_true( cinder_conf.has_option('DEFAULT', 'san_secondary_ip')) asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_login')) asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_password'))
def mcollective_nodes_online(self): nodes_uids = set( [str(node['id']) for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id)] ) ssh_manager = SSHManager() out = ssh_manager.execute_on_remote( ip=ssh_manager.admin_ip, cmd='mco find', assert_ec_equal=[0, 1] )['stdout_str'] ready_nodes_uids = set(out.split('\n')) unavailable_nodes = nodes_uids - ready_nodes_uids logger.debug('Nodes {0} are not reacheable via' ' mcollective'.format(unavailable_nodes)) return not unavailable_nodes
def upload_nailgun_agent_rpm(): """Upload nailgun_agent.rpm on master node """ ssh = SSHManager() logger.info("Upload nailgun-agent") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') pack_path = '/var/www/nailgun/nailgun-agent-review/' ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Extract rpm context cmd = 'cd {0}; rpm2cpio {1} | cpio -idmv'.format( pack_path, 'nailgun-agent-*.noarch.rpm ') ssh.execute_on_remote(ssh.admin_ip, cmd)
def get_puppet_report(node): """Get puppet run report from corresponding node :param node: a dictionary with node description :return: a dictionary with puppet report data """ ssh = SSHManager() ip = node['ip'] report_file = "/var/lib/puppet/state/last_run_report.yaml" asserts.assert_true(ssh.isfile_on_remote(ip, report_file), 'File {!r} not found on node {!r}' .format(report_file, node['id'])) with ssh.open_on_remote(ip, report_file) as f: data = yaml.load(f) ssh.rm_rf_on_remote(ip, report_file) return data
def check_emc_cinder_config(cls, ip, path): with SSHManager().open_on_remote(ip=ip, path=path) as f: cinder_conf = configparser.ConfigParser() cinder_conf.readfp(f) asserts.assert_equal( cinder_conf.get('DEFAULT', 'volume_driver'), 'cinder.volume.drivers.emc.emc_cli_iscsi.EMCCLIISCSIDriver') asserts.assert_equal( cinder_conf.get('DEFAULT', 'storage_vnx_authentication_type'), 'global') asserts.assert_false( cinder_conf.getboolean('DEFAULT', 'destroy_empty_storage_group')) asserts.assert_true( cinder_conf.getboolean('DEFAULT', 'initiator_auto_registration')) asserts.assert_equal( cinder_conf.getint('DEFAULT', 'attach_detach_batch_interval'), -1) asserts.assert_equal(cinder_conf.getint('DEFAULT', 'default_timeout'), 10) asserts.assert_equal(cinder_conf.get('DEFAULT', 'naviseccli_path'), '/opt/Navisphere/bin/naviseccli') asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_ip')) asserts.assert_true( cinder_conf.has_option('DEFAULT', 'san_secondary_ip')) asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_login')) asserts.assert_true(cinder_conf.has_option('DEFAULT', 'san_password'))
def connect_slaves_to_repo(environment, nodes, repo_name): repo_ip = environment.get_admin_node_ip() repo_port = '8080' repourl = 'http://{master_ip}:{repo_port}/{repo_name}/'.format( master_ip=repo_ip, repo_name=repo_name, repo_port=repo_port) if settings.OPENSTACK_RELEASE == settings.OPENSTACK_RELEASE_UBUNTU: cmds = [ "echo -e '\ndeb {repourl} /' > /etc/apt/sources.list.d/{repo_name}" ".list".format(repourl=repourl, repo_name=repo_name), "apt-key add <(curl -s '{repourl}/Release.key') || :".format( repourl=repourl), # Set highest priority to all repositories located on master node "echo -e 'Package: *\nPin: origin {0}\nPin-Priority: 1060' > " "/etc/apt/preferences.d/custom_repo".format( environment.get_admin_node_ip()), "apt-get update" ] else: cmds = [ "yum-config-manager --add-repo {url}".format(url=repourl), "echo -e 'gpgcheck=0\npriority=20' >>/etc/yum.repos.d/{ip}_{port}_" "{repo}_.repo".format(ip=repo_ip, repo=repo_name, port=repo_port), "yum -y clean all", ] for slave in nodes: for cmd in cmds: SSHManager().execute_on_remote(ip=slave['ip'], cmd=cmd)
def configure_second_admin_dhcp(ip, interface): dhcp_conf_file = '/etc/cobbler/dnsmasq.template' cmd = ("sed '0,/^interface.*/s//\\0\\ninterface={0}/' -i {1};" "cobbler sync").format(interface, dhcp_conf_file) result = SSHManager().execute(ip=ip, cmd=cmd) assert_equal(result['exit_code'], 0, ('Failed to add second admin ' 'network to DHCP server: {0}').format(result))
def check_cinder_vmware_srv(self): """Verify cinder-vmware service.""" ctrl_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( self.cluster_id, ["controller"]) cmd = '. openrc; cinder-manage service list | grep vcenter | ' \ 'grep ":-)"' logger.debug('CMD: {}'.format(cmd)) SSHManager().execute_on_remote(ctrl_nodes[0]['ip'], cmd)
def get_file_size(ip, file_name, file_path): # Moved from checkers.py for improvement of code file_size = SSHManager().execute( ip, 'stat -c "%s" {0}/{1}'.format(file_path, file_name)) assert_equal( int(file_size['exit_code']), 0, "Failed to get '{0}/{1}' file stats on" " remote node".format(file_path, file_name)) return int(file_size['stdout'][0].rstrip())
def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions()
def parse_pcs_status_xml(remote_ip): """Parse 'pcs status xml'. <Nodes> section :param remote_ip: remote IP address :return: nested dictionary with node-fqdn and attribute name as keys """ pcs_status_dict = SSHManager().execute_on_remote( remote_ip, 'pcs status xml')['stdout_str'] return pcs_status_dict
def patch_centos_bootstrap(): """Replaced initramfs.img in /var/www/nailgun/ with newly_builded from review environment - Environment Model object - self.env """ logger.info("Update fuel-agent code and assemble new bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) try: pack_path = '/var/www/nailgun/fuel-agent-review/' ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.FUEL_AGENT_REPO_PATH.rstrip('/'), target=pack_path) # Step 1 - unpack bootstrap bootstrap_var = "/var/initramfs" bootstrap = "/var/www/nailgun/bootstrap" cmd = ("mkdir {0}; cp /{1}/initramfs.img {0}/; cd {0}; " "cat initramfs.img | gunzip | cpio -imudv;").format( bootstrap_var, bootstrap) result = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.debug("Patching bootsrap finishes with {0}".format(result)) # Step 2 - replace fuel-agent code in unpacked bootstrap agent_path = "/usr/lib/python2.7/site-packages/fuel_agent" image_rebuild = "{} | {} | {}".format( "find . -xdev", "cpio --create --format='newc'", "gzip -9 > /var/initramfs.img.updated") cmd = ("rm -rf {0}/initramfs.img; " "rsync -r {2}fuel_agent/* {0}{1}/;" "cd {0}/;" "{3};").format(bootstrap_var, agent_path, pack_path, image_rebuild) result = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.debug("Failed to rebuild image with {0}".format(result)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
def check_package_origin(ip, package, origin): """Check that given package was installed from given repository""" version_cmd = ("apt-cache policy {package} | " "awk '$1 == \"Installed:\" {{print $2}}'").format( package=package) version = SSHManager().execute_on_remote(ip, version_cmd)['stdout_str'] origin_cmd = ("apt-cache madison {package} | " "grep '{version}'").format(package=package, version=version) result = SSHManager().execute_on_remote(ip, origin_cmd)['stdout'] # we only want to check for the UCA uri because it might be in main # or proposed repos = [str.strip(line.split("|")[2]) for line in result] assert_true( any([origin in repo for repo in repos]), "Package {!r}: repository {!r} not found in {!r}".format( package, origin, repos) )
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600): """Check Swift ring and rebalance it if needed. Replication should be performed on primary controller node. Retry check several times. Wait for replication due to LP1498368. """ ssh = SSHManager() cmd = "/usr/local/bin/swift-rings-rebalance.sh" logger.debug('Check swift ring and rebalance it.') for _ in xrange(retry_count): try: checkers.check_swift_ring(controller_ip) break except AssertionError: result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd) logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(controller_ip)
class BaseActions(object): """BaseActions.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() self.admin_ip = self.ssh_manager.admin_ip def __repr__(self): klass, obj_id = type(self), hex(id(self)) return "[{klass}({obj_id})]".format( klass=klass, obj_id=obj_id) def restart_service(self, service): self.ssh_manager.execute_on_remote( ip=self.admin_ip, cmd="systemctl restart {0}".format(service), err_msg="Failed to restart service {!r}, please inspect logs for " "details".format(service))
def check_ping(ip, host, deadline=10, size=56, timeout=1, interval=1): """Check network connectivity from remote to host using ICMP (ping) :param ip: remote ip :param host: string IP address or host/domain name :param deadline: time in seconds before ping exits :param size: size of data to be sent :param timeout: time to wait for a response, in seconds :param interval: wait interval seconds between sending each packet :return: bool: True if ping command """ ssh_manager = SSHManager() cmd = ("ping -W {timeout} -i {interval} -s {size} -c 1 -w {deadline} " "{host}".format(host=host, size=size, timeout=timeout, interval=interval, deadline=deadline)) res = ssh_manager.execute(ip, cmd) return int(res['exit_code']) == 0
def replace_centos_bootstrap(environment): """Replaced initramfs.img in /var/www/nailgun/ with re-builded with review code environment - Environment Model object - self.env """ logger.info("Updating bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) rebuilded_bootstrap = '/var/initramfs.img.updated' with environment.d_env.get_admin_remote() as remote: checkers.check_file_exists( remote, '{0}'.format(rebuilded_bootstrap)) logger.info("Assigning new bootstrap from {}".format(rebuilded_bootstrap)) bootstrap = "/var/www/nailgun/bootstrap" cmd = ("mv {0}/initramfs.img /var/initramfs.img;" "cp /var/initramfs.img.updated {0}/initramfs.img;" "chmod +r {0}/initramfs.img;").format(bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) cmd = "cobbler sync" ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), login=settings.SSH_CREDENTIALS["login"], password=settings.SSH_CREDENTIALS["password"], ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions()
def replace_fuel_agent_rpm(): """Replaced fuel_agent.rpm on master node with fuel_agent.rpm from review """ ssh = SSHManager() logger.info("Patching fuel-agent") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') try: pack_path = '/var/www/nailgun/fuel-agent/' full_pack_path = os.path.join(pack_path, 'fuel-agent*.noarch.rpm') ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Update fuel-agent on master node cmd = "rpm -q fuel-agent" old_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] logger.info("Updating package {0} with {1}" .format(old_package, new_package)) if old_package != new_package: logger.info("Updating fuel-agent package on master node") logger.info('Try to install package {0}'.format( new_package)) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd) cmd = "rpm -q fuel-agent" installed_package = ssh.execute_on_remote( ssh.admin_ip, cmd)['stdout_str'] assert_equal(installed_package, new_package, "The new package {0} was not installed". format(new_package)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), admin_login=settings.SSH_FUEL_CREDENTIALS['login'], admin_password=settings.SSH_FUEL_CREDENTIALS['password'], slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, '_fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time() return self.nailgun_nodes(devops_nodes) def sync_time(self, nodes_names=None, skip_sync=False): if nodes_names is None: roles = ['fuel_master', 'fuel_slave'] nodes_names = [node.name for node in self.d_env.get_nodes() if node.role in roles and node.driver.node_active(node)] logger.info("Please wait while time on nodes: {0} " "will be synchronized" .format(', '.join(sorted(nodes_names)))) new_time = sync_time(self.d_env, nodes_names, skip_sync) for name in sorted(new_time): logger.info("New time on '{0}' = {1}".format(name, new_time[name])) @logwrap def get_admin_node_ip(self): return str( self.d_env.nodes( ).admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'device_label': settings.ISO_LABEL, 'iface': iface_alias('eth0'), 'ip': node.get_ip_address_by_network_name( self.d_env.admin_net), 'mask': self.d_env.get_network( name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': self.d_env.nat_interface, 'nameserver': settings.DNS, 'showmenu': 'yes' if settings.SHOW_FUELMENU else 'no', 'wait_for_external_config': 'yes', 'build_images': '1' if build_images else '0', 'MASTER_NODE_EXTRA_PACKAGES': settings.MASTER_NODE_EXTRA_PACKAGES } # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" ) else: # cdrom is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" ) keys += ( "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img" " inst.ks=cdrom:LABEL=%(device_label)s:/ks.cfg" " inst.repo=cdrom:LABEL=%(device_label)s:/" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":%(iface)s:off::: nameserver=%(nameserver)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s" " build_images=%(build_images)s\n" " MASTER_NODE_EXTRA_PACKAGES='%(MASTER_NODE_EXTRA_PACKAGES)s'\n" " <Enter>\n" ) % params return keys @staticmethod def get_target_devs(devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config[ 'template']['devops_settings']['env_name']) except Exception: self._virt_env = Environment.create_environment( full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin self.ssh_manager.clean_all_connections() try: admin.await(self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin.await(self.d_env.admin_net) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend() time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return [self.fuel_web.get_nailgun_node_by_devops_node(node) for node in devops_nodes] def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError( "Node {0} does not become online".format(node.name)) return True def revert_snapshot(self, name, skip_timesync=False, skip_slaves_check=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: {:s}'.format(name)) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: self.sync_time() try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() if not skip_slaves_check: _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_FUEL_CREDENTIALS['login'] new_password = settings.SSH_FUEL_CREDENTIALS['password'] try: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='date' ) logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login='******', admin_password='******', slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, new_password) ) self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login=new_login, admin_password=new_password, slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.ssh_manager.update_connection( ip=self.ssh_manager.admin_ip, login=new_login, password=new_password ) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password']) ) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def insert_cdrom_tray(self): # This is very rude implementation and it SHOULD be changes after # implementation this feature in fuel-devops name = "{}_{}".format(settings.ENV_NAME, self.d_env.nodes().admin.name) name_size = 80 if len(name) > name_size: hash_str = str(hash(name)) name = (hash_str + name)[:name_size] cmd = """EDITOR="sed -i s/tray=\\'open\\'//" virsh edit {}""".format( name) subprocess.check_call(cmd, shell=True) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # Create environment and start the Fuel master node admin = self.d_env.nodes().admin self.d_env.start([admin]) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if settings.SHOW_FUELMENU: self.wait_for_fuelmenu() else: self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() self.admin_actions.wait_for_fuel_ready() @logwrap def enable_force_https(self, admin_node_ip): cmd = """ echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml """ self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = "find / -name \"nginx_services.pp\"" puppet_manifest = \ self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0].strip() cmd = 'puppet apply {0}'.format(puppet_manifest) self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = """ systemctl status nginx.service | awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}' """ wait(lambda: ( self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0] != 'dead'), interval=10, timeout=30) # pylint: disable=no-self-use @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...') # pylint: enable=no-self-use @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): _wait(lambda: _tcp_ping( self.d_env.nodes( ).admin.get_ip_address_by_network_name (self.d_env.admin_net), 22), timeout=timeout) @logwrap def wait_for_fuelmenu(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): def check_ssh_connection(): """Try to close fuelmenu and check ssh connection""" try: _tcp_ping( self.d_env.nodes( ).admin.get_ip_address_by_network_name (self.d_env.admin_net), 22) except Exception: # send F8 trying to exit fuelmenu self.d_env.nodes().admin.send_keys("<F8>\n") return False return True wait(check_ssh_connection, interval=30, timeout=timeout, timeout_msg="Fuelmenu hasn't appeared during allocated timeout") @logwrap def wait_for_external_config(self, timeout=120): wait(lambda: self.ssh_manager.exists_on_remote( self.ssh_manager.admin_ip, '/var/lock/wait_for_external_config'), timeout=600) check_cmd = 'pkill -0 -f wait_for_external_config' wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=check_cmd)['exit_code'] == 0, timeout=timeout) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=kill_cmd ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=check_cmd ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Running bootstrap (timeout: {0})".format( float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))) with TimeStat("admin_node_bootsrap_time", is_uniq=True): wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment' '{:s}'".format(log_path) )['exit_code'] == 0, timeout=(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT)) ) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment " "complete' '{:s}'".format(log_path))['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = iface_alias('eth0') command = "dhcpcheck discover " \ "--ifaces {iface} " \ "--repeat 3 " \ "--timeout 10".format(iface=iface) out = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=command )['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu': logger.warning('Default image for bootstrap ' 'is not based on Ubuntu!') return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel-bootstrap --quiet list' )['stdout'] assert_true(any('active' in line for line in bootstrap_images), 'Ubuntu bootstrap image wasn\'t built and activated! ' 'See logs in /var/log/fuel-bootstrap-image-build.log ' 'for details.') def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name) ) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name) ) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name) ) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name) ) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) # Execute yum updates # If updates installed, # then `bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info('Searching for updates..') update_command = 'yum clean expire-cache; yum update -y' update_result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=update_command ) logger.info('Result of "{1}" command on master node: ' '{0}'.format(update_result, update_command)) assert_equal(int(update_result['exit_code']), 0, 'Packages update failed, ' 'inspect logs for details') # Check if any packets were updated and update was successful for str_line in update_result['stdout']: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates)\ and not match_complete_message: logger.warning('No updates were found or update was incomplete.') return logger.info('{0} packet(s) were updated'.format(updates_count)) cmd = 'bootstrap_admin_node.sh;' result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) logger.info('Result of "{1}" command on master node: ' '{0}'.format(result, cmd)) assert_equal(int(result['exit_code']), 0, 'bootstrap failed, ' 'inspect logs for details') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd='cat /etc/resolv.conf' ) assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}' .format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=echo_cmd ) assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}' .format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @staticmethod @logwrap def execute_remote_cmd(remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result)) return result['stdout'] @logwrap def describe_other_admin_interfaces(self, admin): admin_networks = [iface.network.name for iface in admin.interfaces] iface_name = None for i, network_name in enumerate(admin_networks): if 'admin' in network_name and 'admin' != network_name: # This will be replaced with actual interface labels # form fuel-devops iface_name = 'enp0s' + str(i + 3) logger.info("Describe Fuel admin node interface {0} for " "network {1}".format(iface_name, network_name)) self.describe_admin_interface(iface_name, network_name) if iface_name: return self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="cobbler sync") @logwrap def describe_admin_interface(self, admin_if, network_name): admin_net_object = self.d_env.get_network(name=network_name) admin_network = admin_net_object.ip.network admin_netmask = admin_net_object.ip.netmask admin_ip = str(self.d_env.nodes( ).admin.get_ip_address_by_network_name(network_name)) logger.info(('Parameters for admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(admin_network, admin_netmask, admin_if, admin_ip)) add_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(admin_if, admin_ip, admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_admin_ip, admin_if, admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(admin_ip, admin_if)) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp( self.ssh_manager.admin_ip, admin_if ) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, admin_network, admin_netmask, admin_if, self.get_admin_node_ip() ) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
def __init__(self): self.ssh_manager = SSHManager() self.admin_ip = self.ssh_manager.admin_ip
class BaseActions(object): """BaseActions.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() self.admin_ip = self.ssh_manager.admin_ip def __repr__(self): klass, obj_id = type(self), hex(id(self)) return "[{klass}({obj_id})]".format( klass=klass, obj_id=obj_id) # TODO(kozhukalov): This method seems not needed and # can easily be replaced by using execute_on_remote # available in SSHManager (up to the type of return value) def execute(self, cmd, exit_code=None, stdin=None): if stdin is not None: cmd = 'echo "{0}" | {1}'.format(stdin, cmd) result = self.ssh_manager.execute( ip=self.admin_ip, cmd=cmd ) if exit_code is not None: assert_equal(exit_code, result['exit_code'], ('Command {cmd} returned exit code "{e}", but ' 'expected "{c}". Output: {out}; {err} ').format( cmd=cmd, e=result['exit_code'], c=exit_code, out=result['stdout'], err=result['stderr'] )) return ''.join(result['stdout']).strip() def restart_service(self, service): result = self.ssh_manager( ip=self.admin_ip, cmd="systemctl restart {0}".format(service)) return result['exit_code'] == 0 def put_value_to_local_yaml(self, old_file, new_file, element, value): """Changes content in old_file at element is given to the new value and creates new file with changed content :param old_file: a path to the file content from to be changed :param new_file: a path to the new file to ve created with new content :param element: tuple with path to element to be changed for example: ['root_elem', 'first_elem', 'target_elem'] if there are a few elements with equal names use integer to identify which element should be used :return: nothing """ with open(old_file, 'r') as f_old: yaml_dict = yaml.load(f_old) origin_yaml = yaml_dict for k in element[:-1]: yaml_dict = yaml_dict[k] yaml_dict[element[-1]] = value with open(new_file, 'w') as f_new: yaml.dump(origin_yaml, f_new, default_flow_style=False, default_style='"') def get_value_from_local_yaml(self, yaml_file, element): """Get a value of the element from the local yaml file :param str yaml_file: a path to the yaml file :param list element: list with path to element to be read for example: ['root_elem', 'first_elem', 'target_elem'] if there are a few elements with equal names use integer to identify which element should be used :return obj: value """ with open(yaml_file, 'r') as f_old: yaml_dict = yaml.load(f_old) for i, k in enumerate(element): try: yaml_dict = yaml_dict[k] except IndexError: raise IndexError("Element {0} not found in the file {1}" .format(element[: i + 1], f_old)) except KeyError: raise KeyError("Element {0} not found in the file {1}" .format(element[: i + 1], f_old)) return yaml_dict def change_remote_yaml(self, path_to_file, element, value): """Changes values in the yaml file stored There is no need to copy file manually :param path_to_file: absolute path to the file :param element: list with path to the element be changed :param value: new value for element :return: Nothing """ old_file = '/tmp/temp_file_{0}.old.yaml'.format(str(os.getpid())) new_file = '/tmp/temp_file_{0}.new.yaml'.format(str(os.getpid())) self.ssh_manager.download_from_remote( ip=self.admin_ip, destination=path_to_file, target=old_file ) self.put_value_to_local_yaml(old_file, new_file, element, value) self.ssh_manager.upload_to_remote( ip=self.admin_ip, source=new_file, target=path_to_file ) os.remove(old_file) os.remove(new_file) def get_value_from_remote_yaml(self, path_to_file, element): """Get a value from the yaml file stored on the master node :param str path_to_file: absolute path to the file :param list element: list with path to the element :return obj: value """ host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid())) self.ssh_manager.download_from_remote( ip=self.admin_ip, destination=path_to_file, target=host_tmp_file ) value = self.get_value_from_local_yaml(host_tmp_file, element) os.remove(host_tmp_file) return value def put_value_to_remote_yaml(self, path_to_file, element, value): """Put a value to the yaml file stored on the master node :param str path_to_file: absolute path to the file :param list element: list with path to the element be changed :param value: new value for element :return: None """ host_tmp_file = '/tmp/temp_file_{0}.yaml'.format(str(os.getpid())) self.ssh_manager.download_from_remote( ip=self.admin_ip, destination=path_to_file, target=host_tmp_file ) self.put_value_to_local_yaml(host_tmp_file, host_tmp_file, element, value) self.ssh_manager.upload_to_remote( ip=self.admin_ip, source=host_tmp_file, target=path_to_file ) os.remove(host_tmp_file)
def generate_facts(ip): ssh_manager = SSHManager() facter_dir = '/var/lib/puppet/lib/facter' exluded_facts = ['naily.rb'] if not ssh_manager.isdir_on_remote(ip, facter_dir): ssh_manager.mkdir_on_remote(ip, facter_dir) logger.debug('Directory {0} was created'.format(facter_dir)) ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir)) logger.debug('rb files were removed from {0}'.format(facter_dir)) facts_files = ssh_manager.execute_on_remote( ip, 'find /etc/puppet/modules/ -wholename "*/lib/facter/*.rb"')['stdout'] facts_files = [i.strip() for i in facts_files] logger.debug('The following facts {0} will' ' be copied to {1}'.format(facts_files, facter_dir)) for fact in facts_files: if not fact or re.sub(r'.*/', '', fact) in exluded_facts: continue ssh_manager.execute_on_remote(ip, 'cp {0} {1}/'.format(fact, facter_dir)) logger.debug('Facts were copied') ssh_manager.execute_on_remote(ip, 'facter -p -y > /tmp/facts.yaml') logger.info('Facts yaml was created') ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir)) logger.debug('rb files were removed from {0}'.format(facter_dir))
def centos_setup_fuel(self, hostname): logger.info("upload fuel-release packet") if not settings.FUEL_RELEASE_PATH: raise exceptions.FuelQAVariableNotSet('FUEL_RELEASE_PATH', '/path') try: ssh = SSHManager() pack_path = '/tmp/' full_pack_path = os.path.join(pack_path, 'fuel-release*.noarch.rpm') ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.FUEL_RELEASE_PATH.rstrip('/'), target=pack_path) except Exception: logger.exception("Could not upload package") logger.debug("Update host information") cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "echo {0} {1} {2} >> /etc/hosts".format( ssh.admin_ip, hostname, settings.FUEL_MASTER_HOSTNAME) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "hostname {}".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) logger.debug("setup MOS repositories") cmd = "rpm -ivh {}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y fuel-setup" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y screen" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) logger.info("Install Fuel services") cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \ "bootstrap_admin_node.sh'" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) self.env.wait_for_external_config() self.env.admin_actions.modify_configs(self.env.d_env.router()) self.env.kill_wait_for_external_config() self.env.wait_bootstrap() logger.debug("Check Fuel services") self.env.admin_actions.wait_for_fuel_ready() logger.debug("post-installation configuration of Fuel services") self.fuel_post_install_actions()
def centos_setup_fuel(self, hostname): with TimeStat("bootstrap_centos_node", is_uniq=True): admin = list(self.env.d_env.get_nodes(role__contains='master'))[0] self.env.d_env.start([admin]) logger.info("Waiting for Centos node to start up") wait(lambda: admin.driver.node_active(admin), 60, timeout_msg='Centos node failed to start') logger.info("Waiting for Centos node ssh ready") self.env.wait_for_provisioning() ssh = SSHManager() logger.debug("Update host information") cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "echo {0} {1} {2} >> /etc/hosts".format( ssh.admin_ip, hostname, settings.FUEL_MASTER_HOSTNAME) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "hostname {}".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y screen" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) install_mos_repos() logger.info("Install Fuel services") cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \ "bootstrap_admin_node.sh'" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) self.env.wait_for_external_config() self.env.admin_actions.modify_configs(self.env.d_env.router()) if CUSTOM_FUEL_SETTING_YAML: self.env.admin_actions.update_fuel_setting_yaml( CUSTOM_FUEL_SETTING_YAML) self.env.kill_wait_for_external_config() self.env.wait_bootstrap() logger.debug("Check Fuel services") self.env.admin_actions.wait_for_fuel_ready() logger.debug("post-installation configuration of Fuel services") self.fuel_post_install_actions()