def _turnoff_executable_ruby(node): """Set mode -x for /usr/bin/ruby :param node: dict, node attributes """ ssh = SSHManager() cmd = 'chmod -x /usr/bin/ruby' ssh.execute_on_remote(node['ip'], cmd)
def _turnon_executable_ruby(node): """Set mode +x for /usr/bin/ruby :param node: dict, node attributes """ ssh = SSHManager() cmd = 'chmod +x /usr/bin/ruby' ssh.execute_on_remote(node['ip'], cmd)
def _turnon_executable_ruby(node): """Set mode +x for /usr/bin/ruby :param node: dict, node attributes """ ssh = SSHManager() cmd = "chmod +x /usr/bin/ruby" ssh.execute_on_remote(node["ip"], cmd)
def generate_facts(ip): ssh_manager = SSHManager() facter_dir = '/var/lib/puppet/lib/facter' exluded_facts = ['naily.rb'] if not ssh_manager.isdir_on_remote(ip, facter_dir): ssh_manager.mkdir_on_remote(ip, facter_dir) logger.debug('Directory {0} was created'.format(facter_dir)) ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir)) logger.debug('rb files were removed from {0}'.format(facter_dir)) facts_files = ssh_manager.execute_on_remote( ip, 'find /etc/puppet/modules/ -wholename "*/lib/facter/*.rb"')['stdout'] facts_files = [i.strip() for i in facts_files] logger.debug('The following facts {0} will' ' be copied to {1}'.format(facts_files, facter_dir)) for fact in facts_files: if not fact or re.sub(r'.*/', '', fact) in exluded_facts: continue ssh_manager.execute_on_remote(ip, 'cp {0} {1}/'.format(fact, facter_dir)) logger.debug('Facts were copied') ssh_manager.execute_on_remote(ip, 'facter -p -y > /tmp/facts.yaml') logger.info('Facts yaml was created') ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir)) logger.debug('rb files were removed from {0}'.format(facter_dir))
def delete_astute_log(): """Delete astute.log file(s) on master node. This is to ensure that no unwanted tasks are used by tests (e.g. from previous deployments). :return: None """ ssh = SSHManager() ssh.execute_on_remote(ssh.admin_ip, "rm /var/log/astute/astute*") ssh.execute_on_remote(ssh.admin_ip, "systemctl restart astute.service")
def inject_nailgun_agent_ubuntu_bootstrap(environment): """Inject nailgun agent packet from review into ubuntu bootsrap environment - Environment Model object - self.env """ logger.info("Update nailgun-agent code and assemble new ubuntu bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) pack_path = '/var/www/nailgun/nailgun-agent-review/' # Step 1 - install squashfs-tools cmd = "yum install -y squashfs-tools" ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 2 - unpack bootstrap bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d /var/root.squashfs {}/root.squashfs".format( bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 3 - replace nailgun-agent code in unpacked bootstrap agent_path = "/usr/bin/nailgun-agent" bootstrap_file = bootstrap + "/root.squashfs" logger.info('bootsrap file {0}{1}'.format(bootstrap_var, agent_path)) old_sum = get_sha_sum('{0}{1}'.format(bootstrap_var, agent_path)) logger.info('Old sum is {0}'.format(old_sum)) cmd_etc_sync = ('rsync -r {1}etc/* {0}/etc/'.format( bootstrap_var, pack_path)) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd_etc_sync) cmd = ("rsync -r {1}usr/* {0}/usr/;" "mv {2} " "/var/root.squashfs.old;" "").format(bootstrap_var, pack_path, bootstrap_file) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) new_sum = get_sha_sum('{0}{1}'.format(bootstrap_var, agent_path)) logger.info('new sum is {0}'.format(new_sum)) assert_equal(new_sum != old_sum, True) # Step 4 - assemble new bootstrap compression = "-comp xz" no_progress_bar = "-no-progress" no_append = "-noappend" image_rebuild = "mksquashfs {0} {1} {2} {3} {4}".format( bootstrap_var, bootstrap_file, compression, no_progress_bar, no_append) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=image_rebuild) checkers.check_file_exists(ssh.admin_ip, bootstrap_file)
def centos_setup_fuel(self, hostname): with TimeStat("bootstrap_centos_node", is_uniq=True): admin = list(self.env.d_env.get_nodes(role__contains='master'))[0] self.env.d_env.start([admin]) logger.info("Waiting for Centos node to start up") wait(lambda: admin.driver.node_active(admin), 60, timeout_msg='Centos node failed to start') logger.info("Waiting for Centos node ssh ready") self.env.wait_for_provisioning() ssh = SSHManager() logger.debug("Update host information") cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "echo {0} {1} {2} >> /etc/hosts".format( ssh.admin_ip, hostname, settings.FUEL_MASTER_HOSTNAME) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "hostname {}".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y screen" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) install_mos_repos() logger.info("Install Fuel services") cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \ "bootstrap_admin_node.sh'" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) self.env.wait_for_external_config() self.env.admin_actions.modify_configs(self.env.d_env.router()) if CUSTOM_FUEL_SETTING_YAML: self.env.admin_actions.update_fuel_setting_yaml( CUSTOM_FUEL_SETTING_YAML) self.env.kill_wait_for_external_config() self.env.wait_bootstrap() logger.debug("Check Fuel services") self.env.admin_actions.wait_for_fuel_ready() logger.debug("post-installation configuration of Fuel services") self.fuel_post_install_actions()
def check_service(ip, commands): """Check that required nova services are running on controller. :param ip: ip address of node :param commands: type list, nova commands to execute on controller, example of commands: ['nova-manage service list | grep vcenter-vmcluster1' """ ssh_manager = SSHManager() ssh_manager.execute_on_remote(ip=ip, cmd='source openrc') for cmd in commands: wait(lambda: ':-)' in ssh_manager.execute_on_remote(ip=ip, cmd=cmd)[ 'stdout'][-1], timeout=200)
def upload_nailgun_agent_rpm(): """Upload nailgun_agent.rpm on master node """ ssh = SSHManager() logger.info("Upload nailgun-agent") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') pack_path = '/var/www/nailgun/nailgun-agent-review/' ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Extract rpm context cmd = 'cd {0}; rpm2cpio {1} | cpio -idmv'.format( pack_path, 'nailgun-agent-*.noarch.rpm ') ssh.execute_on_remote(ssh.admin_ip, cmd)
def get_nodes_tasks(node_id): """ :param node_id: an integer number of node id :return: a set of deployment tasks for corresponding node """ tasks = set() ssh = SSHManager() result = ssh.execute_on_remote(ssh.admin_ip, "ls /var/log/astute") filenames = [filename.strip() for filename in result['stdout']] for filename in filenames: ssh.download_from_remote( ssh.admin_ip, destination="/var/log/astute/{0}".format(filename), target="/tmp/{0}".format(filename)) data = fileinput.FileInput( files=["/tmp/{0}".format(filename) for filename in filenames], openhook=fileinput.hook_compressed) for line in data: if "Task time summary" in line \ and "node {}".format(node_id) in line: # FIXME: define an exact search of task task_name = line.split("Task time summary: ")[1].split()[0] check = any([excluded_task in task_name for excluded_task in TASKS_BLACKLIST]) if check: continue tasks.add(task_name) return tasks
def upload_nailgun_agent_rpm(): """Upload nailgun_agent.rpm on master node """ ssh = SSHManager() logger.info("Upload nailgun-agent") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') pack_path = '/var/www/nailgun/nailgun-agent-review/' ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Extract rpm context cmd = 'cd {0}; rpm2cpio {1} | cpio -idmv'.format( pack_path, 'nailgun-agent-*.noarch.rpm ') ssh.execute_on_remote(ssh.admin_ip, cmd)
def inject_nailgun_agent_ubuntu_bootstrap(environment): """Inject nailgun agent packet from review into ubuntu bootsrap environment - Environment Model object - self.env """ logger.info("Update nailgun-agent code and assemble new ubuntu bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist".format(settings.UPDATE_FUEL)) pack_path = '/var/www/nailgun/nailgun-agent-review/' # Step 1 - install squashfs-tools cmd = "yum install -y squashfs-tools" ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 2 - unpack bootstrap bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d /var/root.squashfs {}/root.squashfs".format(bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 3 - replace nailgun-agent code in unpacked bootstrap agent_path = "/usr/bin/nailgun-agent" bootstrap_file = bootstrap + "/root.squashfs" logger.info('bootsrap file {0}{1}'.format(bootstrap_var, agent_path)) old_sum = get_sha_sum('{0}{1}'.format(bootstrap_var, agent_path)) logger.info('Old sum is {0}'.format(old_sum)) cmd_etc_sync = ('rsync -r {1}etc/* {0}/etc/'.format( bootstrap_var, pack_path)) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd_etc_sync) cmd = ("rsync -r {1}usr/* {0}/usr/;" "mv {2} " "/var/root.squashfs.old;" "").format(bootstrap_var, pack_path, bootstrap_file) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) new_sum = get_sha_sum('{0}{1}'.format(bootstrap_var, agent_path)) logger.info('new sum is {0}'.format(new_sum)) assert_equal(new_sum != old_sum, True) # Step 4 - assemble new bootstrap compression = "-comp xz" no_progress_bar = "-no-progress" no_append = "-noappend" image_rebuild = "mksquashfs {0} {1} {2} {3} {4}".format( bootstrap_var, bootstrap_file, compression, no_progress_bar, no_append) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=image_rebuild) checkers.check_file_exists(ssh.admin_ip, bootstrap_file)
def check_service(ip, commands): """Check that required nova services are running on controller. :param ip: ip address of node :param commands: type list, nova commands to execute on controller, example of commands: ['nova-manage service list | grep vcenter-vmcluster1' """ ssh_manager = SSHManager() ssh_manager.execute_on_remote(ip=ip, cmd='source openrc') for cmd in commands: wait( lambda: ':-)' in ssh_manager.execute_on_remote(ip=ip, cmd=cmd)['stdout'][-1], timeout=200)
def patch_centos_bootstrap(): """Replaced initramfs.img in /var/www/nailgun/ with newly_builded from review environment - Environment Model object - self.env """ logger.info("Update fuel-agent code and assemble new bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) try: pack_path = '/var/www/nailgun/fuel-agent-review/' ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.FUEL_AGENT_REPO_PATH.rstrip('/'), target=pack_path) # Step 1 - unpack bootstrap bootstrap_var = "/var/initramfs" bootstrap = "/var/www/nailgun/bootstrap" cmd = ("mkdir {0}; cp /{1}/initramfs.img {0}/; cd {0}; " "cat initramfs.img | gunzip | cpio -imudv;").format( bootstrap_var, bootstrap) result = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.debug("Patching bootsrap finishes with {0}".format(result)) # Step 2 - replace fuel-agent code in unpacked bootstrap agent_path = "/usr/lib/python2.7/site-packages/fuel_agent" image_rebuild = "{} | {} | {}".format( "find . -xdev", "cpio --create --format='newc'", "gzip -9 > /var/initramfs.img.updated") cmd = ("rm -rf {0}/initramfs.img; " "rsync -r {2}fuel_agent/* {0}{1}/;" "cd {0}/;" "{3};").format(bootstrap_var, agent_path, pack_path, image_rebuild) result = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.debug("Failed to rebuild image with {0}".format(result)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
def install_mos_repos(): """ Upload and install fuel-release packet with mos-repo description and install necessary packets for packetary Fuel installation :return: nothing """ logger.info("upload fuel-release packet") if not settings.FUEL_RELEASE_PATH: raise exceptions.FuelQAVariableNotSet('FUEL_RELEASE_PATH', '/path') try: ssh = SSHManager() pack_path = '/tmp/' full_pack_path = os.path.join(pack_path, 'fuel-release*.noarch.rpm') ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.FUEL_RELEASE_PATH.rstrip('/'), target=pack_path) if settings.RPM_REPOS_YAML: with ssh.open_on_remote( ip=ssh.admin_ip, path='/etc/yum.repos.d/custom.repo', mode="w") as f: f.write(generate_yum_repos_config(settings.RPM_REPOS_YAML)) if settings.DEB_REPOS_YAML: ssh = SSHManager() pack_path = "/root/default_deb_repos.yaml" ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.DEB_REPOS_YAML, target=pack_path) except Exception: logger.exception("Could not upload package") raise logger.debug("setup MOS repositories") cmd = "rpm -ivh {}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y fuel-setup" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)
def patch_and_assemble_ubuntu_bootstrap(environment): """Replaced initramfs.img in /var/www/nailgun/ with newly_builded from review environment - Environment Model object - self.env """ logger.info("Update fuel-agent code and assemble new ubuntu bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) try: pack_path = '/var/www/nailgun/fuel-agent-review/' ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.FUEL_AGENT_REPO_PATH.rstrip('/'), target=pack_path) # renew code in bootstrap # Step 1 - install squashfs-tools cmd = "yum install -y squashfs-tools" ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 2 - unpack bootstrap bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d /var/root.squashfs {}/root.squashfs".format( bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 3 - replace fuel-agent code in unpacked bootstrap agent_path = "/usr/lib/python2.7/dist-packages/fuel_agent" bootstrap_file = bootstrap + "/root.squashfs" cmd = ("rsync -r {2}fuel_agent/* {0}{1}/;" "mv {3} /var/root.squashfs.old;" ).format(bootstrap_var, agent_path, pack_path, bootstrap_file) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 4 - assemble new bootstrap compression = "-comp xz" no_progress_bar = "-no-progress" no_append = "-noappend" image_rebuild = "mksquashfs {0} {1} {2} {3} {4}".format( bootstrap_var, bootstrap_file, compression, no_progress_bar, no_append) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=image_rebuild) with environment.d_env.get_admin_remote() as remote: checkers.check_file_exists(remote, '{0}'.format(bootstrap_file)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
class BaseActions(object): """BaseActions.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() self.admin_ip = self.ssh_manager.admin_ip def __repr__(self): klass, obj_id = type(self), hex(id(self)) return "[{klass}({obj_id})]".format( klass=klass, obj_id=obj_id) def restart_service(self, service): self.ssh_manager.execute_on_remote( ip=self.admin_ip, cmd="systemctl restart {0}".format(service), err_msg="Failed to restart service {!r}, please inspect logs for " "details".format(service))
def replace_fuel_agent_rpm(): """Replaced fuel_agent.rpm on master node with fuel_agent.rpm from review """ ssh = SSHManager() logger.info("Patching fuel-agent") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') try: pack_path = '/var/www/nailgun/fuel-agent/' full_pack_path = os.path.join(pack_path, 'fuel-agent*.noarch.rpm') ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Update fuel-agent on master node cmd = "rpm -q fuel-agent" old_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] logger.info("Updating package {0} with {1}" .format(old_package, new_package)) if old_package != new_package: logger.info("Updating fuel-agent package on master node") logger.info('Try to install package {0}'.format( new_package)) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd) cmd = "rpm -q fuel-agent" installed_package = ssh.execute_on_remote( ssh.admin_ip, cmd)['stdout_str'] assert_equal(installed_package, new_package, "The new package {0} was not installed". format(new_package)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
def hiera_json_out(node_ip, parameter): hiera_cmd = "ruby -rhiera -rjson -e \"h = Hiera.new(); " \ "Hiera.logger = 'noop'; " \ "puts JSON.dump(h.lookup(\'{0}\', " \ "[], {{}}, nil, nil))\"".format(parameter) ssh_manager = SSHManager() config = ssh_manager.execute_on_remote( ip=node_ip, cmd=hiera_cmd, jsonify=True, err_msg='Cannot get floating ranges')['stdout_json'] return config
def replace_fuel_agent_rpm(): """Replaced fuel_agent.rpm on master node with fuel_agent.rpm from review """ ssh = SSHManager() logger.info("Patching fuel-agent") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') try: pack_path = '/var/www/nailgun/fuel-agent/' full_pack_path = os.path.join(pack_path, 'fuel-agent*.noarch.rpm') ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Update fuel-agent on master node cmd = "rpm -q fuel-agent" old_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] logger.info("Updating package {0} with {1}".format( old_package, new_package)) if old_package != new_package: logger.info("Updating fuel-agent package on master node") logger.info('Try to install package {0}'.format(new_package)) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd) cmd = "rpm -q fuel-agent" installed_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] assert_equal( installed_package, new_package, "The new package {0} was not installed".format(new_package)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
def patch_and_assemble_ubuntu_bootstrap(environment): """Replaced initramfs.img in /var/www/nailgun/ with newly_builded from review environment - Environment Model object - self.env """ logger.info("Update fuel-agent code and assemble new ubuntu bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist".format(settings.UPDATE_FUEL)) try: pack_path = '/var/www/nailgun/fuel-agent-review/' ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.FUEL_AGENT_REPO_PATH.rstrip('/'), target=pack_path) # renew code in bootstrap # Step 1 - install squashfs-tools cmd = "yum install -y squashfs-tools" ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 2 - unpack bootstrap bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d /var/root.squashfs {}/root.squashfs".format( bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 3 - replace fuel-agent code in unpacked bootstrap agent_path = "/usr/lib/python2.7/dist-packages/fuel_agent" bootstrap_file = bootstrap + "/root.squashfs" cmd = ("rsync -r {2}fuel_agent/* {0}{1}/;" "mv {3} /var/root.squashfs.old;").format( bootstrap_var, agent_path, pack_path, bootstrap_file) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 4 - assemble new bootstrap compression = "-comp xz" no_progress_bar = "-no-progress" no_append = "-noappend" image_rebuild = "mksquashfs {0} {1} {2} {3} {4}".format( bootstrap_var, bootstrap_file, compression, no_progress_bar, no_append) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=image_rebuild) checkers.check_file_exists(ssh.admin_ip, '{0}'.format(bootstrap_file)) except Exception as e: logger.error("Could not upload package {e}".format(e=e)) raise
def replace_centos_bootstrap(environment): """Replaced initramfs.img in /var/www/nailgun/ with re-builded with review code environment - Environment Model object - self.env """ logger.info("Updating bootstrap") ssh = SSHManager() if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) rebuilded_bootstrap = '/var/initramfs.img.updated' with environment.d_env.get_admin_remote() as remote: checkers.check_file_exists( remote, '{0}'.format(rebuilded_bootstrap)) logger.info("Assigning new bootstrap from {}".format(rebuilded_bootstrap)) bootstrap = "/var/www/nailgun/bootstrap" cmd = ("mv {0}/initramfs.img /var/initramfs.img;" "cp /var/initramfs.img.updated {0}/initramfs.img;" "chmod +r {0}/initramfs.img;").format(bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) cmd = "cobbler sync" ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
def mcollective_nodes_online(self): nodes_uids = set([ str(node['id']) for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id) ]) ssh_manager = SSHManager() out = ssh_manager.execute_on_remote(ip=ssh_manager.admin_ip, cmd='mco find', assert_ec_equal=[0, 1])['stdout_str'] ready_nodes_uids = set(out.split('\n')) unavailable_nodes = nodes_uids - ready_nodes_uids logger.debug('Nodes {0} are not reacheable via' ' mcollective'.format(unavailable_nodes)) return not unavailable_nodes
def mcollective_nodes_online(self): nodes_uids = set( [str(node['id']) for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id)] ) ssh_manager = SSHManager() out = ssh_manager.execute_on_remote( ip=ssh_manager.admin_ip, cmd='mco find', assert_ec_equal=[0, 1] )['stdout_str'] ready_nodes_uids = set(out.split('\n')) unavailable_nodes = nodes_uids - ready_nodes_uids logger.debug('Nodes {0} are not reacheable via' ' mcollective'.format(unavailable_nodes)) return not unavailable_nodes
def rebalance_swift_ring(controller_ip, retry_count=5, sleep=600): """Check Swift ring and rebalance it if needed. Replication should be performed on primary controller node. Retry check several times. Wait for replication due to LP1498368. """ ssh = SSHManager() cmd = "/usr/local/bin/swift-rings-rebalance.sh" logger.debug('Check swift ring and rebalance it.') for _ in xrange(retry_count): try: checkers.check_swift_ring(controller_ip) break except AssertionError: result = ssh.execute_on_remote(ip=controller_ip, cmd=cmd) logger.debug("command execution result is {0}".format(result)) else: checkers.check_swift_ring(controller_ip)
def check_package_version_injected_in_bootstraps(package, cluster_id=None, ironic=None): ssh = SSHManager() try: pack_path = '/var/www/nailgun/{}/'.format(package) ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) except Exception: logger.exception("Could not upload package") raise # Step 1 - unpack active bootstrap logger.info("unpack active bootstrap") if ironic: bootstrap = "/var/www/nailgun/bootstrap/ironic/{}".format(cluster_id) else: bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d {} {}/root.squashfs".format(bootstrap_var, bootstrap) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd) # Step 2 - check package version logger.info("check package {} version injected in ubuntu bootstrap".format( package)) cmd = "ls {}|grep {} |grep deb |cut -f 2 -d '_'".format(pack_path, package) package_from_review = ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("package from review is {}".format(package_from_review)) awk_pattern = "awk '{print $2}'" cmd = "chroot {}/ /bin/bash -c \"dpkg -s {}\"|grep Version|{}".format( bootstrap_var, package, awk_pattern) installed_package = ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("injected package is {}".format(installed_package)) assert_equal( installed_package, package_from_review, "The new package {0} wasn't injected in bootstrap".format( package_from_review)) # Step 3 - remove unpacked bootstrap cmd = "rm -rf {}".format(bootstrap_var) ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation __metaclass__ = SingletonMeta def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), login=settings.SSH_CREDENTIALS["login"], password=settings.SSH_CREDENTIALS["password"], ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, "_fuel_web"): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, "api/v1/json") @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time() return self.nailgun_nodes(devops_nodes) def sync_time(self, nodes_names=None, skip_sync=False): if nodes_names is None: roles = ["fuel_master", "fuel_slave"] nodes_names = [ node.name for node in self.d_env.get_nodes() if node.role in roles and node.driver.node_active(node) ] logger.info( "Please wait while time on nodes: {0} " "will be synchronized".format(", ".join(sorted(nodes_names))) ) new_time = sync_time(self.d_env, nodes_names, skip_sync) for name in sorted(new_time): logger.info("New time on '{0}' = {1}".format(name, new_time[name])) @logwrap def get_admin_node_ip(self): return str(self.d_env.nodes().admin.get_ip_address_by_network_name(self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as="cdrom"): params = { "ks": "hd:LABEL=Mirantis_Fuel:/ks.cfg" if iso_connect_as == "usb" else "cdrom:/ks.cfg", "repo": "hd:LABEL=Mirantis_Fuel:/", # only required for USB boot "ip": node.get_ip_address_by_network_name(self.d_env.admin_net), "mask": self.d_env.get_network(name=self.d_env.admin_net).ip.netmask, "gw": self.d_env.router(), "hostname": "".join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), "nat_interface": self.d_env.nat_interface, "dns1": settings.DNS, "showmenu": "no", "wait_for_external_config": "yes", "build_images": "1" if build_images else "0", } if iso_connect_as == "usb": keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params else: # cdrom case is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params if MASTER_IS_CENTOS7: # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. iface = "enp0s3" if iso_connect_as == "usb": keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":{iface}:off::: dns1=%(dns1)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n".format(iface=iface) ) % params else: # cdrom case is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":{iface}:off::: dns1=%(dns1)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n".format(iface=iface) ) % params return keys def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [val for var in map(lambda node: node.interfaces, devops_nodes) for val in var] ] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment(boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config["template"]["devops_settings"]["env_name"]) except Exception: self._virt_env = Environment.create_environment(full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin try: admin.await(self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info("Admin node was destroyed. Wait 10 sec.") time.sleep(10) admin.start() logger.info("Admin node started second time.") self.d_env.nodes().admin.await(self.d_env.admin_net) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL ) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info( "Enabled sending of statistics to {0}:{1}".format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT ) ) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend(verbose=False) time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return map(lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(node)["online"], timeout=60 * 6) except TimeoutError: raise TimeoutError("Node {0} does not become online".format(node.name)) return True def revert_snapshot(self, name, skip_timesync=False): if not self.d_env.has_snapshot(name): return False logger.info("We have snapshot with such name: %s" % name) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: self.sync_time() try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_CREDENTIALS["login"] new_password = settings.SSH_CREDENTIALS["password"] try: self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd="date") logger.debug("Accessing admin node using SSH: SUCCESS") except Exception: logger.debug("Accessing admin node using SSH credentials:" " FAIL, trying to change password from default") self.ssh_manager.initialize(admin_ip=self.ssh_manager.admin_ip, login="******", password="******") self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, new_password) ) self.ssh_manager.initialize(admin_ip=self.ssh_manager.admin_ip, login=new_login, password=new_password) self.ssh_manager.update_connection(ip=self.ssh_manager.admin_ip, login=new_login, password=new_password) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format(new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="fuel user --newpass {0} --change-password".format(settings.KEYSTONE_CREDS["password"]), ) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'.format( settings.KEYSTONE_CREDS["username"], settings.KEYSTONE_CREDS["password"] ) ) def insert_cdrom_tray(self): # This is very rude implementation and it SHOULD be changes after # implementation this feature in fuel-devops name = "{}_{}".format(settings.ENV_NAME, self.d_env.nodes().admin.name) NAME_SIZE = 80 if len(name) > NAME_SIZE: hash_str = str(hash(name)) name = (hash_str + name)[:NAME_SIZE] cmd = """EDITOR="sed -i s/tray=\\'open\\'//" virsh edit {}""".format(name) subprocess.check_call(cmd, shell=True) def reinstall_master_node(self): """Erase boot sector and run setup_environment""" with self.d_env.get_admin_remote() as remote: erase_data_from_hdd(remote, mount_point="/boot") remote.execute("/sbin/shutdown") self.d_env.nodes().admin.destroy() self.insert_cdrom_tray() self.setup_environment() def setup_environment( self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST, ): # start admin node admin = self.d_env.nodes().admin if iso_connect_as == "usb": admin.disk_devices.get(device="disk", bus="usb").volume.upload(settings.ISO_PATH) else: # cdrom is default admin.disk_devices.get(device="cdrom").volume.upload(settings.ISO_PATH) self.d_env.start(self.d_env.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() if settings.UPDATE_FUEL: # Update Ubuntu packages self.admin_actions.upload_packages( local_packages_dir=settings.UPDATE_FUEL_PATH, centos_repo_path=None, ubuntu_repo_path=settings.LOCAL_MIRROR_UBUNTU, ) self.admin_actions.wait_for_fuel_ready() time.sleep(10) self.set_admin_keystone_password() self.sync_time(["admin"]) if settings.UPDATE_MASTER: if settings.UPDATE_FUEL_MIRROR: for i, url in enumerate(settings.UPDATE_FUEL_MIRROR): conf_file = "/etc/yum.repos.d/temporary-{}.repo".format(i) cmd = ( "echo -e" " '[temporary-{0}]\nname=" "temporary-{0}\nbaseurl={1}/" "\ngpgcheck=0\npriority=" "1' > {2}" ).format(i, url, conf_file) self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) self.admin_install_updates() if settings.MULTIPLE_NETWORKS: self.describe_other_admin_interfaces(admin) if not MASTER_IS_CENTOS7: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL ) # Restart statsenderd to apply settings(Collector address) self.nailgun_actions.force_fuel_stats_sending() if settings.FUEL_STATS_ENABLED and not MASTER_IS_CENTOS7: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info( "Enabled sending of statistics to {0}:{1}".format(settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT) ) if settings.PATCHING_DISABLE_UPDATES: cmd = ( "find /etc/yum.repos.d/ -type f -regextype posix-egrep" " -regex '.*/mos[0-9,\.]+\-(updates|security).repo' | " "xargs -n1 -i sed '$aenabled=0' -i {}" ) self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=cmd) @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info("Installing custom packages/manifests " "before master node bootstrap...") @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): _wait( lambda: _tcp_ping(self.d_env.nodes().admin.get_ip_address_by_network_name(self.d_env.admin_net), 22), timeout=timeout, ) @logwrap def wait_for_external_config(self, timeout=120): check_cmd = "pkill -0 -f wait_for_external_config" if MASTER_IS_CENTOS7: self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=check_cmd) else: wait( lambda: self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=check_cmd)["exit_code"] == 0, timeout=timeout, ) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=kill_cmd) self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=check_cmd) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format(float(settings.PUPPET_TIMEOUT))) with self.d_env.get_admin_remote() as admin_remote: wait( lambda: not admin_remote.execute("grep 'Fuel node deployment' '%s'" % log_path)["exit_code"], timeout=(float(settings.PUPPET_TIMEOUT)), ) result = admin_remote.execute("grep 'Fuel node deployment " "complete' '%s'" % log_path)["exit_code"] if result != 0: raise Exception("Fuel node deployment failed.") self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = "enp0s3" command = "dhcpcheck discover " "--ifaces {iface} " "--repeat 3 " "--timeout 10".format(iface=iface) out = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=command)["stdout"] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings["BOOTSTRAP"]["flavor"].lower() != "ubuntu": logger.warning("Default image for bootstrap " "is not based on Ubuntu!") return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="fuel-bootstrap --quiet list" )["stdout"] assert_true( any("active" in line for line in bootstrap_images), "Ubuntu bootstrap image wasn't built and activated! " "See logs in /var/log/fuel-bootstrap-image-build.log " "for details.", ) def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name)) if remote_status["exit_code"] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name) ) logger.info( "Installation of the package '{0}' has been" " completed with exit code {1}".format(pkg_name, remote_status["exit_code"]) ) return remote_status["exit_code"] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name) ) if any("running..." in status for status in remote_status["stdout"]): logger.info("Service '{0}' is running".format(service_name)) else: logger.info( "Service '{0}' failed to start" " with exit code {1} :\n{2}".format(service_name, remote_status["exit_code"], remote_status["stdout"]) ) # Execute yum updates # If updates installed, # then `bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info("Searching for updates..") update_command = "yum clean expire-cache; yum update -y" update_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=update_command) logger.info('Result of "{1}" command on master node: ' "{0}".format(update_result, update_command)) assert_equal(int(update_result["exit_code"]), 0, "Packages update failed, " "inspect logs for details") # Check if any packets were updated and update was successful for str_line in update_result["stdout"]: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates) and not match_complete_message: logger.warning("No updates were found or update was incomplete.") return logger.info("{0} packet(s) were updated".format(updates_count)) cmd = "bootstrap_admin_node.sh;" result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) logger.info('Result of "{1}" command on master node: ' "{0}".format(result, cmd)) assert_equal(int(result["exit_code"]), 0, "bootstrap failed, " "inspect logs for details") # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="cat /etc/resolv.conf") assert_equal( 0, resolv_conf["exit_code"], 'Executing "{0}" on the admin node has failed with: {1}'.format( "cat /etc/resolv.conf", resolv_conf["stderr"] ), ) if merge: nameservers.extend(resolv_conf["stdout"]) resolv_keys = ["search", "domain", "nameserver"] resolv_new = "".join("{0}\n".format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=echo_cmd) assert_equal( 0, echo_result["exit_code"], 'Executing "{0}" on the admin node has failed with: {1}'.format(echo_cmd, echo_result["stderr"]), ) return resolv_conf["stdout"] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result["exit_code"], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format(cmd, result)) return result["stdout"] @logwrap def describe_other_admin_interfaces(self, admin): admin_networks = [iface.network.name for iface in admin.interfaces] iface_name = None for i, network_name in enumerate(admin_networks): if "admin" in network_name and "admin" != network_name: # This will be replaced with actual interface labels # form fuel-devops iface_name = "enp0s" + str(i + 3) logger.info( "Describe Fuel admin node interface {0} for " "network {1}".format(iface_name, network_name) ) self.describe_admin_interface(iface_name, network_name) if iface_name: return self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="cobbler sync") @logwrap def describe_admin_interface(self, admin_if, network_name): admin_net_object = self.d_env.get_network(name=network_name) admin_network = admin_net_object.ip.network admin_netmask = admin_net_object.ip.netmask admin_ip = str(self.d_env.nodes().admin.get_ip_address_by_network_name(network_name)) logger.info( ( "Parameters for admin interface configuration: " "Network - {0}, Netmask - {1}, Interface - {2}, " "IP Address - {3}" ).format(admin_network, admin_netmask, admin_if, admin_ip) ) add_admin_ip = ( "DEVICE={0}\\n" "ONBOOT=yes\\n" "NM_CONTROLLED=no\\n" "USERCTL=no\\n" "PEERDNS=no\\n" "BOOTPROTO=static\\n" "IPADDR={1}\\n" "NETMASK={2}\\n" ).format(admin_if, admin_ip, admin_netmask) cmd = ( 'echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' "ifup {1}; ip -o -4 a s {1} | grep -w {2}" ).format(add_admin_ip, admin_if, admin_ip) logger.debug("Trying to assign {0} IP to the {1} on master node...".format(admin_ip, admin_if)) result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) assert_equal( result["exit_code"], 0, ("Failed to assign second admin " "IP address on master node: {0}").format(result) ) logger.debug("Done: {0}".format(result["stdout"])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp(self.ssh_manager.admin_ip, admin_if) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, admin_network, admin_netmask, admin_if, self.get_admin_node_ip() ) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db="nailgun", query="select master_node_uid from master_node_settings limit 1;" )
class EnvironmentModel(six.with_metaclass(SingletonMeta, object)): """EnvironmentModel.""" # TODO documentation def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), admin_login=settings.SSH_FUEL_CREDENTIALS["login"], admin_password=settings.SSH_FUEL_CREDENTIALS["password"], slave_login=settings.SSH_SLAVE_CREDENTIALS["login"], slave_password=settings.SSH_SLAVE_CREDENTIALS["password"], ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, "_fuel_web"): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, "api/v1/json") @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait( lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout, timeout_msg="Bootstrap timeout for nodes: {}" "".format([node.name for node in devops_nodes]), ) if not skip_timesync: self.sync_time() return self.nailgun_nodes(devops_nodes) def sync_time(self, nodes_names=None, skip_sync=False): if nodes_names is None: roles = ["fuel_master", "fuel_slave"] nodes_names = [ node.name for node in self.d_env.get_nodes() if node.role in roles and node.driver.node_active(node) ] logger.info( "Please wait while time on nodes: {0} " "will be synchronized".format(", ".join(sorted(nodes_names))) ) new_time = sync_time(self.d_env, nodes_names, skip_sync) for name in sorted(new_time): logger.info("New time on '{0}' = {1}".format(name, new_time[name])) @logwrap def get_admin_node_ip(self): return str(self.d_env.nodes().admin.get_ip_address_by_network_name(self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as="cdrom"): params = { "device_label": settings.ISO_LABEL, "iface": iface_alias("eth0"), "ip": node.get_ip_address_by_network_name(self.d_env.admin_net), "mask": self.d_env.get_network(name=self.d_env.admin_net).ip.netmask, "gw": self.d_env.router(), "hostname": "".join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), "nat_interface": self.d_env.nat_interface, "nameserver": settings.DNS, "showmenu": "yes" if settings.SHOW_FUELMENU else "no", "wait_for_external_config": "yes", "build_images": "1" if build_images else "0", "MASTER_NODE_EXTRA_PACKAGES": settings.MASTER_NODE_EXTRA_PACKAGES, } # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. if iso_connect_as == "usb": keys = "<Wait>\n" "<F12>\n" "2\n" # USB boot uses boot_menu=yes for master node else: # cdrom is default keys = "<Wait>\n" "<Wait>\n" "<Wait>\n" keys += ( "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img" " inst.ks=cdrom:LABEL=%(device_label)s:/ks.cfg" " inst.repo=cdrom:LABEL=%(device_label)s:/" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":%(iface)s:off::: nameserver=%(nameserver)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s" " build_images=%(build_images)s\n" " MASTER_NODE_EXTRA_PACKAGES='%(MASTER_NODE_EXTRA_PACKAGES)s'\n" " <Enter>\n" ) % params return keys @staticmethod def get_target_devs(devops_nodes): return [ interface.target_dev for interface in [val for var in map(lambda node: node.interfaces, devops_nodes) for val in var] ] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment(boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config["template"]["devops_settings"]["env_name"]) except Exception: self._virt_env = Environment.create_environment(full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin self.ssh_manager.clean_all_connections() try: admin.await(self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info("Admin node was destroyed. Wait 10 sec.") time.sleep(10) admin.start() logger.info("Admin node started second time.") self.d_env.nodes().admin.await(self.d_env.admin_net) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL ) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info( "Enabled sending of statistics to {0}:{1}".format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT ) ) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend() time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return [self.fuel_web.get_nailgun_node_by_devops_node(node) for node in devops_nodes] def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) self.fuel_web.wait_nodes_get_online_state(devops_nodes, timeout=60 * 6) return True def revert_snapshot(self, name, skip_timesync=False, skip_slaves_check=False): if not self.d_env.has_snapshot(name): return False logger.info("We have snapshot with such name: {:s}".format(name)) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: self.sync_time() else: self.sync_time(["admin"]) try: with QuietLogger(upper_log_level=logging.CRITICAL): # TODO(astudenov): add timeout_msg wait_pass( self.fuel_web.client.get_releases, expected=(exceptions.RetriableConnectionFailure, exceptions.UnknownConnectionError), timeout=300, ) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() if not skip_slaves_check: # TODO(astudenov): add timeout_msg wait_pass(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_FUEL_CREDENTIALS["login"] new_password = settings.SSH_FUEL_CREDENTIALS["password"] try: self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd="date") logger.debug("Accessing admin node using SSH: SUCCESS") except Exception: logger.debug("Accessing admin node using SSH credentials:" " FAIL, trying to change password from default") self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login="******", admin_password="******", slave_login=settings.SSH_SLAVE_CREDENTIALS["login"], slave_password=settings.SSH_SLAVE_CREDENTIALS["password"], ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, new_password) ) self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login=new_login, admin_password=new_password, slave_login=settings.SSH_SLAVE_CREDENTIALS["login"], slave_password=settings.SSH_SLAVE_CREDENTIALS["password"], ) self.ssh_manager.update_connection(ip=self.ssh_manager.admin_ip, login=new_login, password=new_password) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format(new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="fuel user --newpass {0} --change-password".format(settings.KEYSTONE_CREDS["password"]), ) config_file = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="ls -1 $HOME/.config/fuel/fuel_client.yaml" )["stdout_str"] with YamlEditor(config_file, ip=self.admin_node_ip) as editor: editor.content["OS_USERNAME"] = settings.KEYSTONE_CREDS["username"] editor.content["OS_PASSWORD"] = settings.KEYSTONE_CREDS["password"] with YamlEditor(settings.FUEL_SETTINGS_YAML, ip=self.admin_node_ip) as editor: editor.content["FUEL_ACCESS"]["user"] = settings.KEYSTONE_CREDS["username"] editor.content["FUEL_ACCESS"]["password"] = settings.KEYSTONE_CREDS["password"] logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'.format( settings.KEYSTONE_CREDS["username"], settings.KEYSTONE_CREDS["password"] ) ) def setup_environment( self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST, ): # Create environment and start the Fuel master node admin = self.d_env.nodes().admin self.d_env.start([admin]) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60, timeout_msg="Admin node startup timeout") logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if settings.SHOW_FUELMENU: self.wait_for_fuelmenu() else: self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) if CUSTOM_FUEL_SETTING_YAML: self.admin_actions.update_fuel_setting_yaml(CUSTOM_FUEL_SETTING_YAML) self.kill_wait_for_external_config() self.wait_bootstrap() self.admin_actions.wait_for_fuel_ready() @logwrap def enable_force_https(self, admin_node_ip): cmd = """ echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml """ self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = 'find / -name "nginx_services.pp"' puppet_manifest = self.ssh_manager.execute_on_remote(admin_node_ip, cmd)["stdout"][0].strip() cmd = "puppet apply {0}".format(puppet_manifest) self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = """ systemctl status nginx.service | awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}' """ wait( lambda: (self.ssh_manager.execute_on_remote(admin_node_ip, cmd)["stdout"][0] != "dead"), interval=10, timeout=30, timeout_msg="Nginx service is dead after trying to enable " "it with the command: {}".format(cmd), ) # pylint: disable=no-self-use @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info("Installing custom packages/manifests " "before master node bootstrap...") # pylint: enable=no-self-use @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): # TODO(astudenov): add timeout_msg wait_pass( lambda: tcp_ping_(self.d_env.nodes().admin.get_ip_address_by_network_name(self.d_env.admin_net), 22), timeout=timeout, ) @logwrap def wait_for_fuelmenu(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): def check_ssh_connection(): """Try to close fuelmenu and check ssh connection""" try: tcp_ping_(self.d_env.nodes().admin.get_ip_address_by_network_name(self.d_env.admin_net), 22) except Exception: # send F8 trying to exit fuelmenu self.d_env.nodes().admin.send_keys("<F8>\n") return False return True wait( check_ssh_connection, interval=30, timeout=timeout, timeout_msg="Fuelmenu hasn't appeared during allocated timeout", ) @logwrap def wait_for_external_config(self, timeout=120): wait( lambda: self.ssh_manager.exists_on_remote(self.ssh_manager.admin_ip, "/var/lock/wait_for_external_config"), timeout=600, timeout_msg="wait_for_external_config lock file timeout " "while bootstrapping the Fuel master node", ) check_cmd = "pkill -0 -f wait_for_external_config" wait( lambda: self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=check_cmd)["exit_code"] == 0, timeout=timeout, timeout_msg="wait_for_external_config process timeout " "while bootstrapping the Fuel master node", ) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=kill_cmd) self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=check_cmd) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Running bootstrap (timeout: {0})".format(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))) with TimeStat("admin_node_bootsrap_time", is_uniq=True): wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment' '{:s}'".format(log_path) )["exit_code"] == 0, timeout=(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT)), timeout_msg="Fuel master node bootstrap timeout, " "please check the log {}".format(log_path), ) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment " "complete' '{:s}'".format(log_path) )["exit_code"] if result != 0: raise Exception("Fuel node deployment failed.") self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = iface_alias("eth0") command = "dhcpcheck discover " "--ifaces {iface} " "--repeat 3 " "--timeout 10".format(iface=iface) out = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=command)["stdout"] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings["BOOTSTRAP"]["flavor"].lower() != "ubuntu": logger.warning("Default image for bootstrap " "is not based on Ubuntu!") return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="fuel-bootstrap --quiet list" )["stdout"] assert_true( any("active" in line for line in bootstrap_images), "Ubuntu bootstrap image wasn't built and activated! " "See logs in /var/log/fuel-bootstrap-image-build.log " "for details.", ) def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name)) if remote_status["exit_code"] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name) ) logger.info( "Installation of the package '{0}' has been" " completed with exit code {1}".format(pkg_name, remote_status["exit_code"]) ) return remote_status["exit_code"] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name) ) if any("running..." in status for status in remote_status["stdout"]): logger.info("Service '{0}' is running".format(service_name)) else: logger.info( "Service '{0}' failed to start" " with exit code {1} :\n{2}".format(service_name, remote_status["exit_code"], remote_status["stdout"]) ) # Execute yum updates # If updates installed, # then `bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info("Searching for updates..") update_command = "yum clean expire-cache && " "yum update -y 2>>/var/log/yum-update-error.log" logger.info("Performing yum clean and update commands") update_result = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=update_command, err_msg="Packages update failed, inspect logs for details" ) logger.info("Packages were updated successfully") # Check if any packets were updated and update was successful match_updated_count = re.search(r"Upgrade\s+(\d+)\s+Package", update_result["stdout_str"]) # In case of package replacement, the new one is marked as # installed and the old one as removed match_installed_count = re.search(r"Install\s+(\d+)\s+Package", update_result["stdout_str"]) match_complete_message = re.search(r"Complete!", update_result["stdout_str"]) match_no_updates = re.search("No Packages marked for Update", update_result["stdout_str"]) if match_no_updates or not match_complete_message or not (match_updated_count or match_installed_count): logger.warning("No updates were found or update was incomplete.") return updates_count = 0 if match_updated_count: updates_count += int(match_updated_count.group(1)) if match_installed_count: updates_count += int(match_installed_count.group(1)) logger.info("{0} package(s) were updated".format(updates_count)) logger.warning( "'bootstrap_admin_node.sh' is used for applying 9.x release." "It should be replaced with proper procedure when it will be " "merged - https://review.openstack.org/#/c/346119/ " ) # this is temporary solution for disabling 50min timeout; # should be removed when the main script for 9.0->9.x will be merged self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="sed -i " '"s/wait_for_external_config=yes/wait_for_external_config=no/"' " /etc/fuel/bootstrap_admin_node.conf", raise_on_assert=False, ) # devops is creating ssh connection without associated tty # which leads to broken fuelmenu. we do not need to call fuelmenu # while applying 9.x so turn it off # this is only addition to fix for fuel-devops for unlocking usb thread # without devops bump self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd="sed -i " '"s/showmenu=yes/showmenu=no/"' " /etc/fuel/bootstrap_admin_node.conf", raise_on_assert=False, ) # end of temporary solutions cmd = "bootstrap_admin_node.sh;" self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd, err_msg="bootstrap failed, inspect logs for details" ) logger.info("bootstrap successfull") # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="cat /etc/resolv.conf") assert_equal( 0, resolv_conf["exit_code"], 'Executing "{0}" on the admin node has failed with: {1}'.format( "cat /etc/resolv.conf", resolv_conf["stderr"] ), ) if merge: nameservers.extend(resolv_conf["stdout"]) resolv_keys = ["search", "domain", "nameserver"] resolv_new = "".join("{0}\n".format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=echo_cmd) assert_equal( 0, echo_result["exit_code"], 'Executing "{0}" on the admin node has failed with: {1}'.format(echo_cmd, echo_result["stderr"]), ) return resolv_conf["stdout"] @staticmethod @logwrap def execute_remote_cmd(remote, cmd, exit_code=0): msg = ( "execute_remote_cmd() is old deprecated method, " "which should not be used anymore. " "please use remote.check_call() instead.\n" "Starting from fuel-devops 2.9.22 this methods will return all " "required data.\n" "{}".format("".join(traceback.format_stack())) ) warn(msg, DeprecationWarning) logger.warning(msg) logger.critical("This method could be deleted on 01.09.2016 " "without any announcement!") result = remote.check_call(command=cmd, expected=[exit_code]) return result["stdout"] @logwrap def describe_other_admin_interfaces(self, admin): admin_networks = [iface.network.name for iface in admin.interfaces] iface_name = None for i, network_name in enumerate(admin_networks): if "admin" in network_name and "admin" != network_name: # This will be replaced with actual interface labels # form fuel-devops iface_name = "enp0s" + str(i + 3) logger.info( "Describe Fuel admin node interface {0} for " "network {1}".format(iface_name, network_name) ) self.describe_admin_interface(iface_name, network_name) if iface_name: return self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="cobbler sync") @logwrap def describe_admin_interface(self, admin_if, network_name): admin_net_object = self.d_env.get_network(name=network_name) admin_network = admin_net_object.ip.network admin_netmask = admin_net_object.ip.netmask admin_ip = str(self.d_env.nodes().admin.get_ip_address_by_network_name(network_name)) logger.info( ( "Parameters for admin interface configuration: " "Network - {0}, Netmask - {1}, Interface - {2}, " "IP Address - {3}" ).format(admin_network, admin_netmask, admin_if, admin_ip) ) add_admin_ip = ( "DEVICE={0}\\n" "ONBOOT=yes\\n" "NM_CONTROLLED=no\\n" "USERCTL=no\\n" "PEERDNS=no\\n" "BOOTPROTO=static\\n" "IPADDR={1}\\n" "NETMASK={2}\\n" ).format(admin_if, admin_ip, admin_netmask) cmd = ( 'echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' "ifup {1}; ip -o -4 a s {1} | grep -w {2}" ).format(add_admin_ip, admin_if, admin_ip) logger.debug("Trying to assign {0} IP to the {1} on master node...".format(admin_ip, admin_if)) result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) assert_equal( result["exit_code"], 0, ("Failed to assign second admin " "IP address on master node: {0}").format(result) ) logger.debug("Done: {0}".format(result["stdout"])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp(self.ssh_manager.admin_ip, admin_if) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, admin_network, admin_netmask, admin_if, self.get_admin_node_ip() ) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db="nailgun", query="select master_node_uid from master_node_settings limit 1;" )
def centos_setup_fuel(self, hostname): logger.info("upload fuel-release packet") if not settings.FUEL_RELEASE_PATH: raise exceptions.FuelQAVariableNotSet('FUEL_RELEASE_PATH', '/path') try: ssh = SSHManager() pack_path = '/tmp/' full_pack_path = os.path.join(pack_path, 'fuel-release*.noarch.rpm') ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.FUEL_RELEASE_PATH.rstrip('/'), target=pack_path) except Exception: logger.exception("Could not upload package") logger.debug("Update host information") cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "echo {0} {1} {2} >> /etc/hosts".format( ssh.admin_ip, hostname, settings.FUEL_MASTER_HOSTNAME) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "hostname {}".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) logger.debug("setup MOS repositories") cmd = "rpm -ivh {}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y fuel-setup" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y screen" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) logger.info("Install Fuel services") cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \ "bootstrap_admin_node.sh'" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) self.env.wait_for_external_config() self.env.admin_actions.modify_configs(self.env.d_env.router()) self.env.kill_wait_for_external_config() self.env.wait_bootstrap() logger.debug("Check Fuel services") self.env.admin_actions.wait_for_fuel_ready() logger.debug("post-installation configuration of Fuel services") self.fuel_post_install_actions()
def centos_setup_fuel(self, hostname): logger.info("upload fuel-release packet") if not settings.FUEL_RELEASE_PATH: raise exceptions.FuelQAVariableNotSet('FUEL_RELEASE_PATH', '/path') try: ssh = SSHManager() pack_path = '/tmp/' full_pack_path = os.path.join(pack_path, 'fuel-release*.noarch.rpm') ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.FUEL_RELEASE_PATH.rstrip('/'), target=pack_path) except Exception: logger.exception("Could not upload package") logger.debug("Update host information") cmd = "echo HOSTNAME={} >> /etc/sysconfig/network".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "echo {0} {1} {2} >> /etc/hosts".format( ssh.admin_ip, hostname, settings.FUEL_MASTER_HOSTNAME) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "hostname {}".format(hostname) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) logger.debug("setup MOS repositories") cmd = "rpm -ivh {}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y fuel-setup" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "yum install -y screen" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) logger.info("Install Fuel services") cmd = "screen -dm bash -c 'showmenu=no wait_for_external_config=yes " \ "bootstrap_admin_node.sh'" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) self.env.wait_for_external_config() self.env.admin_actions.modify_configs(self.env.d_env.router()) self.env.kill_wait_for_external_config() self.env.wait_bootstrap() logger.debug("Check Fuel services") self.env.admin_actions.wait_for_fuel_ready() logger.debug("post-installation configuration of Fuel services") self.fuel_post_install_actions()
class CustomRepo(object): """CustomRepo.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() self.ip = self.ssh_manager.admin_ip self.path_scripts = ('{0}/fuelweb_test/helpers/' .format(os.environ.get("WORKSPACE", "./"))) self.remote_path_scripts = '/tmp/' self.ubuntu_script = 'regenerate_ubuntu_repo' self.centos_script = 'regenerate_centos_repo' self.local_mirror_ubuntu = settings.LOCAL_MIRROR_UBUNTU self.local_mirror_centos = settings.LOCAL_MIRROR_CENTOS self.ubuntu_release = settings.UBUNTU_RELEASE self.centos_supported_archs = ['noarch', 'x86_64'] self.pkgs_list = [] self.custom_pkgs_mirror_path = '' if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: # Trying to determine the root of Ubuntu repository pkgs_path = settings.CUSTOM_PKGS_MIRROR.split('/dists/') if len(pkgs_path) == 2: self.custom_pkgs_mirror = pkgs_path[0] self.custom_pkgs_mirror_path = '/dists/{}'.format(pkgs_path[1]) else: self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR else: self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR def prepare_repository(self): """Prepare admin node to packages testing Scenario: 1. Temporary set nameserver to local router on admin node 2. Install tools to manage rpm/deb repository 3. Retrieve list of packages from custom repository 4. Download packages to local rpm/deb repository 5. Update .yaml file with new packages version 6. Re-generate repo using shell scripts on admin node """ # Check necessary settings and revert a snapshot if not self.custom_pkgs_mirror: return logger.info("Custom mirror with new packages: {0}" .format(settings.CUSTOM_PKGS_MIRROR)) if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: # Ubuntu master_tools = ['dpkg', 'dpkg-devel', 'dpkg-dev'] self.install_tools(master_tools) self.get_pkgs_list_ubuntu() pkgs_local_path = ('{0}/pool/' .format(self.local_mirror_ubuntu)) self.download_pkgs(pkgs_local_path) self.regenerate_repo(self.ubuntu_script, self.local_mirror_ubuntu) else: # CentOS master_tools = ['createrepo'] self.install_tools(master_tools) self.get_pkgs_list_centos() pkgs_local_path = '{0}/Packages/'.format(self.local_mirror_centos) self.download_pkgs(pkgs_local_path) self.regenerate_repo(self.centos_script, self.local_mirror_centos) # Install tools to masternode def install_tools(self, master_tools=None): if master_tools is None: master_tools = [] logger.info("Installing necessary tools for {0}" .format(settings.OPENSTACK_RELEASE)) for master_tool in master_tools: exit_code = install_pkg_2( ip=self.ip, pkg_name=master_tool ) assert_equal(0, exit_code, 'Cannot install package {0} ' 'on admin node.'.format(master_tool)) # Ubuntu: Creating list of packages from the additional mirror def get_pkgs_list_ubuntu(self): url = "{0}/{1}/Packages".format(self.custom_pkgs_mirror, self.custom_pkgs_mirror_path) logger.info("Retrieving additional packages from the custom mirror:" " {0}".format(url)) try: pkgs_release = urllib2.urlopen(url).read() except (urllib2.HTTPError, urllib2.URLError): logger.error(traceback.format_exc()) url_gz = '{0}.gz'.format(url) logger.info( "Retrieving additional packages from the custom mirror:" " {0}".format(url_gz)) try: pkgs_release_gz = urllib2.urlopen(url_gz).read() except (urllib2.HTTPError, urllib2.URLError): logger.error(traceback.format_exc()) raise try: d = zlib.decompressobj(zlib.MAX_WBITS | 32) pkgs_release = d.decompress(pkgs_release_gz) except Exception: logger.error('Ubuntu mirror error: Could not decompress {0}\n' '{1}'.format(url_gz, traceback.format_exc())) raise packages = (pkg for pkg in pkgs_release.split("\n\n") if pkg) for package in packages: upkg = {pstr.split()[0].lower(): ''.join(pstr.split()[1:]) for pstr in package.split("\n") if pstr[0].strip()} upkg_keys = ["package:", "version:", "filename:"] assert_equal(True, all(x in upkg for x in upkg_keys), 'Missing one of the statements ["Package:", ' '"Version:", "Filename:"] in {0}'.format(url)) # TODO: add dependencies list to upkg self.pkgs_list.append(upkg) # Centos: Creating list of packages from the additional mirror def get_pkgs_list_centos(self): logger.info("Retrieving additional packages from the custom mirror:" " {0}".format(self.custom_pkgs_mirror)) url = "{0}/repodata/repomd.xml".format(self.custom_pkgs_mirror) try: repomd_data = urllib2.urlopen(url).read() except (urllib2.HTTPError, urllib2.URLError): logger.error(traceback.format_exc()) raise # Remove namespace attribute before parsing XML repomd_data = re.sub(' xmlns="[^"]+"', '', repomd_data, count=1) tree_repomd_data = ElementTree.fromstring(repomd_data) lists_location = '' for repomd in tree_repomd_data.findall('data'): if repomd.get('type') == 'primary': repomd_location = repomd.find('location') lists_location = repomd_location.get('href') assert_equal(True, lists_location is not '', 'CentOS mirror error:' ' Could not parse {0}\nlists_location = "{1}"\n{2}' .format(url, lists_location, traceback.format_exc())) url = "{0}/{1}".format(self.custom_pkgs_mirror, lists_location) try: lists_data = urllib2.urlopen(url).read() except (urllib2.HTTPError, urllib2.URLError): logger.error(traceback.format_exc()) raise if '.xml.gz' in lists_location: try: d = zlib.decompressobj(zlib.MAX_WBITS | 32) lists_data = d.decompress(lists_data) except Exception: logger.error('CentOS mirror error: Could not decompress {0}\n' '{1}'.format(url, traceback.format_exc())) raise # Remove namespace attribute before parsing XML lists_data = re.sub(' xmlns="[^"]+"', '', lists_data, count=1) tree_lists_data = ElementTree.fromstring(lists_data) for flist in tree_lists_data.findall('package'): if flist.get('type') == 'rpm': flist_arch = flist.find('arch').text if flist_arch in self.centos_supported_archs: flist_name = flist.find('name').text flist_location = flist.find('location') flist_file = flist_location.get('href') flist_version = flist.find('version') flist_ver = '{0}-{1}'.format(flist_version.get('ver'), flist_version.get('rel')) cpkg = {'package:': flist_name, 'version:': flist_ver, 'filename:': flist_file} # TODO: add dependencies list to cpkg self.pkgs_list.append(cpkg) # Download packages (local_folder) def download_pkgs(self, pkgs_local_path): # Process the packages list: total_pkgs = len(self.pkgs_list) logger.info('Found {0} custom package(s)'.format(total_pkgs)) for npkg, pkg in enumerate(self.pkgs_list): # TODO: Previous versions of the updating packages must be removed # to avoid unwanted packet manager dependencies resolution # (when some package still depends on other package which # is not going to be installed) logger.info('({0}/{1}) Downloading package: {2}/{3}' .format(npkg + 1, total_pkgs, self.custom_pkgs_mirror, pkg["filename:"])) pkg_ext = pkg["filename:"].split('.')[-1] if pkg_ext == 'deb': path_suff = 'main/' elif pkg_ext == 'udeb': path_suff = 'debian-installer/' else: path_suff = '' wget_cmd = "wget --no-verbose --directory-prefix {0} {1}/{2}"\ .format(pkgs_local_path + path_suff, self.custom_pkgs_mirror, pkg["filename:"]) wget_result = self.ssh_manager.execute( ip=self.ip, cmd=wget_cmd ) assert_equal(0, wget_result['exit_code'], self.assert_msg(wget_cmd, wget_result['stderr'])) # Upload regenerate* script to masternode (script name) def regenerate_repo(self, regenerate_script, local_mirror_path): # Uploading scripts that prepare local repositories: # 'regenerate_centos_repo' and 'regenerate_ubuntu_repo' try: self.ssh_manager.upload_to_remote( ip=self.ip, source='{0}/{1}'.format(self.path_scripts, regenerate_script), target=self.remote_path_scripts ) self.ssh_manager.execute_on_remote( ip=self.ip, cmd='chmod 755 {0}/{1}'.format(self.remote_path_scripts, regenerate_script) ) except Exception: logger.error('Could not upload scripts for updating repositories.' '\n{0}'.format(traceback.format_exc())) raise # Update the local repository using previously uploaded script. script_cmd = '{0}/{1} {2} {3}'.format(self.remote_path_scripts, regenerate_script, local_mirror_path, self.ubuntu_release) script_result = self.ssh_manager.execute( ip=self.ip, cmd=script_cmd ) assert_equal(0, script_result['exit_code'], self.assert_msg(script_cmd, script_result['stderr'])) logger.info('Local repository {0} has been updated successfully.' .format(local_mirror_path)) def assert_msg(self, cmd, err): return 'Executing \'{0}\' on the admin node has failed with: {1}'\ .format(cmd, err) def check_puppet_logs(self): logger.info("Check puppet logs for packages with unmet dependencies.") if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: err_deps = self.check_puppet_logs_ubuntu() else: err_deps = self.check_puppet_logs_centos() for err_deps_key in err_deps.keys(): logger.info('Error: Package: {0} has unmet dependencies:' .format(err_deps_key)) for dep in err_deps[err_deps_key]: logger.info(' {0}'.format(dep.strip())) logger.info("Check puppet logs completed.") def check_puppet_logs_ubuntu(self): """ Check puppet-agent.log files on all nodes for package dependency errors during a cluster deployment (ubuntu)""" err_start = 'The following packages have unmet dependencies:' err_end = ('Unable to correct problems,' ' you have held broken packages.') cmd = ('fgrep -h -e " Depends: " -e "{0}" -e "{1}" ' '/var/log/docker-logs/remote/node-*/' 'puppet*.log'.format(err_start, err_end)) result = self.ssh_manager.execute( ip=self.ip, cmd=cmd )['stdout'] err_deps = {} err_deps_key = '' err_deps_flag = False # Forming a dictionary of package names # with sets of required packages. for res_str in result: if err_deps_flag: if err_end in res_str: err_deps_flag = False elif ": Depends:" in res_str: str0, str1, str2 = res_str.partition(': Depends:') err_deps_key = ''.join(str0.split()[-1:]) if err_deps_key not in err_deps: err_deps[err_deps_key] = set() if 'but it is not' in str2 or 'is to be installed' in str2: err_deps[err_deps_key].add('Depends:{0}' .format(str2)) elif 'Depends:' in res_str and err_deps_key: str0, str1, str2 = res_str.partition('Depends:') if 'but it is not' in str2 or 'is to be installed' in str2: err_deps[err_deps_key].add(str1 + str2) else: err_deps_key = '' elif err_start in res_str: err_deps_flag = True return err_deps def check_puppet_logs_centos(self): """ Check puppet-agent.log files on all nodes for package dependency errors during a cluster deployment (centos)""" cmd = ('fgrep -h -e "Error: Package: " -e " Requires: " /var/log/' 'docker-logs/remote/node-*/puppet*.log') result = self.ssh_manager.execute( ip=self.ip, cmd=cmd )['stdout'] err_deps = {} err_deps_key = '' # Forming a dictionary of package names # with sets of required packages. for res_str in result: if 'Error: Package:' in res_str: err_deps_key = res_str.partition('Error: Package: ')[2] if err_deps_key not in err_deps: err_deps[err_deps_key] = set() elif ' Requires: ' in res_str and err_deps_key: str0, str1, str2 = res_str.partition(' Requires: ') err_deps[err_deps_key].add(str1 + str2) else: err_deps_key = '' return err_deps
def update_ostf(): logger.info("Uploading new package from {0}".format( settings.UPDATE_FUEL_PATH)) ssh = SSHManager() pack_path = '/var/www/nailgun/fuel-ostf/' full_pack_path = os.path.join(pack_path, 'fuel-ostf*.noarch.rpm') ssh.upload_to_remote(ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Check old fuel-ostf package cmd = "rpm -q fuel-ostf" old_package = ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info('Current package version of ' 'fuel-ostf: {0}'.format(old_package)) cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info('Package from review {0}'.format(new_package)) if old_package == new_package: logger.info('Package {0} is installed'.format(new_package)) return cmd = "service ostf stop" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "service ostf status" helpers.wait(lambda: "dead" in ssh.execute_on_remote( ssh.admin_ip, cmd=cmd, raise_on_assert=False, assert_ec_equal=[3])[ 'stdout_str'], timeout=60) logger.info("OSTF status: inactive") cmd = "rpm -e fuel-ostf" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "rpm -q fuel-ostf" installed_package = ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)['stdout_str'] assert_equal( installed_package, new_package, "The new package {0} was not installed. Actual {1}".format( new_package, installed_package)) cmd = "service ostf start" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "service ostf status" helpers.wait(lambda: "running" in ssh.execute_on_remote( ssh.admin_ip, cmd=cmd)['stdout_str'], timeout=60) cmd = "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8777" helpers.wait(lambda: "401" in ssh.execute_on_remote( ssh.admin_ip, cmd=cmd, raise_on_assert=False)['stdout_str'], timeout=60) logger.info("OSTF status: RUNNING")
class CustomRepo(object): """CustomRepo.""" # TODO documentation def __init__(self): self.ssh_manager = SSHManager() self.ip = self.ssh_manager.admin_ip self.path_scripts = ('{0}/fuelweb_test/helpers/' .format(os.environ.get("WORKSPACE", "./"))) self.remote_path_scripts = '/tmp/' self.ubuntu_script = 'regenerate_ubuntu_repo' self.centos_script = 'regenerate_centos_repo' self.local_mirror_ubuntu = settings.LOCAL_MIRROR_UBUNTU self.local_mirror_centos = settings.LOCAL_MIRROR_CENTOS self.ubuntu_release = settings.UBUNTU_RELEASE self.centos_supported_archs = ['noarch', 'x86_64'] self.pkgs_list = [] self.custom_pkgs_mirror_path = '' if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: # Trying to determine the root of Ubuntu repository pkgs_path = settings.CUSTOM_PKGS_MIRROR.split('/dists/') if len(pkgs_path) == 2: self.custom_pkgs_mirror = pkgs_path[0] self.custom_pkgs_mirror_path = '/dists/{}'.format(pkgs_path[1]) else: self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR else: self.custom_pkgs_mirror = settings.CUSTOM_PKGS_MIRROR def prepare_repository(self): """Prepare admin node to packages testing Scenario: 1. Temporary set nameserver to local router on admin node 2. Install tools to manage rpm/deb repository 3. Retrieve list of packages from custom repository 4. Download packages to local rpm/deb repository 5. Update .yaml file with new packages version 6. Re-generate repo using shell scripts on admin node """ # Check necessary settings and revert a snapshot if not self.custom_pkgs_mirror: return logger.info("Custom mirror with new packages: {0}" .format(settings.CUSTOM_PKGS_MIRROR)) if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: # Ubuntu master_tools = ['dpkg', 'dpkg-devel', 'dpkg-dev'] self.install_tools(master_tools) self.get_pkgs_list_ubuntu() pkgs_local_path = ('{0}/pool/' .format(self.local_mirror_ubuntu)) self.download_pkgs(pkgs_local_path) self.regenerate_repo(self.ubuntu_script, self.local_mirror_ubuntu) else: # CentOS master_tools = ['createrepo'] self.install_tools(master_tools) self.get_pkgs_list_centos() pkgs_local_path = '{0}/Packages/'.format(self.local_mirror_centos) self.download_pkgs(pkgs_local_path) self.regenerate_repo(self.centos_script, self.local_mirror_centos) # Install tools to masternode def install_tools(self, master_tools=None): if master_tools is None: master_tools = [] logger.info("Installing necessary tools for {0}" .format(settings.OPENSTACK_RELEASE)) for master_tool in master_tools: exit_code = install_pkg_2( ip=self.ip, pkg_name=master_tool ) assert_equal(0, exit_code, 'Cannot install package {0} ' 'on admin node.'.format(master_tool)) # Ubuntu: Creating list of packages from the additional mirror def get_pkgs_list_ubuntu(self): url = "{0}/{1}/Packages".format(self.custom_pkgs_mirror, self.custom_pkgs_mirror_path) logger.info("Retrieving additional packages from the custom mirror:" " {0}".format(url)) try: pkgs_release = urlopen(url).read() except (HTTPError, URLError): logger.error(traceback.format_exc()) url_gz = '{0}.gz'.format(url) logger.info( "Retrieving additional packages from the custom mirror:" " {0}".format(url_gz)) try: pkgs_release_gz = urlopen(url_gz).read() except (HTTPError, URLError): logger.error(traceback.format_exc()) raise try: d = zlib.decompressobj(zlib.MAX_WBITS | 32) pkgs_release = d.decompress(pkgs_release_gz) except Exception: logger.error('Ubuntu mirror error: Could not decompress {0}\n' '{1}'.format(url_gz, traceback.format_exc())) raise packages = (pkg for pkg in pkgs_release.split("\n\n") if pkg) for package in packages: upkg = {pstr.split()[0].lower(): ''.join(pstr.split()[1:]) for pstr in package.split("\n") if pstr[0].strip()} upkg_keys = ["package:", "version:", "filename:"] assert_equal(True, all(x in upkg for x in upkg_keys), 'Missing one of the statements ["Package:", ' '"Version:", "Filename:"] in {0}'.format(url)) # TODO: add dependencies list to upkg self.pkgs_list.append(upkg) # Centos: Creating list of packages from the additional mirror def get_pkgs_list_centos(self): logger.info("Retrieving additional packages from the custom mirror:" " {0}".format(self.custom_pkgs_mirror)) url = "{0}/repodata/repomd.xml".format(self.custom_pkgs_mirror) try: repomd_data = urlopen(url).read() except (HTTPError, URLError): logger.error(traceback.format_exc()) raise # Remove namespace attribute before parsing XML repomd_data = re.sub(' xmlns="[^"]+"', '', repomd_data, count=1) tree_repomd_data = ElementTree.fromstring(repomd_data) lists_location = '' for repomd in tree_repomd_data.findall('data'): if repomd.get('type') == 'primary': repomd_location = repomd.find('location') lists_location = repomd_location.get('href') assert_equal(True, lists_location is not '', 'CentOS mirror error:' ' Could not parse {0}\nlists_location = "{1}"\n{2}' .format(url, lists_location, traceback.format_exc())) url = "{0}/{1}".format(self.custom_pkgs_mirror, lists_location) try: lists_data = urlopen(url).read() except (HTTPError, URLError): logger.error(traceback.format_exc()) raise if '.xml.gz' in lists_location: try: d = zlib.decompressobj(zlib.MAX_WBITS | 32) lists_data = d.decompress(lists_data) except Exception: logger.error('CentOS mirror error: Could not decompress {0}\n' '{1}'.format(url, traceback.format_exc())) raise # Remove namespace attribute before parsing XML lists_data = re.sub(' xmlns="[^"]+"', '', lists_data, count=1) tree_lists_data = ElementTree.fromstring(lists_data) for flist in tree_lists_data.findall('package'): if flist.get('type') == 'rpm': flist_arch = flist.find('arch').text if flist_arch in self.centos_supported_archs: flist_name = flist.find('name').text flist_location = flist.find('location') flist_file = flist_location.get('href') flist_version = flist.find('version') flist_ver = '{0}-{1}'.format(flist_version.get('ver'), flist_version.get('rel')) cpkg = {'package:': flist_name, 'version:': flist_ver, 'filename:': flist_file} # TODO: add dependencies list to cpkg self.pkgs_list.append(cpkg) # Download packages (local_folder) def download_pkgs(self, pkgs_local_path): # Process the packages list: total_pkgs = len(self.pkgs_list) logger.info('Found {0} custom package(s)'.format(total_pkgs)) for npkg, pkg in enumerate(self.pkgs_list): # TODO: Previous versions of the updating packages must be removed # to avoid unwanted packet manager dependencies resolution # (when some package still depends on other package which # is not going to be installed) logger.info('({0}/{1}) Downloading package: {2}/{3}' .format(npkg + 1, total_pkgs, self.custom_pkgs_mirror, pkg["filename:"])) pkg_ext = pkg["filename:"].split('.')[-1] if pkg_ext == 'deb': path_suff = 'main/' elif pkg_ext == 'udeb': path_suff = 'debian-installer/' else: path_suff = '' wget_cmd = "wget --no-verbose --directory-prefix {0} {1}/{2}"\ .format(pkgs_local_path + path_suff, self.custom_pkgs_mirror, pkg["filename:"]) wget_result = self.ssh_manager.execute( ip=self.ip, cmd=wget_cmd ) assert_equal(0, wget_result['exit_code'], self.assert_msg(wget_cmd, wget_result['stderr'])) # Upload regenerate* script to masternode (script name) def regenerate_repo(self, regenerate_script, local_mirror_path): # Uploading scripts that prepare local repositories: # 'regenerate_centos_repo' and 'regenerate_ubuntu_repo' try: self.ssh_manager.upload_to_remote( ip=self.ip, source='{0}/{1}'.format(self.path_scripts, regenerate_script), target=self.remote_path_scripts ) self.ssh_manager.execute_on_remote( ip=self.ip, cmd='chmod 755 {0}/{1}'.format(self.remote_path_scripts, regenerate_script) ) except Exception: logger.error('Could not upload scripts for updating repositories.' '\n{0}'.format(traceback.format_exc())) raise # Update the local repository using previously uploaded script. script_cmd = '{0}/{1} {2} {3}'.format(self.remote_path_scripts, regenerate_script, local_mirror_path, self.ubuntu_release) script_result = self.ssh_manager.execute( ip=self.ip, cmd=script_cmd ) assert_equal(0, script_result['exit_code'], self.assert_msg(script_cmd, script_result['stderr'])) logger.info('Local repository {0} has been updated successfully.' .format(local_mirror_path)) @staticmethod def assert_msg(cmd, err): return 'Executing \'{0}\' on the admin node has failed with: {1}'\ .format(cmd, err) def check_puppet_logs(self): logger.info("Check puppet logs for packages with unmet dependencies.") if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: err_deps = self.check_puppet_logs_ubuntu() else: err_deps = self.check_puppet_logs_centos() for err_deps_key in err_deps.keys(): logger.info('Error: Package: {0} has unmet dependencies:' .format(err_deps_key)) for dep in err_deps[err_deps_key]: logger.info(' {0}'.format(dep.strip())) logger.info("Check puppet logs completed.") def check_puppet_logs_ubuntu(self): """ Check puppet-agent.log files on all nodes for package dependency errors during a cluster deployment (ubuntu)""" err_start = 'The following packages have unmet dependencies:' err_end = ('Unable to correct problems,' ' you have held broken packages.') cmd = ('fgrep -h -e " Depends: " -e "{0}" -e "{1}" ' '/var/log/remote/node-*/' 'puppet*.log'.format(err_start, err_end)) result = self.ssh_manager.execute( ip=self.ip, cmd=cmd )['stdout'] err_deps = {} err_deps_key = '' err_deps_flag = False # Forming a dictionary of package names # with sets of required packages. for res_str in result: if err_deps_flag: if err_end in res_str: err_deps_flag = False elif ": Depends:" in res_str: str0, str1, str2 = res_str.partition(': Depends:') err_deps_key = ''.join(str0.split()[-1:]) if err_deps_key not in err_deps: err_deps[err_deps_key] = set() if 'but it is not' in str2 or 'is to be installed' in str2: err_deps[err_deps_key].add('Depends:{0}' .format(str2)) elif 'Depends:' in res_str and err_deps_key: str0, str1, str2 = res_str.partition('Depends:') if 'but it is not' in str2 or 'is to be installed' in str2: err_deps[err_deps_key].add(str1 + str2) else: err_deps_key = '' elif err_start in res_str: err_deps_flag = True return err_deps def check_puppet_logs_centos(self): """ Check puppet-agent.log files on all nodes for package dependency errors during a cluster deployment (centos)""" cmd = ('fgrep -h -e "Error: Package: " -e " Requires: " /var/log/' 'remote/node-*/puppet*.log') result = self.ssh_manager.execute( ip=self.ip, cmd=cmd )['stdout'] err_deps = {} err_deps_key = '' # Forming a dictionary of package names # with sets of required packages. for res_str in result: if 'Error: Package:' in res_str: err_deps_key = res_str.partition('Error: Package: ')[2] if err_deps_key not in err_deps: err_deps[err_deps_key] = set() elif ' Requires: ' in res_str and err_deps_key: _, str1, str2 = res_str.partition(' Requires: ') err_deps[err_deps_key].add(str1 + str2) else: err_deps_key = '' return err_deps
class EnvironmentModel(six.with_metaclass(SingletonMeta, object)): """EnvironmentModel.""" # TODO documentation def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), admin_login=settings.SSH_FUEL_CREDENTIALS['login'], admin_password=settings.SSH_FUEL_CREDENTIALS['password'], slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, '_fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout, timeout_msg='Bootstrap timeout for nodes: {}' ''.format([node.name for node in devops_nodes])) wait_pass( lambda: checkers.validate_minimal_amount_nodes( nodes=self.nailgun_nodes(devops_nodes), expected_amount=len(devops_nodes) ), timeout=30) if not skip_timesync: self.sync_time() return self.nailgun_nodes(devops_nodes) def sync_time(self, nodes_names=None, skip_sync=False): if nodes_names is None: roles = ['fuel_master', 'fuel_slave'] nodes_names = [node.name for node in self.d_env.get_nodes() if node.role in roles and node.driver.node_active(node)] logger.info("Please wait while time on nodes: {0} " "will be synchronized" .format(', '.join(sorted(nodes_names)))) new_time = sync_time(self.d_env, nodes_names, skip_sync) for name in sorted(new_time): logger.info("New time on '{0}' = {1}".format(name, new_time[name])) @logwrap def get_admin_node_ip(self): return str( self.d_env.nodes().admin.get_ip_address_by_network_name('admin')) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'device_label': settings.ISO_LABEL, 'iface': iface_alias('eth0'), 'ip': node.get_ip_address_by_network_name('admin'), 'mask': self.d_env.get_network(name='admin').ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': '', 'nameserver': settings.DNS, 'showmenu': 'yes' if settings.SHOW_FUELMENU else 'no', 'wait_for_external_config': 'yes', 'build_images': '1' if build_images else '0', 'MASTER_NODE_EXTRA_PACKAGES': settings.MASTER_NODE_EXTRA_PACKAGES } # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" ) else: # cdrom is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" ) keys += ( "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img" " inst.ks=cdrom:LABEL=%(device_label)s:/ks.cfg" " inst.repo=cdrom:LABEL=%(device_label)s:/" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":%(iface)s:off::: nameserver=%(nameserver)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s" " build_images=%(build_images)s\n" " MASTER_NODE_EXTRA_PACKAGES='%(MASTER_NODE_EXTRA_PACKAGES)s'\n" " <Enter>\n" ) % params return keys @staticmethod def get_target_devs(devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config[ 'template']['devops_settings']['env_name']) except Exception: self._virt_env = Environment.create_environment( full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin self.ssh_manager.clean_all_connections() try: admin.await('admin', timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin.await('admin') self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend() time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return [self.fuel_web.get_nailgun_node_by_devops_node(node) for node in devops_nodes] def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) self.fuel_web.wait_nodes_get_online_state(devops_nodes, timeout=60 * 6) return True def revert_snapshot(self, name, skip_timesync=False, skip_slaves_check=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: {:s}'.format(name)) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: self.sync_time() else: self.sync_time(['admin']) try: with QuietLogger(upper_log_level=logging.CRITICAL): # TODO(astudenov): add timeout_msg wait_pass( self.fuel_web.client.get_releases, expected=( exceptions.RetriableConnectionFailure, exceptions.UnknownConnectionError), timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() if not skip_slaves_check: # TODO(astudenov): add timeout_msg wait_pass(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_FUEL_CREDENTIALS['login'] new_password = settings.SSH_FUEL_CREDENTIALS['password'] try: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='date' ) logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login='******', admin_password='******', slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, new_password) ) self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login=new_login, admin_password=new_password, slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.ssh_manager.update_connection( ip=self.ssh_manager.admin_ip, login=new_login, password=new_password ) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password']) ) config_file = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='ls -1 $HOME/.config/fuel/fuel_client.yaml')['stdout_str'] with YamlEditor(config_file, ip=self.admin_node_ip) as editor: editor.content["OS_USERNAME"] = \ settings.KEYSTONE_CREDS['username'] editor.content["OS_PASSWORD"] = \ settings.KEYSTONE_CREDS['password'] with YamlEditor(settings.FUEL_SETTINGS_YAML, ip=self.admin_node_ip) as editor: editor.content["FUEL_ACCESS"]['user'] = \ settings.KEYSTONE_CREDS['username'] editor.content["FUEL_ACCESS"]['password'] = \ settings.KEYSTONE_CREDS['password'] logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # Create environment and start the Fuel master node admin = self.d_env.nodes().admin self.d_env.start([admin]) def provision_admin(admin_node): logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin_node), 60, timeout_msg='Admin node startup timeout') logger.info("Proceed with installation") # update network parameters at boot screen admin_node.send_keys(self.get_keys( admin_node, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if settings.SHOW_FUELMENU: self.wait_for_fuelmenu() else: self.wait_for_provisioning() try: provision_admin(admin) except Exception as e: logger.info('Master node restart: LP1587411') logger.info('Exception is: {e}'.format(e=e)) admin.reset() provision_admin(admin) self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) if CUSTOM_FUEL_SETTING_YAML: self.admin_actions.update_fuel_setting_yaml( CUSTOM_FUEL_SETTING_YAML) self.kill_wait_for_external_config() self.wait_bootstrap() self.admin_actions.wait_for_fuel_ready() @logwrap def enable_force_https(self, admin_node_ip): cmd = """ echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml """ self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = "find / -name \"nginx_services.pp\"" puppet_manifest = \ self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0].strip() cmd = 'puppet apply {0}'.format(puppet_manifest) self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = """ systemctl status nginx.service | awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}' """ wait(lambda: ( self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0] != 'dead'), interval=10, timeout=30, timeout_msg='Nginx service is dead after trying to enable ' 'it with the command: {}'.format(cmd)) # pylint: disable=no-self-use @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...') # pylint: enable=no-self-use @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): # TODO(astudenov): add timeout_msg wait_pass(lambda: tcp_ping_( self.d_env.nodes( ).admin.get_ip_address_by_network_name ('admin'), 22), timeout=timeout) @logwrap def wait_for_fuelmenu(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): def check_ssh_connection(): """Try to close fuelmenu and check ssh connection""" try: tcp_ping_( self.d_env.nodes().admin.get_ip_address_by_network_name( 'admin'), 22) except Exception: # send F8 trying to exit fuelmenu self.d_env.nodes().admin.send_keys("<F8>\n") return False return True wait(check_ssh_connection, interval=30, timeout=timeout, timeout_msg="Fuelmenu hasn't appeared during allocated timeout") @logwrap def wait_for_external_config(self, timeout=120): wait(lambda: self.ssh_manager.exists_on_remote( self.ssh_manager.admin_ip, '/var/lock/wait_for_external_config'), timeout=600, timeout_msg='wait_for_external_config lock file timeout ' 'while bootstrapping the Fuel master node') check_cmd = 'pkill -0 -f wait_for_external_config' wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=check_cmd)['exit_code'] == 0, timeout=timeout, timeout_msg='wait_for_external_config process timeout ' 'while bootstrapping the Fuel master node') @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=kill_cmd ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=check_cmd ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Running bootstrap (timeout: {0})".format( float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))) with TimeStat("admin_node_bootsrap_time", is_uniq=True): wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment' '{:s}'".format(log_path) )['exit_code'] == 0, timeout=(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT)), timeout_msg='Fuel master node bootstrap timeout, ' 'please check the log {}'.format(log_path) ) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment " "complete' '{:s}'".format(log_path))['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = iface_alias('eth0') command = "dhcpcheck discover " \ "--ifaces {iface} " \ "--repeat 3 " \ "--timeout 10".format(iface=iface) out = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=command )['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu': logger.warning('Default image for bootstrap ' 'is not based on Ubuntu!') return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel-bootstrap --quiet list' )['stdout'] assert_true(any('active' in line for line in bootstrap_images), 'Ubuntu bootstrap image wasn\'t built and activated! ' 'See logs in /var/log/fuel-bootstrap-image-build.log ' 'for details.') def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name) ) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name) ) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name) ) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name) ) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) def admin_install_updates(self): """Update packages using yum and install updates via update-master-node.sh tool""" logger.info('Searching for updates..') update_command = 'yum clean expire-cache && ' \ 'yum update -y 2>>/var/log/yum-update-error.log' logger.info('Performing yum clean and update commands') update_result = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=update_command, err_msg='Packages update failed, inspect logs for details') logger.info('Packages were updated successfully') # Check if any packets were updated and update was successful match_updated_count = re.search(r'Upgrade\s+(\d+)\s+Package', update_result['stdout_str']) # In case of package replacement, the new one is marked as # installed and the old one as removed match_installed_count = re.search(r'Install\s+(\d+)\s+Package', update_result['stdout_str']) match_complete_message = re.search(r'Complete!', update_result['stdout_str']) match_no_updates = re.search("No Packages marked for Update", update_result['stdout_str']) if match_no_updates or not match_complete_message \ or not (match_updated_count or match_installed_count): logger.warning('No updates were found or update was incomplete.') return updates_count = 0 if match_updated_count: updates_count += int(match_updated_count.group(1)) if match_installed_count: updates_count += int(match_installed_count.group(1)) logger.info('{0} package(s) were updated'.format(updates_count)) logger.info('Applying updates via update-master-node.sh') cmd = '/usr/share/fuel-utils/update-master-node.sh' self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd, err_msg='Update failed, inspect logs for details', ) logger.info('Update successful') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] with self.ssh_manager.open_on_remote( ip=self.ssh_manager.admin_ip, path='/etc/resolv.conf', ) as f: resolv_conf = f.readlines() if merge: nameservers.extend(resolv_conf) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join( '{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) with self.ssh_manager.open_on_remote( ip=self.ssh_manager.admin_ip, path='/etc/resolv.conf', mode='w' ) as f: f.write(resolv_new) return resolv_conf @logwrap def describe_other_admin_interfaces(self, admin): admin_networks = [iface.network.name for iface in admin.interfaces] iface_name = None for i, network_name in enumerate(admin_networks): if 'admin' in network_name and 'admin' != network_name: # This will be replaced with actual interface labels # form fuel-devops iface_name = 'enp0s' + str(i + 3) logger.info("Describe Fuel admin node interface {0} for " "network {1}".format(iface_name, network_name)) self.describe_admin_interface(iface_name, network_name) if iface_name: return self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="cobbler sync") @logwrap def describe_admin_interface(self, admin_if, network_name): admin_net_object = self.d_env.get_network(name=network_name) admin_network = admin_net_object.ip.network admin_netmask = admin_net_object.ip.netmask admin_ip = str(self.d_env.nodes( ).admin.get_ip_address_by_network_name(network_name)) logger.info(('Parameters for admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(admin_network, admin_netmask, admin_if, admin_ip)) add_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(admin_if, admin_ip, admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_admin_ip, admin_if, admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(admin_ip, admin_if)) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp( self.ssh_manager.admin_ip, admin_if ) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, admin_network, admin_netmask, admin_if, self.get_admin_node_ip() ) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
def replace_fuel_nailgun_rpm(): """ Replace fuel_nailgun*.rpm from review """ logger.info("Patching fuel-nailgun") ssh = SSHManager() if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') pack_path = '/var/www/nailgun/fuel-nailgun/' full_pack_path = os.path.join(pack_path, 'fuel-nailgun*.noarch.rpm') logger.info('Package path {0}'.format(full_pack_path)) ssh.upload_to_remote(ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Check old fuel-nailgun package cmd = "rpm -q fuel-nailgun" old_package = ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info('Current package version of ' 'fuel-nailgun: {0}'.format(old_package)) cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote(ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("Updating package {0} with {1}".format(old_package, new_package)) if old_package == new_package: logger.debug('Looks like package from review ' 'was installed during setups of master node') return # stop services service_list = ['assassind', 'receiverd', 'nailgun', 'statsenderd'] for service in service_list: ssh.execute_on_remote(ip=ssh.admin_ip, cmd='systemctl stop {0}'.format(service)) logger.info('statistic services {0}'.format(get_oswl_services_names())) # stop statistic services for service in get_oswl_services_names(): ssh.execute_on_remote(ip=ssh.admin_ip, cmd='systemctl stop {0}'.format(service)) # Drop nailgun db manage.py dropdb cmd = 'manage.py dropdb' ssh.execute_on_remote(ssh.admin_ip, cmd) # Delete package logger.info("Delete package {0}".format(old_package)) cmd = "rpm -e fuel-nailgun" ssh.execute_on_remote(ssh.admin_ip, cmd) logger.info("Install package {0}".format(new_package)) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd) cmd = "rpm -q fuel-nailgun" installed_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] assert_equal(installed_package, new_package, "The new package {0} was not installed".format(new_package)) cmd = ('puppet apply --debug ' '/etc/puppet/modules/fuel/examples/nailgun.pp') ssh.execute_on_remote(ssh.admin_ip, cmd) cmd_sync = 'fuel release --sync-deployment-tasks --dir /etc/puppet/' ssh.execute_on_remote(ssh.admin_ip, cmd=cmd_sync)
def update_ostf(): logger.info("Uploading new package from {0}".format( settings.UPDATE_FUEL_PATH)) ssh = SSHManager() pack_path = '/var/www/nailgun/fuel-ostf/' full_pack_path = os.path.join(pack_path, 'fuel-ostf*.noarch.rpm') ssh.upload_to_remote( ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Check old fuel-ostf package cmd = "rpm -q fuel-ostf" old_package = ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info( 'Current package version of ' 'fuel-ostf: {0}'.format(old_package)) cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info('Package from review {0}'.format(new_package)) if old_package == new_package: logger.info('Package {0} is installed'.format(new_package)) return cmd = "service ostf stop" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "service ostf status" helpers.wait(lambda: "dead" in ssh.execute_on_remote( ssh.admin_ip, cmd=cmd, raise_on_assert=False, assert_ec_equal=[3])['stdout_str'], timeout=60) logger.info("OSTF status: inactive") cmd = "rpm -e fuel-ostf" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "rpm -q fuel-ostf" installed_package = ssh.execute_on_remote( ssh.admin_ip, cmd=cmd)['stdout_str'] assert_equal( installed_package, new_package, "The new package {0} was not installed. Actual {1}".format( new_package, installed_package)) cmd = "service ostf start" ssh.execute_on_remote(ssh.admin_ip, cmd=cmd) cmd = "service ostf status" helpers.wait( lambda: "running" in ssh.execute_on_remote(ssh.admin_ip, cmd=cmd)['stdout_str'], timeout=60) cmd = "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8777" helpers.wait( lambda: "401" in ssh.execute_on_remote( ssh.admin_ip, cmd=cmd, raise_on_assert=False)['stdout_str'], timeout=60) logger.info("OSTF status: RUNNING")
def check_package_version_injected_in_bootstraps( package, cluster_id=None, ironic=None): ssh = SSHManager() try: pack_path = '/var/www/nailgun/{}/'.format(package) ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) except Exception: logger.exception("Could not upload package") raise # Step 1 - unpack active bootstrap logger.info("unpack active bootstrap") if ironic: bootstrap = "/var/www/nailgun/bootstrap/ironic/{}".format(cluster_id) else: bootstrap = "/var/www/nailgun/bootstraps/active_bootstrap" bootstrap_var = "/var/root.squashfs" cmd = "unsquashfs -d {} {}/root.squashfs".format( bootstrap_var, bootstrap) ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd) # Step 2 - check package version logger.info( "check package {} version injected in ubuntu bootstrap".format( package)) cmd = "ls {}|grep {} |grep deb |cut -f 2 -d '_'".format( pack_path, package) package_from_review = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("package from review is {}".format(package_from_review)) awk_pattern = "awk '{print $2}'" cmd = "chroot {}/ /bin/bash -c \"dpkg -s {}\"|grep Version|{}".format( bootstrap_var, package, awk_pattern) installed_package = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("injected package is {}".format(installed_package)) assert_equal(installed_package, package_from_review, "The new package {0} wasn't injected in bootstrap".format( package_from_review)) # Step 3 - remove unpacked bootstrap cmd = "rm -rf {}".format(bootstrap_var) ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)
def replace_fuel_nailgun_rpm(): """ Replace fuel_nailgun*.rpm from review """ logger.info("Patching fuel-nailgun") ssh = SSHManager() if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') pack_path = '/var/www/nailgun/fuel-nailgun/' full_pack_path = os.path.join(pack_path, 'fuel-nailgun*.noarch.rpm') logger.info('Package path {0}'.format(full_pack_path)) ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=pack_path) # Check old fuel-nailgun package cmd = "rpm -q fuel-nailgun" old_package = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info( 'Current package version of ' 'fuel-nailgun: {0}'.format(old_package)) cmd = "rpm -qp {0}".format(full_pack_path) new_package = ssh.execute_on_remote( ip=ssh.admin_ip, cmd=cmd)['stdout_str'] logger.info("Updating package {0} with {1}".format( old_package, new_package)) if old_package == new_package: logger.debug('Looks like package from review ' 'was installed during setups of master node') return # stop services service_list = ['assassind', 'receiverd', 'nailgun', 'statsenderd'] for service in service_list: ssh.execute_on_remote( ip=ssh.admin_ip, cmd='systemctl stop {0}'.format(service)) logger.info('statistic services {0}'.format(get_oswl_services_names())) # stop statistic services for service in get_oswl_services_names(): ssh.execute_on_remote( ip=ssh.admin_ip, cmd='systemctl stop {0}'.format(service)) # Drop nailgun db manage.py dropdb cmd = 'manage.py dropdb' ssh.execute_on_remote(ssh.admin_ip, cmd) # Delete package logger.info("Delete package {0}".format(old_package)) cmd = "rpm -e fuel-nailgun" ssh.execute_on_remote(ssh.admin_ip, cmd) logger.info("Install package {0}".format(new_package)) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) ssh.execute_on_remote(ssh.admin_ip, cmd) cmd = "rpm -q fuel-nailgun" installed_package = ssh.execute_on_remote(ssh.admin_ip, cmd)['stdout_str'] assert_equal(installed_package, new_package, "The new package {0} was not installed".format(new_package)) cmd = ('puppet apply --debug ' '/etc/puppet/modules/fuel/examples/nailgun.pp') ssh.execute_on_remote(ssh.admin_ip, cmd) cmd_sync = 'fuel release --sync-deployment-tasks --dir /etc/puppet/' ssh.execute_on_remote(ssh.admin_ip, cmd=cmd_sync)
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, '_fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time() return self.nailgun_nodes(devops_nodes) def sync_time(self, nodes_names=None, skip_sync=False): if nodes_names is None: roles = ['fuel_master', 'fuel_slave'] nodes_names = [ node.name for node in self.d_env.get_nodes() if node.role in roles and node.driver.node_active(node) ] logger.info("Please wait while time on nodes: {0} " "will be synchronized".format(', '.join( sorted(nodes_names)))) new_time = sync_time(self.d_env, nodes_names, skip_sync) for name in sorted(new_time): logger.info("New time on '{0}' = {1}".format(name, new_time[name])) @logwrap def get_admin_node_ip(self): return str(self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'device_label': settings.ISO_LABEL, 'iface': iface_alias('eth0'), 'ip': node.get_ip_address_by_network_name(self.d_env.admin_net), 'mask': self.d_env.get_network(name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': self.d_env.nat_interface, 'nameserver': settings.DNS, 'showmenu': 'yes' if settings.SHOW_FUELMENU else 'no', 'wait_for_external_config': 'yes', 'build_images': '1' if build_images else '0' } # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n") else: # cdrom is default keys = ("<Wait>\n" "<Wait>\n" "<Wait>\n") keys += ("<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img" " inst.ks=cdrom:LABEL=%(device_label)s:/ks.cfg" " inst.repo=cdrom:LABEL=%(device_label)s:/" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":%(iface)s:off::: nameserver=%(nameserver)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s" " build_images=%(build_images)s\n" " <Enter>\n") % params return keys @staticmethod def get_target_devs(devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config['template'] ['devops_settings']['env_name']) except Exception: self._virt_env = Environment.create_environment( full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin self.ssh_manager.clean_all_connections() try: admin. await (self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin. await (self.d_env.admin_net) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend() time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return [ self.fuel_web.get_nailgun_node_by_devops_node(node) for node in devops_nodes ] def check_slaves_are_ready(self): devops_nodes = [ node for node in self.d_env.nodes().slaves if node.driver.node_active(node) ] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError("Node {0} does not become online".format( node.name)) return True def revert_snapshot(self, name, skip_timesync=False, skip_slaves_check=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: {:s}'.format(name)) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: self.sync_time() try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() if not skip_slaves_check: _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_CREDENTIALS['login'] new_password = settings.SSH_CREDENTIALS['password'] try: self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd='date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') self.ssh_manager.initialize(admin_ip=self.ssh_manager.admin_ip, login='******', password='******') self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format( new_login, new_password)) self.ssh_manager.initialize(admin_ip=self.ssh_manager.admin_ip, login=new_login, password=new_password) self.ssh_manager.update_connection(ip=self.ssh_manager.admin_ip, login=new_login, password=new_password) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format( new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'. format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def insert_cdrom_tray(self): # This is very rude implementation and it SHOULD be changes after # implementation this feature in fuel-devops name = "{}_{}".format(settings.ENV_NAME, self.d_env.nodes().admin.name) name_size = 80 if len(name) > name_size: hash_str = str(hash(name)) name = (hash_str + name)[:name_size] cmd = """EDITOR="sed -i s/tray=\\'open\\'//" virsh edit {}""".format( name) subprocess.check_call(cmd, shell=True) def reinstall_master_node(self): """Erase boot sector and run setup_environment""" with self.d_env.get_admin_remote() as remote: erase_data_from_hdd(remote, mount_point='/boot') remote.execute("/sbin/shutdown") self.d_env.nodes().admin.destroy() self.insert_cdrom_tray() self.setup_environment() def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST, force_ssl=settings.FORCE_HTTPS_MASTER_NODE): # Create environment and start the Fuel master node admin = self.d_env.nodes().admin self.d_env.start([admin]) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys( self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if settings.SHOW_FUELMENU: self.wait_for_fuelmenu() else: self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() if settings.UPDATE_FUEL: # Update Ubuntu packages self.admin_actions.upload_packages( local_packages_dir=settings.UPDATE_FUEL_PATH, centos_repo_path=None, ubuntu_repo_path=settings.LOCAL_MIRROR_UBUNTU) self.admin_actions.wait_for_fuel_ready() time.sleep(10) self.set_admin_keystone_password() self.sync_time(['admin']) if settings.UPDATE_MASTER: if settings.UPDATE_FUEL_MIRROR: for i, url in enumerate(settings.UPDATE_FUEL_MIRROR): conf_file = '/etc/yum.repos.d/temporary-{}.repo'.format(i) cmd = ("echo -e" " '[temporary-{0}]\nname=" "temporary-{0}\nbaseurl={1}/" "\ngpgcheck=0\npriority=" "1' > {2}").format(i, url, conf_file) self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) self.admin_install_updates() if settings.MULTIPLE_NETWORKS: self.describe_other_admin_interfaces(admin) self.nailgun_actions.set_collector_address(settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd to apply settings(Collector address) self.nailgun_actions.force_fuel_stats_sending() if settings.FUEL_STATS_ENABLED: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) if settings.PATCHING_DISABLE_UPDATES: cmd = "find /etc/yum.repos.d/ -type f -regextype posix-egrep" \ " -regex '.*/mos[0-9,\.]+\-(updates|security).repo' | " \ "xargs -n1 -i sed '$aenabled=0' -i {}" self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=cmd) if settings.DISABLE_OFFLOADING: logger.info('========================================' 'Applying workaround for bug #1526544' '========================================') # Disable TSO offloading for every network interface # that is not virtual (loopback, bridges, etc) ifup_local = ( """#!/bin/bash\n""" """if [[ -z "${1}" ]]; then\n""" """ exit\n""" """fi\n""" """devpath=$(readlink -m /sys/class/net/${1})\n""" """if [[ "${devpath}" == /sys/devices/virtual/* ]]; then\n""" """ exit\n""" """fi\n""" """ethtool -K ${1} tso off\n""") cmd = ("echo -e '{0}' | sudo tee /sbin/ifup-local;" "sudo chmod +x /sbin/ifup-local;").format(ifup_local) self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=cmd) cmd = ('for ifname in $(ls /sys/class/net); do ' 'sudo /sbin/ifup-local ${ifname}; done') self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=cmd) # Log interface settings cmd = ('for ifname in $(ls /sys/class/net); do ' '([[ $(readlink -e /sys/class/net/${ifname}) == ' '/sys/devices/virtual/* ]] ' '|| ethtool -k ${ifname}); done') result = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd) logger.debug('Offloading settings:\n{0}\n'.format(''.join( result['stdout']))) if force_ssl: self.enable_force_https(self.ssh_manager.admin_ip) @logwrap def enable_force_https(self, admin_node_ip): cmd = """ echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml """ self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = "find / -name \"nginx_services.pp\"" puppet_manifest = \ self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0].strip() cmd = 'puppet apply {0}'.format(puppet_manifest) self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = """ systemctl status nginx.service | awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}' """ wait(lambda: (self.ssh_manager.execute_on_remote(admin_node_ip, cmd)[ 'stdout'][0] != 'dead'), interval=10, timeout=30) # pylint: disable=no-self-use @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...') # pylint: enable=no-self-use @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): _wait(lambda: _tcp_ping( self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net), 22), timeout=timeout) @logwrap def wait_for_fuelmenu(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): def check_ssh_connection(): """Try to close fuelmenu and check ssh connection""" try: _tcp_ping( self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net), 22) except Exception: # send F8 trying to exit fuelmenu self.d_env.nodes().admin.send_keys("<F8>\n") return False return True wait(check_ssh_connection, interval=30, timeout=timeout, timeout_msg="Fuelmenu hasn't appeared during allocated timeout") @logwrap def wait_for_external_config(self, timeout=120): check_cmd = 'pkill -0 -f wait_for_external_config' wait(lambda: self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=check_cmd)['exit_code'] == 0, timeout=timeout) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=kill_cmd) self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=check_cmd) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Running bootstrap (timeout: {0})".format( float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))) with TimeStat("admin_node_bootsrap_time", is_uniq=True): wait(lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment' '{:s}'".format(log_path))[ 'exit_code'] == 0, timeout=(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment " "complete' '{:s}'".format(log_path))['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = iface_alias('eth0') command = "dhcpcheck discover " \ "--ifaces {iface} " \ "--repeat 3 " \ "--timeout 10".format(iface=iface) out = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=command)['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu': logger.warning('Default image for bootstrap ' 'is not based on Ubuntu!') return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel-bootstrap --quiet list')['stdout'] assert_true( any('active' in line for line in bootstrap_images), 'Ubuntu bootstrap image wasn\'t built and activated! ' 'See logs in /var/log/fuel-bootstrap-image-build.log ' 'for details.') def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}".format( pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}".format( service_name, remote_status['exit_code'], remote_status['stdout'])) # Execute yum updates # If updates installed, # then `bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info('Searching for updates..') update_command = 'yum clean expire-cache; yum update -y' update_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=update_command) logger.info('Result of "{1}" command on master node: ' '{0}'.format(update_result, update_command)) assert_equal(int(update_result['exit_code']), 0, 'Packages update failed, ' 'inspect logs for details') # Check if any packets were updated and update was successful for str_line in update_result['stdout']: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates)\ and not match_complete_message: logger.warning('No updates were found or update was incomplete.') return logger.info('{0} packet(s) were updated'.format(updates_count)) cmd = 'bootstrap_admin_node.sh;' result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) logger.info('Result of "{1}" command on master node: ' '{0}'.format(result, cmd)) assert_equal(int(result['exit_code']), 0, 'bootstrap failed, ' 'inspect logs for details') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd='cat /etc/resolv.conf') assert_equal( 0, resolv_conf['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}'.format( 'cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=echo_cmd) assert_equal( 0, echo_result['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}'.format( echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @staticmethod @logwrap def execute_remote_cmd(remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal( result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format(cmd, result)) return result['stdout'] @logwrap def describe_other_admin_interfaces(self, admin): admin_networks = [iface.network.name for iface in admin.interfaces] iface_name = None for i, network_name in enumerate(admin_networks): if 'admin' in network_name and 'admin' != network_name: # This will be replaced with actual interface labels # form fuel-devops iface_name = 'enp0s' + str(i + 3) logger.info("Describe Fuel admin node interface {0} for " "network {1}".format(iface_name, network_name)) self.describe_admin_interface(iface_name, network_name) if iface_name: return self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="cobbler sync") @logwrap def describe_admin_interface(self, admin_if, network_name): admin_net_object = self.d_env.get_network(name=network_name) admin_network = admin_net_object.ip.network admin_netmask = admin_net_object.ip.netmask admin_ip = str(self.d_env.nodes().admin.get_ip_address_by_network_name( network_name)) logger.info(('Parameters for admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(admin_network, admin_netmask, admin_if, admin_ip)) add_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(admin_if, admin_ip, admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_admin_ip, admin_if, admin_ip) logger.debug( 'Trying to assign {0} IP to the {1} on master node...'.format( admin_ip, admin_if)) result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp( self.ssh_manager.admin_ip, admin_if) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, admin_network, admin_netmask, admin_if, self.get_admin_node_ip()) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(EnvironmentModel, cls).__new__( cls, *args, **kwargs) return cls._instance def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password'] ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.docker_actions = DockerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, '_fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=900, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) if not MASTER_IS_CENTOS7: with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait( lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) else: wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time([node for node in self.nailgun_nodes(devops_nodes)]) return self.nailgun_nodes(devops_nodes) @logwrap def get_admin_node_ip(self): return str( self.d_env.nodes( ).admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'ks': 'hd:LABEL="Mirantis_Fuel":/ks.cfg' if iso_connect_as == 'usb' else 'cdrom:/ks.cfg', 'repo': 'hd:LABEL="Mirantis_Fuel":/', # only required for USB boot 'ip': node.get_ip_address_by_network_name( self.d_env.admin_net), 'mask': self.d_env.get_network( name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': self.d_env.nat_interface, 'dns1': settings.DNS, 'showmenu': 'no', 'wait_for_external_config': 'yes', 'build_images': '1' if build_images else '0' } if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params else: # cdrom case is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n" ) % params if MASTER_IS_CENTOS7: # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. iface = 'enp0s3' if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":{iface}:off::: dns1=%(dns1)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n".format(iface=iface) ) % params else: # cdrom case is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":{iface}:off::: dns1=%(dns1)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n".format(iface=iface) ) % params return keys def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config[ 'template']['devops_settings']['env_name']) except Exception: self._virt_env = Environment.create_environment( full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin try: admin.await(self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin.await(self.d_env.admin_net) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend(verbose=False) time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes ) def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError( "Node {0} does not become online".format(node.name)) return True def revert_snapshot(self, name, skip_timesync=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: %s' % name) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: nailgun_nodes = [self.fuel_web.get_nailgun_node_by_name(node.name) for node in self.d_env.nodes().slaves if node.driver.node_active(node)] self.sync_time(nailgun_nodes) try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_CREDENTIALS['login'] new_password = settings.SSH_CREDENTIALS['password'] try: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='date' ) logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') self.ssh_manager.update_connection( ip=self.ssh_manager.admin_ip, login='******', password='******' ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, new_password) ) self.ssh_manager.update_connection( ip=self.ssh_manager.admin_ip, login=new_login, password=new_password ) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password']) ) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # start admin node admin = self.d_env.nodes().admin if iso_connect_as == 'usb': admin.disk_devices.get(device='disk', bus='usb').volume.upload(settings.ISO_PATH) else: # cdrom is default admin.disk_devices.get( device='cdrom').volume.upload(settings.ISO_PATH) self.d_env.start(self.d_env.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() if settings.UPDATE_FUEL: # Update Ubuntu packages self.admin_actions.upload_packages( local_packages_dir=settings.UPDATE_FUEL_PATH, centos_repo_path=None, ubuntu_repo_path=settings.LOCAL_MIRROR_UBUNTU) self.docker_actions.wait_for_ready_containers() time.sleep(10) self.set_admin_keystone_password() if not MASTER_IS_CENTOS7: self.sync_time() if settings.UPDATE_MASTER: if settings.UPDATE_FUEL_MIRROR: for i, url in enumerate(settings.UPDATE_FUEL_MIRROR): conf_file = '/etc/yum.repos.d/temporary-{}.repo'.format(i) cmd = ("echo -e" " '[temporary-{0}]\nname=" "temporary-{0}\nbaseurl={1}/" "\ngpgcheck=0\npriority=" "1' > {2}").format(i, url, conf_file) self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) self.admin_install_updates() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() if not MASTER_IS_CENTOS7: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd to apply settings(Collector address) self.nailgun_actions.force_fuel_stats_sending() if settings.FUEL_STATS_ENABLED and not MASTER_IS_CENTOS7: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) if settings.PATCHING_DISABLE_UPDATES: cmd = "find /etc/yum.repos.d/ -type f -regextype posix-egrep" \ " -regex '.*/mos[0-9,\.]+\-(updates|security).repo' | " \ "xargs -n1 -i sed '$aenabled=0' -i {}" self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd ) @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...') @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): _wait(lambda: _tcp_ping( self.d_env.nodes( ).admin.get_ip_address_by_network_name (self.d_env.admin_net), 22), timeout=timeout) @logwrap def wait_for_external_config(self, timeout=120): check_cmd = 'pkill -0 -f wait_for_external_config' if MASTER_IS_CENTOS7: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=check_cmd ) else: wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=check_cmd)['exit_code'] == 0, timeout=timeout) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=kill_cmd ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=check_cmd ) @retry(count=3, delay=60) def sync_time(self, nailgun_nodes=None): # with @retry, failure on any step of time synchronization causes # restart the time synchronization starting from the admin node if nailgun_nodes is None: nailgun_nodes = [] controller_nodes = [ n for n in nailgun_nodes if "controller" in n['roles']] other_nodes = [ n for n in nailgun_nodes if "controller" not in n['roles']] # 1. The first time source for the environment: admin node logger.info("Synchronizing time on Fuel admin node") with GroupNtpSync(self, sync_admin_node=True) as g_ntp: g_ntp.do_sync_time() # 2. Controllers should be synchronized before providing time to others if controller_nodes: logger.info("Synchronizing time on all controllers") with GroupNtpSync(self, nailgun_nodes=controller_nodes) as g_ntp: g_ntp.do_sync_time() # 3. Synchronize time on all the rest nodes if other_nodes: logger.info("Synchronizing time on other active nodes") with GroupNtpSync(self, nailgun_nodes=other_nodes) as g_ntp: g_ntp.do_sync_time() def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) with self.d_env.get_admin_remote() as admin_remote: wait( lambda: not admin_remote.execute( "grep 'Fuel node deployment' '%s'" % log_path )['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT)) ) result = admin_remote.execute( "grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = 'enp0s3' command = "dhcpcheck discover " \ "--ifaces {iface} " \ "--repeat 3 " \ "--timeout 10".format(iface=iface) out = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=command )['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu': logger.warning('Default image for bootstrap ' 'is not based on Ubuntu!') return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel-bootstrap --quiet list' )['stdout'] assert_true(any('active' in line for line in bootstrap_images), 'Ubuntu bootstrap image wasn\'t built and activated! ' 'See logs in /var/log/fuel-bootstrap-image-build.log ' 'for details.') def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name) ) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name) ) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name) ) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name) ) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) # Execute yum updates # If updates installed, # then `dockerctl destroy all; bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info('Searching for updates..') update_command = 'yum clean expire-cache; yum update -y' update_result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=update_command ) logger.info('Result of "{1}" command on master node: ' '{0}'.format(update_result, update_command)) assert_equal(int(update_result['exit_code']), 0, 'Packages update failed, ' 'inspect logs for details') # Check if any packets were updated and update was successful for str_line in update_result['stdout']: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates)\ and not match_complete_message: logger.warning('No updates were found or update was incomplete.') return logger.info('{0} packet(s) were updated'.format(updates_count)) cmd = 'dockerctl destroy all; bootstrap_admin_node.sh;' result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) logger.info('Result of "{1}" command on master node: ' '{0}'.format(result, cmd)) assert_equal(int(result['exit_code']), 0, 'bootstrap failed, ' 'inspect logs for details') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd='cat /etc/resolv.conf' ) assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}' .format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=echo_cmd ) assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}' .format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): admin_net2_object = self.d_env.get_network(name=self.d_env.admin_net2) second_admin_network = admin_net2_object.ip.network second_admin_netmask = admin_net2_object.ip.netmask second_admin_if = settings.INTERFACES.get(self.d_env.admin_net2) second_admin_ip = str(self.d_env.nodes( ).admin.get_ip_address_by_network_name(self.d_env.admin_net2)) logger.info(('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(second_admin_ip, second_admin_if)) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp( self.ssh_manager.admin_ip, second_admin_if ) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, second_admin_network, second_admin_netmask, second_admin_if, self.get_admin_node_ip() ) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), admin_login=settings.SSH_FUEL_CREDENTIALS['login'], admin_password=settings.SSH_FUEL_CREDENTIALS['password'], slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, '_fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server( cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=settings.BOOTSTRAP_TIMEOUT, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time() return self.nailgun_nodes(devops_nodes) def sync_time(self, nodes_names=None, skip_sync=False): if nodes_names is None: roles = ['fuel_master', 'fuel_slave'] nodes_names = [node.name for node in self.d_env.get_nodes() if node.role in roles and node.driver.node_active(node)] logger.info("Please wait while time on nodes: {0} " "will be synchronized" .format(', '.join(sorted(nodes_names)))) new_time = sync_time(self.d_env, nodes_names, skip_sync) for name in sorted(new_time): logger.info("New time on '{0}' = {1}".format(name, new_time[name])) @logwrap def get_admin_node_ip(self): return str( self.d_env.nodes( ).admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'device_label': settings.ISO_LABEL, 'iface': iface_alias('eth0'), 'ip': node.get_ip_address_by_network_name( self.d_env.admin_net), 'mask': self.d_env.get_network( name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': self.d_env.nat_interface, 'nameserver': settings.DNS, 'showmenu': 'yes' if settings.SHOW_FUELMENU else 'no', 'wait_for_external_config': 'yes', 'build_images': '1' if build_images else '0', 'MASTER_NODE_EXTRA_PACKAGES': settings.MASTER_NODE_EXTRA_PACKAGES } # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" ) else: # cdrom is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" ) keys += ( "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img" " inst.ks=cdrom:LABEL=%(device_label)s:/ks.cfg" " inst.repo=cdrom:LABEL=%(device_label)s:/" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":%(iface)s:off::: nameserver=%(nameserver)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s" " build_images=%(build_images)s\n" " MASTER_NODE_EXTRA_PACKAGES='%(MASTER_NODE_EXTRA_PACKAGES)s'\n" " <Enter>\n" ) % params return keys @staticmethod def get_target_devs(devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var]] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config[ 'template']['devops_settings']['env_name']) except Exception: self._virt_env = Environment.create_environment( full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin self.ssh_manager.clean_all_connections() try: admin.await(self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin.await(self.d_env.admin_net) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT )) self.set_admin_ssh_password() self.admin_actions.wait_for_fuel_ready() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend() time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return [self.fuel_web.get_nailgun_node_by_devops_node(node) for node in devops_nodes] def check_slaves_are_ready(self): devops_nodes = [node for node in self.d_env.nodes().slaves if node.driver.node_active(node)] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError( "Node {0} does not become online".format(node.name)) return True def revert_snapshot(self, name, skip_timesync=False, skip_slaves_check=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: {:s}'.format(name)) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: self.sync_time() try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() if not skip_slaves_check: _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_FUEL_CREDENTIALS['login'] new_password = settings.SSH_FUEL_CREDENTIALS['password'] try: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='date' ) logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login='******', admin_password='******', slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format(new_login, new_password) ) self.ssh_manager.initialize( admin_ip=self.ssh_manager.admin_ip, admin_login=new_login, admin_password=new_password, slave_login=settings.SSH_SLAVE_CREDENTIALS['login'], slave_password=settings.SSH_SLAVE_CREDENTIALS['password'] ) self.ssh_manager.update_connection( ip=self.ssh_manager.admin_ip, login=new_login, password=new_password ) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'". format(new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password']) ) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"' .format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def insert_cdrom_tray(self): # This is very rude implementation and it SHOULD be changes after # implementation this feature in fuel-devops name = "{}_{}".format(settings.ENV_NAME, self.d_env.nodes().admin.name) name_size = 80 if len(name) > name_size: hash_str = str(hash(name)) name = (hash_str + name)[:name_size] cmd = """EDITOR="sed -i s/tray=\\'open\\'//" virsh edit {}""".format( name) subprocess.check_call(cmd, shell=True) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # Create environment and start the Fuel master node admin = self.d_env.nodes().admin self.d_env.start([admin]) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if settings.SHOW_FUELMENU: self.wait_for_fuelmenu() else: self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() self.admin_actions.wait_for_fuel_ready() @logwrap def enable_force_https(self, admin_node_ip): cmd = """ echo -e '"SSL":\n "force_https": "true"' >> /etc/fuel/astute.yaml """ self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = "find / -name \"nginx_services.pp\"" puppet_manifest = \ self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0].strip() cmd = 'puppet apply {0}'.format(puppet_manifest) self.ssh_manager.execute_on_remote(admin_node_ip, cmd) cmd = """ systemctl status nginx.service | awk 'match($0, /\s+Active:.*\((\w+)\)/, a) {print a[1]}' """ wait(lambda: ( self.ssh_manager.execute_on_remote( admin_node_ip, cmd)['stdout'][0] != 'dead'), interval=10, timeout=30) # pylint: disable=no-self-use @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...') # pylint: enable=no-self-use @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): _wait(lambda: _tcp_ping( self.d_env.nodes( ).admin.get_ip_address_by_network_name (self.d_env.admin_net), 22), timeout=timeout) @logwrap def wait_for_fuelmenu(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): def check_ssh_connection(): """Try to close fuelmenu and check ssh connection""" try: _tcp_ping( self.d_env.nodes( ).admin.get_ip_address_by_network_name (self.d_env.admin_net), 22) except Exception: # send F8 trying to exit fuelmenu self.d_env.nodes().admin.send_keys("<F8>\n") return False return True wait(check_ssh_connection, interval=30, timeout=timeout, timeout_msg="Fuelmenu hasn't appeared during allocated timeout") @logwrap def wait_for_external_config(self, timeout=120): wait(lambda: self.ssh_manager.exists_on_remote( self.ssh_manager.admin_ip, '/var/lock/wait_for_external_config'), timeout=600) check_cmd = 'pkill -0 -f wait_for_external_config' wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=check_cmd)['exit_code'] == 0, timeout=timeout) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=kill_cmd ) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=check_cmd ) def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Running bootstrap (timeout: {0})".format( float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT))) with TimeStat("admin_node_bootsrap_time", is_uniq=True): wait( lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment' '{:s}'".format(log_path) )['exit_code'] == 0, timeout=(float(settings.ADMIN_NODE_BOOTSTRAP_TIMEOUT)) ) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="grep 'Fuel node deployment " "complete' '{:s}'".format(log_path))['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = iface_alias('eth0') command = "dhcpcheck discover " \ "--ifaces {iface} " \ "--repeat 3 " \ "--timeout 10".format(iface=iface) out = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=command )['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu': logger.warning('Default image for bootstrap ' 'is not based on Ubuntu!') return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel-bootstrap --quiet list' )['stdout'] assert_true(any('active' in line for line in bootstrap_images), 'Ubuntu bootstrap image wasn\'t built and activated! ' 'See logs in /var/log/fuel-bootstrap-image-build.log ' 'for details.') def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name) ) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name) ) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}" .format(pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name) ) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name) ) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}" .format(service_name, remote_status['exit_code'], remote_status['stdout'])) # Execute yum updates # If updates installed, # then `bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info('Searching for updates..') update_command = 'yum clean expire-cache; yum update -y' update_result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=update_command ) logger.info('Result of "{1}" command on master node: ' '{0}'.format(update_result, update_command)) assert_equal(int(update_result['exit_code']), 0, 'Packages update failed, ' 'inspect logs for details') # Check if any packets were updated and update was successful for str_line in update_result['stdout']: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates)\ and not match_complete_message: logger.warning('No updates were found or update was incomplete.') return logger.info('{0} packet(s) were updated'.format(updates_count)) cmd = 'bootstrap_admin_node.sh;' result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) logger.info('Result of "{1}" command on master node: ' '{0}'.format(result, cmd)) assert_equal(int(result['exit_code']), 0, 'bootstrap failed, ' 'inspect logs for details') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd='cat /etc/resolv.conf' ) assert_equal(0, resolv_conf['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}' .format('cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=echo_cmd ) assert_equal(0, echo_result['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}' .format(echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @staticmethod @logwrap def execute_remote_cmd(remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal(result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'. format(cmd, result)) return result['stdout'] @logwrap def describe_other_admin_interfaces(self, admin): admin_networks = [iface.network.name for iface in admin.interfaces] iface_name = None for i, network_name in enumerate(admin_networks): if 'admin' in network_name and 'admin' != network_name: # This will be replaced with actual interface labels # form fuel-devops iface_name = 'enp0s' + str(i + 3) logger.info("Describe Fuel admin node interface {0} for " "network {1}".format(iface_name, network_name)) self.describe_admin_interface(iface_name, network_name) if iface_name: return self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="cobbler sync") @logwrap def describe_admin_interface(self, admin_if, network_name): admin_net_object = self.d_env.get_network(name=network_name) admin_network = admin_net_object.ip.network admin_netmask = admin_net_object.ip.netmask admin_ip = str(self.d_env.nodes( ).admin.get_ip_address_by_network_name(network_name)) logger.info(('Parameters for admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(admin_network, admin_netmask, admin_if, admin_ip)) add_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format(admin_if, admin_ip, admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_admin_ip, admin_if, admin_ip) logger.debug('Trying to assign {0} IP to the {1} on master node...'. format(admin_ip, admin_if)) result = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=cmd ) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp( self.ssh_manager.admin_ip, admin_if ) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, admin_network, admin_netmask, admin_if, self.get_admin_node_ip() ) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")
class EnvironmentModel(object): """EnvironmentModel.""" # TODO documentation _instance = None def __new__(cls, *args, **kwargs): if not cls._instance: cls._instance = super(EnvironmentModel, cls).__new__(cls, *args, **kwargs) return cls._instance def __init__(self, config=None): if not hasattr(self, "_virt_env"): self._virt_env = None if not hasattr(self, "_fuel_web"): self._fuel_web = None self._config = config self.ssh_manager = SSHManager() self.ssh_manager.initialize( self.get_admin_node_ip(), login=settings.SSH_CREDENTIALS['login'], password=settings.SSH_CREDENTIALS['password']) self.admin_actions = AdminActions() self.base_actions = BaseActions() self.cobbler_actions = CobblerActions() self.docker_actions = DockerActions() self.nailgun_actions = NailgunActions() self.postgres_actions = PostgresActions() self.fuel_bootstrap_actions = FuelBootstrapCliActions() @property def fuel_web(self): if self._fuel_web is None: self._fuel_web = FuelWebClient(self) return self._fuel_web def __repr__(self): klass, obj_id = type(self), hex(id(self)) if getattr(self, '_fuel_web'): ip = self.fuel_web.admin_node_ip else: ip = None return "[{klass}({obj_id}), ip:{ip}]".format(klass=klass, obj_id=obj_id, ip=ip) @property def admin_node_ip(self): return self.fuel_web.admin_node_ip @property def collector(self): return CollectorClient(settings.ANALYTICS_IP, 'api/v1/json') @logwrap def add_syslog_server(self, cluster_id, port=5514): self.fuel_web.add_syslog_server(cluster_id, self.d_env.router(), port) def bootstrap_nodes(self, devops_nodes, timeout=900, skip_timesync=False): """Lists registered nailgun nodes Start vms and wait until they are registered on nailgun. :rtype : List of registered nailgun nodes """ # self.dhcrelay_check() for node in devops_nodes: logger.info("Bootstrapping node: {}".format(node.name)) node.start() # TODO(aglarendil): LP#1317213 temporary sleep # remove after better fix is applied time.sleep(5) if not MASTER_IS_CENTOS7: with TimeStat("wait_for_nodes_to_start_and_register_in_nailgun"): wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) else: wait(lambda: all(self.nailgun_nodes(devops_nodes)), 15, timeout) if not skip_timesync: self.sync_time([node for node in self.nailgun_nodes(devops_nodes)]) return self.nailgun_nodes(devops_nodes) @logwrap def get_admin_node_ip(self): return str(self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net)) @logwrap def get_ebtables(self, cluster_id, devops_nodes): return Ebtables(self.get_target_devs(devops_nodes), self.fuel_web.client.get_cluster_vlans(cluster_id)) def get_keys(self, node, custom=None, build_images=None, iso_connect_as='cdrom'): params = { 'ks': 'hd:LABEL="Mirantis_Fuel":/ks.cfg' if iso_connect_as == 'usb' else 'cdrom:/ks.cfg', 'repo': 'hd:LABEL="Mirantis_Fuel":/', # only required for USB boot 'ip': node.get_ip_address_by_network_name(self.d_env.admin_net), 'mask': self.d_env.get_network(name=self.d_env.admin_net).ip.netmask, 'gw': self.d_env.router(), 'hostname': ''.join((settings.FUEL_MASTER_HOSTNAME, settings.DNS_SUFFIX)), 'nat_interface': self.d_env.nat_interface, 'dns1': settings.DNS, 'showmenu': 'no', 'wait_for_external_config': 'yes', 'build_images': '1' if build_images else '0' } if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n") % params else: # cdrom case is default keys = ("<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s\n" " netmask=%(mask)s\n" " gw=%(gw)s\n" " dns1=%(dns1)s\n" " hostname=%(hostname)s\n" " dhcp_interface=%(nat_interface)s\n" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n") % params if MASTER_IS_CENTOS7: # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) add tests for menu items/kernel parameters # TODO(akostrikov) refactor it. iface = 'enp0s3' if iso_connect_as == 'usb': keys = ( "<Wait>\n" # USB boot uses boot_menu=yes for master node "<F12>\n" "2\n" "<Esc><Enter>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " repo=%(repo)s\n" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":{iface}:off::: dns1=%(dns1)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n".format(iface=iface)) % params else: # cdrom case is default keys = ( "<Wait>\n" "<Wait>\n" "<Wait>\n" "<Esc>\n" "<Wait>\n" "vmlinuz initrd=initrd.img ks=%(ks)s\n" " ip=%(ip)s::%(gw)s:%(mask)s:%(hostname)s" ":{iface}:off::: dns1=%(dns1)s" " showmenu=%(showmenu)s\n" " wait_for_external_config=%(wait_for_external_config)s\n" " build_images=%(build_images)s\n" " <Enter>\n".format(iface=iface)) % params return keys def get_target_devs(self, devops_nodes): return [ interface.target_dev for interface in [ val for var in map(lambda node: node.interfaces, devops_nodes) for val in var ] ] @property def d_env(self): if self._virt_env is None: if not self._config: try: return Environment.get(name=settings.ENV_NAME) except Exception: self._virt_env = Environment.describe_environment( boot_from=settings.ADMIN_BOOT_DEVICE) self._virt_env.define() else: try: return Environment.get(name=self._config['template'] ['devops_settings']['env_name']) except Exception: self._virt_env = Environment.create_environment( full_config=self._config) self._virt_env.define() return self._virt_env def resume_environment(self): self.d_env.resume() admin = self.d_env.nodes().admin try: admin. await (self.d_env.admin_net, timeout=30, by_port=8000) except Exception as e: logger.warning("From first time admin isn't reverted: " "{0}".format(e)) admin.destroy() logger.info('Admin node was destroyed. Wait 10 sec.') time.sleep(10) admin.start() logger.info('Admin node started second time.') self.d_env.nodes().admin. await (self.d_env.admin_net) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers(timeout=600) # set collector address in case of admin node destroy if settings.FUEL_STATS_ENABLED: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd in order to apply new collector address self.nailgun_actions.force_fuel_stats_sending() self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) self.set_admin_ssh_password() self.docker_actions.wait_for_ready_containers() def make_snapshot(self, snapshot_name, description="", is_make=False): if settings.MAKE_SNAPSHOT or is_make: self.d_env.suspend(verbose=False) time.sleep(10) self.d_env.snapshot(snapshot_name, force=True, description=description) revert_info(snapshot_name, self.get_admin_node_ip(), description) if settings.FUEL_STATS_CHECK: self.resume_environment() def nailgun_nodes(self, devops_nodes): return map( lambda node: self.fuel_web.get_nailgun_node_by_devops_node(node), devops_nodes) def check_slaves_are_ready(self): devops_nodes = [ node for node in self.d_env.nodes().slaves if node.driver.node_active(node) ] # Bug: 1455753 time.sleep(30) for node in devops_nodes: try: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( node)['online'], timeout=60 * 6) except TimeoutError: raise TimeoutError("Node {0} does not become online".format( node.name)) return True def revert_snapshot(self, name, skip_timesync=False): if not self.d_env.has_snapshot(name): return False logger.info('We have snapshot with such name: %s' % name) logger.info("Reverting the snapshot '{0}' ....".format(name)) self.d_env.revert(name) logger.info("Resuming the snapshot '{0}' ....".format(name)) self.resume_environment() if not skip_timesync: nailgun_nodes = [ self.fuel_web.get_nailgun_node_by_name(node.name) for node in self.d_env.nodes().slaves if node.driver.node_active(node) ] self.sync_time(nailgun_nodes) try: _wait(self.fuel_web.client.get_releases, expected=EnvironmentError, timeout=300) except exceptions.Unauthorized: self.set_admin_keystone_password() self.fuel_web.get_nailgun_version() _wait(lambda: self.check_slaves_are_ready(), timeout=60 * 6) return True def set_admin_ssh_password(self): new_login = settings.SSH_CREDENTIALS['login'] new_password = settings.SSH_CREDENTIALS['password'] try: self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd='date') logger.debug('Accessing admin node using SSH: SUCCESS') except Exception: logger.debug('Accessing admin node using SSH credentials:' ' FAIL, trying to change password from default') self.ssh_manager.update_connection(ip=self.ssh_manager.admin_ip, login='******', password='******') self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='echo -e "{1}\\n{1}" | passwd {0}'.format( new_login, new_password)) self.ssh_manager.update_connection(ip=self.ssh_manager.admin_ip, login=new_login, password=new_password) logger.debug("Admin node password has changed.") logger.info("Admin node login name: '{0}' , password: '******'".format( new_login, new_password)) def set_admin_keystone_password(self): try: self.fuel_web.client.get_releases() # TODO(akostrikov) CENTOS7 except exceptions.Unauthorized: except: self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel user --newpass {0} --change-password'.format( settings.KEYSTONE_CREDS['password'])) logger.info( 'New Fuel UI (keystone) username: "******", password: "******"'. format(settings.KEYSTONE_CREDS['username'], settings.KEYSTONE_CREDS['password'])) def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # start admin node admin = self.d_env.nodes().admin if iso_connect_as == 'usb': admin.disk_devices.get(device='disk', bus='usb').volume.upload(settings.ISO_PATH) else: # cdrom is default admin.disk_devices.get(device='cdrom').volume.upload( settings.ISO_PATH) self.d_env.start(self.d_env.nodes().admins) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys( self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() if settings.UPDATE_FUEL: # Update Ubuntu packages self.admin_actions.upload_packages( local_packages_dir=settings.UPDATE_FUEL_PATH, centos_repo_path=None, ubuntu_repo_path=settings.LOCAL_MIRROR_UBUNTU) self.docker_actions.wait_for_ready_containers() time.sleep(10) self.set_admin_keystone_password() if not MASTER_IS_CENTOS7: self.sync_time() if settings.UPDATE_MASTER: if settings.UPDATE_FUEL_MIRROR: for i, url in enumerate(settings.UPDATE_FUEL_MIRROR): conf_file = '/etc/yum.repos.d/temporary-{}.repo'.format(i) cmd = ("echo -e" " '[temporary-{0}]\nname=" "temporary-{0}\nbaseurl={1}/" "\ngpgcheck=0\npriority=" "1' > {2}").format(i, url, conf_file) self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) self.admin_install_updates() if settings.MULTIPLE_NETWORKS: self.describe_second_admin_interface() if not MASTER_IS_CENTOS7: self.nailgun_actions.set_collector_address( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT, settings.FUEL_STATS_SSL) # Restart statsenderd to apply settings(Collector address) self.nailgun_actions.force_fuel_stats_sending() if settings.FUEL_STATS_ENABLED and not MASTER_IS_CENTOS7: self.fuel_web.client.send_fuel_stats(enabled=True) logger.info('Enabled sending of statistics to {0}:{1}'.format( settings.FUEL_STATS_HOST, settings.FUEL_STATS_PORT)) if settings.PATCHING_DISABLE_UPDATES: cmd = "find /etc/yum.repos.d/ -type f -regextype posix-egrep" \ " -regex '.*/mos[0-9,\.]+\-(updates|security).repo' | " \ "xargs -n1 -i sed '$aenabled=0' -i {}" self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=cmd) @update_rpm_packages @upload_manifests def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...') @logwrap def wait_for_provisioning(self, timeout=settings.WAIT_FOR_PROVISIONING_TIMEOUT): _wait(lambda: _tcp_ping( self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net), 22), timeout=timeout) @logwrap def wait_for_external_config(self, timeout=120): check_cmd = 'pkill -0 -f wait_for_external_config' if MASTER_IS_CENTOS7: self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=check_cmd) else: wait(lambda: self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd=check_cmd)['exit_code'] == 0, timeout=timeout) @logwrap def kill_wait_for_external_config(self): kill_cmd = 'pkill -f "^wait_for_external_config"' check_cmd = 'pkill -0 -f "^wait_for_external_config"; [[ $? -eq 1 ]]' self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=kill_cmd) self.ssh_manager.execute_on_remote(ip=self.ssh_manager.admin_ip, cmd=check_cmd) @retry(count=3, delay=60) def sync_time(self, nailgun_nodes=None): # with @retry, failure on any step of time synchronization causes # restart the time synchronization starting from the admin node if nailgun_nodes is None: nailgun_nodes = [] controller_nodes = [ n for n in nailgun_nodes if "controller" in n['roles'] ] other_nodes = [ n for n in nailgun_nodes if "controller" not in n['roles'] ] # 1. The first time source for the environment: admin node logger.info("Synchronizing time on Fuel admin node") with GroupNtpSync(self, sync_admin_node=True) as g_ntp: g_ntp.do_sync_time() # 2. Controllers should be synchronized before providing time to others if controller_nodes: logger.info("Synchronizing time on all controllers") with GroupNtpSync(self, nailgun_nodes=controller_nodes) as g_ntp: g_ntp.do_sync_time() # 3. Synchronize time on all the rest nodes if other_nodes: logger.info("Synchronizing time on other active nodes") with GroupNtpSync(self, nailgun_nodes=other_nodes) as g_ntp: g_ntp.do_sync_time() def wait_bootstrap(self): logger.info("Waiting while bootstrapping is in progress") log_path = "/var/log/puppet/bootstrap_admin_node.log" logger.info("Puppet timeout set in {0}".format( float(settings.PUPPET_TIMEOUT))) with self.d_env.get_admin_remote() as admin_remote: wait(lambda: not admin_remote.execute( "grep 'Fuel node deployment' '%s'" % log_path)['exit_code'], timeout=(float(settings.PUPPET_TIMEOUT))) result = admin_remote.execute("grep 'Fuel node deployment " "complete' '%s'" % log_path)['exit_code'] if result != 0: raise Exception('Fuel node deployment failed.') self.bootstrap_image_check() def dhcrelay_check(self): # CentOS 7 is pretty stable with admin iface. # TODO(akostrikov) refactor it. iface = 'enp0s3' command = "dhcpcheck discover " \ "--ifaces {iface} " \ "--repeat 3 " \ "--timeout 10".format(iface=iface) out = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=command)['stdout'] assert_true(self.get_admin_node_ip() in "".join(out), "dhcpcheck doesn't discover master ip") def bootstrap_image_check(self): fuel_settings = self.admin_actions.get_fuel_settings() if fuel_settings['BOOTSTRAP']['flavor'].lower() != 'ubuntu': logger.warning('Default image for bootstrap ' 'is not based on Ubuntu!') return bootstrap_images = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd='fuel-bootstrap --quiet list')['stdout'] assert_true( any('active' in line for line in bootstrap_images), 'Ubuntu bootstrap image wasn\'t built and activated! ' 'See logs in /var/log/fuel-bootstrap-image-build.log ' 'for details.') def admin_install_pkg(self, pkg_name): """Install a package <pkg_name> on the admin node""" remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="rpm -q {0}'".format(pkg_name)) if remote_status['exit_code'] == 0: logger.info("Package '{0}' already installed.".format(pkg_name)) else: logger.info("Installing package '{0}' ...".format(pkg_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="yum -y install {0}".format(pkg_name)) logger.info("Installation of the package '{0}' has been" " completed with exit code {1}".format( pkg_name, remote_status['exit_code'])) return remote_status['exit_code'] def admin_run_service(self, service_name): """Start a service <service_name> on the admin node""" self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd="service {0} start".format(service_name)) remote_status = self.ssh_manager.execute( ip=self.ssh_manager.admin_ip, cmd="service {0} status".format(service_name)) if any('running...' in status for status in remote_status['stdout']): logger.info("Service '{0}' is running".format(service_name)) else: logger.info("Service '{0}' failed to start" " with exit code {1} :\n{2}".format( service_name, remote_status['exit_code'], remote_status['stdout'])) # Execute yum updates # If updates installed, # then `dockerctl destroy all; bootstrap_admin_node.sh;` def admin_install_updates(self): logger.info('Searching for updates..') update_command = 'yum clean expire-cache; yum update -y' update_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=update_command) logger.info('Result of "{1}" command on master node: ' '{0}'.format(update_result, update_command)) assert_equal(int(update_result['exit_code']), 0, 'Packages update failed, ' 'inspect logs for details') # Check if any packets were updated and update was successful for str_line in update_result['stdout']: match_updated_count = re.search("Upgrade(?:\s*)(\d+).*Package", str_line) if match_updated_count: updates_count = match_updated_count.group(1) match_complete_message = re.search("(Complete!)", str_line) match_no_updates = re.search("No Packages marked for Update", str_line) if (not match_updated_count or match_no_updates)\ and not match_complete_message: logger.warning('No updates were found or update was incomplete.') return logger.info('{0} packet(s) were updated'.format(updates_count)) cmd = 'dockerctl destroy all; bootstrap_admin_node.sh;' result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) logger.info('Result of "{1}" command on master node: ' '{0}'.format(result, cmd)) assert_equal(int(result['exit_code']), 0, 'bootstrap failed, ' 'inspect logs for details') # Modifies a resolv.conf on the Fuel master node and returns # its original content. # * adds 'nameservers' at start of resolv.conf if merge=True # * replaces resolv.conf with 'nameservers' if merge=False def modify_resolv_conf(self, nameservers=None, merge=True): if nameservers is None: nameservers = [] resolv_conf = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd='cat /etc/resolv.conf') assert_equal( 0, resolv_conf['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}'.format( 'cat /etc/resolv.conf', resolv_conf['stderr'])) if merge: nameservers.extend(resolv_conf['stdout']) resolv_keys = ['search', 'domain', 'nameserver'] resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers if any(x in ns for x in resolv_keys)) logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) echo_result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=echo_cmd) assert_equal( 0, echo_result['exit_code'], 'Executing "{0}" on the admin node has failed with: {1}'.format( echo_cmd, echo_result['stderr'])) return resolv_conf['stdout'] @logwrap def execute_remote_cmd(self, remote, cmd, exit_code=0): result = remote.execute(cmd) assert_equal( result['exit_code'], exit_code, 'Failed to execute "{0}" on remote host: {1}'.format(cmd, result)) return result['stdout'] @logwrap def describe_second_admin_interface(self): admin_net2_object = self.d_env.get_network(name=self.d_env.admin_net2) second_admin_network = admin_net2_object.ip.network second_admin_netmask = admin_net2_object.ip.netmask second_admin_if = settings.INTERFACES.get(self.d_env.admin_net2) second_admin_ip = str( self.d_env.nodes().admin.get_ip_address_by_network_name( self.d_env.admin_net2)) logger.info( ('Parameters for second admin interface configuration: ' 'Network - {0}, Netmask - {1}, Interface - {2}, ' 'IP Address - {3}').format(second_admin_network, second_admin_netmask, second_admin_if, second_admin_ip)) add_second_admin_ip = ('DEVICE={0}\\n' 'ONBOOT=yes\\n' 'NM_CONTROLLED=no\\n' 'USERCTL=no\\n' 'PEERDNS=no\\n' 'BOOTPROTO=static\\n' 'IPADDR={1}\\n' 'NETMASK={2}\\n').format( second_admin_if, second_admin_ip, second_admin_netmask) cmd = ('echo -e "{0}" > /etc/sysconfig/network-scripts/ifcfg-{1};' 'ifup {1}; ip -o -4 a s {1} | grep -w {2}').format( add_second_admin_ip, second_admin_if, second_admin_ip) logger.debug( 'Trying to assign {0} IP to the {1} on master node...'.format( second_admin_ip, second_admin_if)) result = self.ssh_manager.execute(ip=self.ssh_manager.admin_ip, cmd=cmd) assert_equal(result['exit_code'], 0, ('Failed to assign second admin ' 'IP address on master node: {0}').format(result)) logger.debug('Done: {0}'.format(result['stdout'])) # TODO for ssh manager multiple_networks_hacks.configure_second_admin_dhcp( self.ssh_manager.admin_ip, second_admin_if) multiple_networks_hacks.configure_second_admin_firewall( self.ssh_manager.admin_ip, second_admin_network, second_admin_netmask, second_admin_if, self.get_admin_node_ip()) @logwrap def get_masternode_uuid(self): return self.postgres_actions.run_query( db='nailgun', query="select master_node_uid from master_node_settings limit 1;")