def fill_root_below_rabbit_disk_free_limit(self): """Fill root more to below rabbit disk free limit""" with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: pacemaker_attributes = run_on_remote_get_results( remote, 'cibadmin --query --scope status')['stdout_str'] controller_space_on_root = get_pacemaker_nodes_attributes( pacemaker_attributes)[self.primary_controller_fqdn][ 'root_free'] logger.info( "Free space in root on primary controller - {}".format( controller_space_on_root )) controller_space_to_filled = str( int( controller_space_on_root ) - self.rabbit_disk_free_limit - 1) logger.info( "Need to fill space on root - {}".format( controller_space_to_filled )) run_on_remote_get_results( remote, 'fallocate -l {}M /root/bigfile2'.format( controller_space_to_filled)) check_file_exists(remote, '/root/bigfile2')
def fill_root_below_rabbit_disk_free_limit(self): """Fill root more to below rabbit disk free limit""" with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: pacemaker_attributes = run_on_remote_get_results( remote, 'cibadmin --query --scope status')['stdout_str'] controller_space_on_root = get_pacemaker_nodes_attributes( pacemaker_attributes)[ self.primary_controller_fqdn]['root_free'] logger.info("Free space in root on primary controller - {}".format( controller_space_on_root)) controller_space_to_filled = str( int(controller_space_on_root) - self.rabbit_disk_free_limit - 1) logger.info("Need to fill space on root - {}".format( controller_space_to_filled)) run_on_remote_get_results( remote, 'fallocate -l {}M /root/bigfile2'.format( controller_space_to_filled)) check_file_exists(remote, '/root/bigfile2')
def verify_bootstrap_on_node(remote, os_type, uuid=None): os_type = os_type.lower() if os_type not in ['ubuntu', 'centos']: raise Exception("Only Ubuntu and CentOS are supported, " "you have chosen {0}".format(os_type)) logger.info("Verify bootstrap on slave {0}".format(remote.host)) cmd = 'cat /etc/*release' output = run_on_remote_get_results(remote, cmd)['stdout_str'].lower() assert_true( os_type in output, "Slave {0} doesn't use {1} image for bootstrap " "after {1} images were enabled, /etc/release " "content: {2}".format(remote.host, os_type, output)) if os_type == 'centos' or uuid is None: return cmd = "cat /etc/nailgun-agent/config.yaml" output = yaml.load(run_on_remote_get_results(remote, cmd)['stdout_str']) actual_uuid = output.get("runtime_uuid") assert_equal( actual_uuid, uuid, "Actual uuid {0} is not the same as expected {1}".format( actual_uuid, uuid))
def clean_up_space_on_root(self): """Clean up space on root filesystem on primary controller""" with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: run_on_remote_get_results(remote, 'rm /root/bigfile /root/bigfile2') run_on_remote_get_results( remote, 'crm node status-attr {} delete "#health_disk"'.format( self.primary_controller_fqdn))
def clean_up_space_on_root(self): """Clean up space on root filesystem on primary controller""" with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: run_on_remote_get_results( remote, 'rm /root/bigfile /root/bigfile2') run_on_remote_get_results( remote, 'crm node status-attr {} delete "#health_disk"'.format( self.primary_controller_fqdn))
def reconfigure_keystone_to_use_ldap(self): """Reconfigure neutron ml2 VLAN range Scenario: 1. Revert snapshot "deploy_neutron_vlan_ha" 2. Upload a new openstack configuration 3. Try to apply a new keystone configuration 4. Wait for failing of deployment task 5. Check that reason of failing is impossibility of the connection to LDAP server Snapshot reconfigure_keystone_to_use_ldap """ self.show_step(1) self.env.revert_snapshot("deploy_neutron_vlan_ha") cluster_id = self.fuel_web.get_last_created_cluster() controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) ldap_cntrllr = controllers[0] self.show_step(2) config = utils.get_config_template('keystone_ldap') self.fuel_web.client.upload_configuration( config, cluster_id, node_id=ldap_cntrllr['id']) self.show_step(3) task = self.fuel_web.client.apply_configuration( cluster_id, node_id=ldap_cntrllr['id']) self.show_step(4) try: self.fuel_web.assert_task_success(task, timeout=1800, interval=30) except AssertionError: pass else: raise Exception("New configuration was not applied") self.show_step(5) with self.env.d_env.get_ssh_to_remote(ldap_cntrllr['ip']) as remote: log_path = '/var/log/puppet.log' cmd = "grep \"Can't contact LDAP server\" {0}".format(log_path) utils.run_on_remote_get_results(remote, cmd) self.env.make_snapshot("reconfigure_keystone_to_use_ldap", is_make=True)
def reconfigure_keystone_to_use_ldap(self): """Reconfigure keystone to use LDAP Scenario: 1. Revert snapshot "basic_env_for_reconfiguration" 2. Upload a new openstack configuration 3. Try to apply a new keystone configuration 4. Wait for failing of deployment task 5. Check that reason of failing is impossibility of the connection to LDAP server Snapshot: reconfigure_keystone_to_use_ldap """ self.show_step(1, initialize=True) self.env.revert_snapshot("basic_env_for_reconfiguration") cluster_id = self.fuel_web.get_last_created_cluster() controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) ldap_cntrllr = controllers[0] self.show_step(2) config = utils.get_config_template('keystone_ldap') self.fuel_web.client.upload_configuration( config, cluster_id, node_id=ldap_cntrllr['id']) self.show_step(3) task = self.fuel_web.client.apply_configuration( cluster_id, node_id=ldap_cntrllr['id']) self.show_step(4) try: self.fuel_web.assert_task_success(task, timeout=1800, interval=30) except AssertionError: pass else: raise Exception("New configuration was not applied") self.show_step(5) with self.env.d_env.get_ssh_to_remote(ldap_cntrllr['ip']) as remote: log_path = '/var/log/puppet.log' cmd = "grep \"Can't contact LDAP server\" {0}".format(log_path) utils.run_on_remote_get_results(remote, cmd) self.env.make_snapshot("reconfigure_keystone_to_use_ldap")
def fill_root_above_rabbit_disk_free_limit(self): """Filling root filesystem on primary controller""" logger.info("Free space in root on primary controller - {}".format( self.primary_controller_space_on_root)) logger.info("Need to fill space on root - {}".format( self.primary_controller_space_to_filled)) with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: run_on_remote_get_results( remote, 'fallocate -l {}M /root/bigfile'.format( self.primary_controller_space_to_filled)) check_file_exists(remote, '/root/bigfile')
def parse_pcs_status_xml(remote): """Parse 'pcs status xml'. <Nodes> section :param remote: SSHClient instance :return: nested dictionary with node-fqdn and attribute name as keys """ pcs_status_dict = run_on_remote_get_results( remote, 'pcs status xml')['stdout_str'] return pcs_status_dict
def checking_health_disk_attribute(): logger.info("Checking for '#health_disk' attribute") cibadmin_status_xml = run_on_remote_get_results( remote, 'cibadmin --query --scope status')['stdout_str'] pcs_attribs = get_pacemaker_nodes_attributes( cibadmin_status_xml) return '#health_disk' in pcs_attribs[ self.primary_controller_fqdn]
def checking_health_disk_attribute(): logger.info("Checking for '#health_disk' attribute") cibadmin_status_xml = run_on_remote_get_results( remote, 'cibadmin --query --scope status')[ 'stdout_str'] pcs_attribs = get_pacemaker_nodes_attributes( cibadmin_status_xml) return '#health_disk' in pcs_attribs[ self.primary_controller_fqdn]
def fill_root_above_rabbit_disk_free_limit(self): """Filling root filesystem on primary controller""" logger.info( "Free space in root on primary controller - {}".format( self.primary_controller_space_on_root )) logger.info( "Need to fill space on root - {}".format( self.primary_controller_space_to_filled )) with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: run_on_remote_get_results( remote, 'fallocate -l {}M /root/bigfile'.format( self.primary_controller_space_to_filled)) check_file_exists(remote, '/root/bigfile')
def get_pcs_initial_state(self): """Get controllers initial status in pacemaker""" self.primary_controller = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) self.primary_controller_fqdn = str( self.fuel_web.fqdn(self.primary_controller)) primary_ctrl = \ self.primary_controller.get_ip_address_by_network_name('admin') pcs_status = parse_pcs_status_xml(primary_ctrl) with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: root_free = run_on_remote_get_results( remote, 'cibadmin --query --scope status')['stdout_str'] self.primary_controller_space_on_root = get_pacemaker_nodes_attributes( root_free)[self.primary_controller_fqdn]['root_free'] self.disk_monitor_limit = 512 self.rabbit_disk_free_limit = 5 self.pacemaker_restart_timeout = 600 self.pcs_check_timeout = 300 self.primary_controller_space_to_filled = str( int( self.primary_controller_space_on_root ) - self.disk_monitor_limit - 1) self.pcs_status = get_pcs_nodes(pcs_status) self.slave_nodes_fqdn = list( set(self.pcs_status.keys()).difference( set(self.primary_controller_fqdn.split()))) running_resources_slave_1 = int( self.pcs_status[self.slave_nodes_fqdn[0]]['resources_running']) running_resources_slave_2 = int( self.pcs_status[self.slave_nodes_fqdn[1]]['resources_running']) self.slave_node_running_resources = str(min(running_resources_slave_1, running_resources_slave_2 ) )
def verify_bootstrap_on_node(remote, os_type, uuid=None): os_type = os_type.lower() if os_type not in ['ubuntu', 'centos']: raise Exception("Only Ubuntu and CentOS are supported, " "you have chosen {0}".format(os_type)) logger.info("Verify bootstrap on slave {0}".format(remote.host)) cmd = 'cat /etc/*release' output = run_on_remote_get_results(remote, cmd)['stdout_str'].lower() assert_true(os_type in output, "Slave {0} doesn't use {1} image for bootstrap " "after {1} images were enabled, /etc/release " "content: {2}".format(remote.host, os_type, output)) if os_type == 'centos' or uuid is None: return cmd = "cat /etc/nailgun-agent/config.yaml" output = yaml.load(run_on_remote_get_results(remote, cmd)['stdout_str']) actual_uuid = output.get("runtime_uuid") assert_equal(actual_uuid, uuid, "Actual uuid {0} is not the same as expected {1}" .format(actual_uuid, uuid))
def get_pcs_initial_state(self): """Get controllers initial status in pacemaker""" self.primary_controller = self.fuel_web.get_nailgun_primary_node( self.env.d_env.nodes().slaves[0]) self.primary_controller_fqdn = str( self.fuel_web.fqdn(self.primary_controller)) primary_ctrl = \ self.primary_controller.get_ip_address_by_network_name('admin') pcs_status = parse_pcs_status_xml(primary_ctrl) with self.fuel_web.get_ssh_for_node( self.primary_controller.name) as remote: root_free = run_on_remote_get_results( remote, 'cibadmin --query --scope status')['stdout_str'] self.primary_controller_space_on_root = get_pacemaker_nodes_attributes( root_free)[self.primary_controller_fqdn]['root_free'] self.disk_monitor_limit = 512 self.rabbit_disk_free_limit = 5 self.pacemaker_restart_timeout = 600 self.pcs_check_timeout = 300 self.primary_controller_space_to_filled = str( int(self.primary_controller_space_on_root) - self.disk_monitor_limit - 1) self.pcs_status = get_pcs_nodes(pcs_status) self.slave_nodes_fqdn = list( set(self.pcs_status.keys()).difference( set(self.primary_controller_fqdn.split()))) running_resources_slave_1 = int( self.pcs_status[self.slave_nodes_fqdn[0]]['resources_running']) running_resources_slave_2 = int( self.pcs_status[self.slave_nodes_fqdn[1]]['resources_running']) self.slave_node_running_resources = str( min(running_resources_slave_1, running_resources_slave_2))
def run_upgrade_script(node_ssh, script_path, script_name, password='******', rollback=False, exit_code=0): path = os.path.join(script_path, script_name) check_file_exists(node_ssh, path) c_res = node_ssh.execute('chmod 755 {0}'.format(path)) logger.debug("Result of chmod is {0}".format(c_res)) if rollback: path = "UPGRADERS='host-system docker openstack" \ " raise-error' {0}/{1}" \ " --password {2}".format(script_path, script_name, password) else: path = "{0}/{1} --no-rollback --password {2}".format( script_path, script_name, password) result = run_on_remote_get_results(node_ssh, path, assert_ec_equal=[exit_code], raise_on_assert=False) # TODO: check that we really need this log from fuel_upgrade.log if result['exit_code'] != exit_code: log = "".join( run_on_remote(node_ssh, "awk -v p=\"UPGRADE FAILED\" 'BEGIN{m=\"\"}" " {if ($0 ~ p) {m=$0} else m=m\"\\n\"$0}" " END{if (m ~ p) print m}'" " /var/log/fuel_upgrade.log", raise_on_assert=False)) logger.error("Message from /var/log/fuel_upgrade.log:\n" "{log}".format(log=log)) assert_equal( result['exit_code'], exit_code, "Upgrade script failed with exit code {exit_code}, " "please inspect logs for details.\n" "last output: \"{output}\"" "".format(exit_code=result['exit_code'], output=''.join(result['stdout'][-5:]) + result['stderr_str']))
def run_upgrade_script(node_ssh, script_path, script_name, password='******', rollback=False, exit_code=0): path = os.path.join(script_path, script_name) check_file_exists(node_ssh, path) c_res = node_ssh.execute('chmod 755 {0}'.format(path)) logger.debug("Result of chmod is {0}".format(c_res)) if rollback: path = "UPGRADERS='host-system docker openstack" \ " raise-error' {0}/{1}" \ " --password {2}".format(script_path, script_name, password) else: path = "{0}/{1} --no-rollback --password {2}".format(script_path, script_name, password) result = run_on_remote_get_results(node_ssh, path, assert_ec_equal=[exit_code], raise_on_assert=False) # TODO: check that we really need this log from fuel_upgrade.log if result['exit_code'] != exit_code: log = "".join( run_on_remote(node_ssh, "awk -v p=\"UPGRADE FAILED\" 'BEGIN{m=\"\"}" " {if ($0 ~ p) {m=$0} else m=m\"\\n\"$0}" " END{if (m ~ p) print m}'" " /var/log/fuel_upgrade.log", raise_on_assert=False) ) logger.error("Message from /var/log/fuel_upgrade.log:\n" "{log}".format(log=log)) assert_equal( result['exit_code'], exit_code, "Upgrade script failed with exit code {exit_code}, " "please inspect logs for details.\n" "last output: \"{output}\"" "".format(exit_code=result['exit_code'], output=''.join(result['stdout'][-5:]) + result['stderr_str']) )
def deploy_multiple_services_local_mirror(self): """Deploy cluster with multiple services using local mirror Scenario: 1. Revert snapshot 'empty' with default set of repositories 2. Bootstrap 5 slave nodes 3. Run 'fuel-createmirror' to replace default repositories with local mirrors 4. Create cluster with many components to check as many packages in local mirrors have correct dependencies 5. Deploy cluster Duration 50m """ self.env.revert_snapshot("empty") self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[:5]) logger.info("Executing 'fuel-createmirror' on Fuel admin node") with self.env.d_env.get_admin_remote() as remote: # TODO(ddmitriev):Enable debug via argument for 'fuel-createmirror' # when bug#1458469 fixed. if OPENSTACK_RELEASE_UBUNTU in OPENSTACK_RELEASE: cmd = ("sed -i 's/DEBUG=\"no\"/DEBUG=\"yes\"/' {}" .format('/etc/fuel-createmirror/ubuntu.cfg')) remote.execute(cmd) else: # CentOS is not supported yet, see bug#1467403 pass run_on_remote_get_results(remote, 'fuel-createmirror') # Check if there all repos were replaced with local mirrors ubuntu_id = self.fuel_web.client.get_release_id( release_name=OPENSTACK_RELEASE_UBUNTU) ubuntu_release = self.fuel_web.client.get_release(ubuntu_id) ubuntu_meta = ubuntu_release["attributes_metadata"] repos_ubuntu = ubuntu_meta["editable"]["repo_setup"]["repos"]['value'] remote_repos = [] for repo_value in repos_ubuntu: if (self.fuel_web.admin_node_ip not in repo_value['uri'] and '{settings.MASTER_IP}' not in repo_value['uri']): remote_repos.append({repo_value['name']: repo_value['uri']}) assert_true(not remote_repos, "Some repositories weren't replaced with local mirrors: " "{0}".format(remote_repos)) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE, settings={ 'sahara': True, 'murano': True, 'ceilometer': True, 'volumes_lvm': True, 'volumes_ceph': False, 'images_ceph': True, 'osd_pool_size': "3" } ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller', 'ceph-osd'], 'slave-02': ['compute', 'ceph-osd'], 'slave-03': ['cinder', 'ceph-osd'], 'slave-04': ['mongo'], 'slave-05': ['mongo'] } ) repos_attr = self.get_cluster_repos(cluster_id) self.fuel_web.report_repos(repos_attr) # (ddmitriev): No additional checks is required after deploy, # just make sure that all components are installed from the # local mirrors without any dependency errors or missing packages. self.fuel_web.deploy_cluster_wait(cluster_id)
def deploy_multiple_services_local_mirror(self): """Deploy cluster with multiple services using local mirror Scenario: 1. Revert snapshot 'empty' with default set of repositories 2. Bootstrap 5 slave nodes 3. Run 'fuel-createmirror' to replace default repositories with local mirrors 4. Create cluster with many components to check as many packages in local mirrors have correct dependencies 5. Deploy cluster Duration 50m """ self.env.revert_snapshot("empty") self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[:5]) logger.info("Executing 'fuel-createmirror' on Fuel admin node") with self.env.d_env.get_admin_remote() as remote: # TODO(ddmitriev):Enable debug via argument for 'fuel-createmirror' # when bug#1458469 fixed. if OPENSTACK_RELEASE_UBUNTU in OPENSTACK_RELEASE: cmd = ("sed -i 's/DEBUG=\"no\"/DEBUG=\"yes\"/' {}".format( '/etc/fuel-createmirror/ubuntu.cfg')) remote.execute(cmd) else: # CentOS is not supported yet, see bug#1467403 pass run_on_remote_get_results(remote, 'fuel-createmirror') # Check if there all repos were replaced with local mirrors ubuntu_id = self.fuel_web.client.get_release_id( release_name=OPENSTACK_RELEASE_UBUNTU) ubuntu_release = self.fuel_web.client.get_release(ubuntu_id) ubuntu_meta = ubuntu_release["attributes_metadata"] repos_ubuntu = ubuntu_meta["editable"]["repo_setup"]["repos"]['value'] remote_repos = [] for repo_value in repos_ubuntu: if (self.fuel_web.admin_node_ip not in repo_value['uri'] and '{settings.MASTER_IP}' not in repo_value['uri']): remote_repos.append({repo_value['name']: repo_value['uri']}) assert_true( not remote_repos, "Some repositories weren't replaced with local mirrors: " "{0}".format(remote_repos)) cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__, mode=DEPLOYMENT_MODE, settings={ 'sahara': True, 'murano': True, 'ceilometer': True, 'volumes_lvm': True, 'volumes_ceph': False, 'images_ceph': True, 'osd_pool_size': "3" }) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller', 'ceph-osd'], 'slave-02': ['compute', 'ceph-osd'], 'slave-03': ['cinder', 'ceph-osd'], 'slave-04': ['mongo'], 'slave-05': ['mongo'] }) repos_attr = self.get_cluster_repos(cluster_id) self.fuel_web.report_repos(repos_attr) # (ddmitriev): No additional checks is required after deploy, # just make sure that all components are installed from the # local mirrors without any dependency errors or missing packages. self.fuel_web.deploy_cluster_wait(cluster_id)