def generate(bundle_id: str, target_path: str, cluster_conf_url: str, **filters): """Generate a tar file.""" # duration = filters.get('duration', 'P5D') # size_limit = filters.get('size_limit', '500MB') # binlogs = filters.get('binlogs', False) # coredumps = filters.get('coredumps', False) # stacktrace = filters.get('stacktrace', False) # TODO process duration, size_limit, binlogs, coredumps and stacktrace # Find log dirs cluster_conf = MappedConf(cluster_conf_url) log_base = cluster_conf.get(CLUSTER_CONF_LOG_KEY) local_base = cluster_conf.get( f'cortx{_DELIM}common{_DELIM}storage{_DELIM}local') machine_id = Conf.machine_id if os.path.exists(_sb_tmp_src): HASupportBundle.__clear_tmp_files() else: os.makedirs(_sb_tmp_src) # Copy log files shutil.copytree(os.path.join(log_base, f'ha/{machine_id}'),\ os.path.join(_sb_tmp_src, 'logs')) # Copy configuration files shutil.copytree(os.path.join(local_base, 'ha'),\ os.path.join(_sb_tmp_src, 'conf')) HASupportBundle.__generate_tar(bundle_id, target_path) HASupportBundle.__clear_tmp_files()
def main(argv: dict): try: desc = "CORTX Kafka Setup command" command = Cmd.get_command(desc, argv[1:]) # Get kafka server list from template file kafka_config = 'kafka_config' Conf.load(kafka_config, command.url) kafka_servers = Conf.get(kafka_config, 'cortx>software>kafka>servers') # Get log path and initialise Log cluster_conf = MappedConf(command.cluster_conf) log_dir = cluster_conf.get(CLUSTER_CONF_LOG_KEY) log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}') log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('kafka_setup', log_path, level=log_level, backup_count=5,\ file_size_in_mb=5) rc = command.process(kafka_servers) if rc != 0: raise ValueError(f"Failed to run {argv[1]}") except KafkaSetupError as e: sys.stderr.write("%s\n" % str(e)) Cmd.usage(argv[0]) return e.rc() except Exception as e: sys.stderr.write("error: %s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) Cmd.usage(argv[0]) return errno.EINVAL
def cluster_upgrade(cortx_conf_url: str, force_override: bool = False): """ Description: Upgrades Cluster Components 1. Reads Cortx Config and obtain cluster components 2. Invoke upgrade phase of cluster components Paramaters: [IN] CORTX Config URL """ cortx_conf = MappedConf(cortx_conf_url) apply_phase = ProvisionerStages.UPGRADE.value node_id, node_name = CortxProvisioner._get_node_info(cortx_conf) is_valid, ret_code = CortxProvisioner._validate_provisioning_status( cortx_conf, node_id, apply_phase) if is_valid is False: if force_override is False: Log.warn('Validation check failed, Aborting upgrade with ' f'return code {ret_code}.') return ret_code else: Log.info( 'Validation check failed, Forcefully overriding upgrade.') Log.info(f"Starting cluster upgrade on {node_id}:{node_name}") CortxProvisioner._update_provisioning_status(cortx_conf, node_id, apply_phase) CortxProvisioner._provision_components(cortx_conf, UpgradeInterfaces, apply_phase) # Update CORTX version, once the upgrade is successful CortxProvisioner._add_version_info(cortx_conf, node_id) CortxProvisioner._update_provisioning_status( cortx_conf, node_id, apply_phase, ProvisionerStatus.SUCCESS.value) Log.info(f"Finished cluster upgrade on {node_id}:{node_name}")
def cluster_bootstrap(cortx_conf_url: str, force_override: bool = False): """ Description: Configures Cluster Components 1. Reads Cortx Config and obtain cluster components 2. Invoke Mini Provisioners of cluster components Paramaters: [IN] CORTX Config URL """ cortx_conf = MappedConf(cortx_conf_url) apply_phase = ProvisionerStages.DEPLOYMENT.value node_id, node_name = CortxProvisioner._get_node_info(cortx_conf) is_valid, ret_code = CortxProvisioner._validate_provisioning_status( cortx_conf, node_id, apply_phase) if is_valid is False: if force_override is False: Log.warn('Validation check failed, Aborting cluster bootstarp' f' with return code {ret_code}') return ret_code else: Log.info( 'Validation check failed, Forcefully overriding deployment.' ) Log.info(f"Starting cluster bootstrap on {node_id}:{node_name}") CortxProvisioner._update_provisioning_status(cortx_conf, node_id, apply_phase) CortxProvisioner._provision_components(cortx_conf, DeploymentInterfaces, apply_phase) CortxProvisioner._add_version_info(cortx_conf, node_id) CortxProvisioner._update_provisioning_status( cortx_conf, node_id, apply_phase, ProvisionerStatus.SUCCESS.value) Log.info(f"Finished cluster bootstrap on {node_id}:{node_name}")
def load_config(cls, solution_conf_url, cortx_conf_url): """ Load config """ cls.solution_conf_url = solution_conf_url cls.cortx_conf_url = cortx_conf_url Conf.load(cls._solution_index, cls.solution_conf_url) cls.cortx_conf = MappedConf(cls.cortx_conf_url)
def _update_provisioning_status( cortx_conf: MappedConf, node_id: str, phase: str, status: str = ProvisionerStatus.DEFAULT.value): """ Description Add phase, status, version, release keys in confstore. Args: cortx_conf: config store url. eg. yaml:///etc/cortx/cluster.conf node_id: machine-id phase: deployment/upgrade status: default/progress/success/error.""" key_prefix = f'node>{node_id}>provisioning>' keys = [(key_prefix + 'phase', phase), (key_prefix + 'status', status)] cortx_conf.set_kvs(keys)
def _get_node_list(version_conf: MappedConf): """ Get list of node Id. Parameters: version_conf - ConfStore instance of Gconf. """ node_list = [] num_storage_set = int(version_conf.get(const.NUM_STORAGESET_KEY)) for storage_set_idx in range(0, num_storage_set): num_nodes = int( version_conf.get(const.NUM_NODES_KEY % storage_set_idx)) for node_idx in range(0, num_nodes): # Add node id to node list. ex: cluster>storage_set[storage_set_idx]>nodes[node_idx]. # node list: [ '0cdf725015124cbbbc578114dbc51982' ]. node_list.append( version_conf.get(const.NODE_ID_KEY % (storage_set_idx, node_idx))) return node_list
def test_mapped_conf_add_num_keys(self): """Test if add_num_keys adds num_xx keys for xx list in the given config.""" data = {'a': '1', 'b': ['2', {'3': ['5', '6']}, '4']} sample_file = f"{dir_path}/sample_conf.yaml" create_file(sample_file, yaml.dump(data)) conf_url = f"yaml:///{sample_file}" cortx_conf = MappedConf(conf_url) cortx_conf.add_num_keys() test_index = 'test_index1' Conf.load(test_index, conf_url) # Test before saving self.assertIsNone(Conf.get(test_index, 'num_b'), "num_b key is added without even saving to conf") self.assertIsNone(Conf.get(test_index, 'b[1]>num_3'), "num_b key is added without even saving to conf") # Test after saving test_index = 'test_index2' conf_id = cortx_conf._conf_idx Conf.save(conf_id) Conf.load(test_index, conf_url) self.assertEqual(Conf.get(test_index, 'num_b'), 3) self.assertEqual(Conf.get(test_index, 'b[1]>num_3'), 2) delete_file(sample_file)
def _provision_components(cortx_conf: MappedConf, interfaces: Enum, apply_phase: str): """Invoke Mini Provisioners of cluster components.""" node_id, node_name = CortxProvisioner._get_node_info(cortx_conf) num_components = int(cortx_conf.get(f'node>{node_id}>num_components')) for interface in interfaces: for comp_idx in range(0, num_components): key_prefix = f'node>{node_id}>components[{comp_idx}]' component_name = cortx_conf.get(f'{key_prefix}>name') # Get services. service_idx = 0 services = [] while (cortx_conf.get(f'{key_prefix}>services[{service_idx}]') is not None): services.append( cortx_conf.get( f'{key_prefix}>services[{service_idx}]')) service_idx = service_idx + 1 service = 'all' if service_idx == 0 else ','.join(services) if apply_phase == ProvisionerStages.UPGRADE.value: version = cortx_conf.get(f'{key_prefix}>version') # Skip update for component if it is already updated. is_updated = CortxProvisioner._is_component_updated( component_name, version) if is_updated is True: Log.info( f'{component_name} is already updated with {version} version.' ) continue CortxProvisioner._update_provisioning_status( cortx_conf, node_id, apply_phase, ProvisionerStatus.PROGRESS.value) cmd = ( f"/opt/seagate/cortx/{component_name}/bin/{component_name}_setup {interface.value}" f" --config {cortx_conf._conf_url} --services {service}") Log.info(f"{cmd}") cmd_proc = SimpleProcess(cmd) _, err, rc = cmd_proc.run() if rc != 0: CortxProvisioner._update_provisioning_status( cortx_conf, node_id, apply_phase, ProvisionerStatus.ERROR.value) raise CortxProvisionerError(rc, "%s phase of %s, failed. %s", interface.value, component_name, err) # Update version for each component if Provisioning successful. component_version = CortxProvisioner.cortx_release.get_component_version( component_name) cortx_conf.set(f'{key_prefix}>version', component_version)
def _get_node_info(cortx_conf: MappedConf): """To get the node information.""" node_id = Conf.machine_id if node_id is None: raise CortxProvisionerError(errno.EINVAL, "Invalid node_id: %s", \ node_id) # Reinitialize logging with configured log path log_path = os.path.join(cortx_conf.get('cortx>common>storage>log'), const.APP_NAME, node_id) log_level = os.getenv('CORTX_PROVISIONER_DEBUG_LEVEL', const.DEFAULT_LOG_LEVEL) CortxProvisionerLog.reinitialize(const.SERVICE_NAME, log_path, level=log_level) if cortx_conf.get(f'node>{node_id}>name') is None: raise CortxProvisionerError( errno.EINVAL, f'Node name not found in cortx config for node {node_id}.') node_name = cortx_conf.get(f'node>{node_id}>name') return node_id, node_name
def main(): from cortx.utils.log import Log from cortx.utils.conf_store import Conf # Setup Parser parser = argparse.ArgumentParser(description='Cortx Support Bundle Interface', \ formatter_class=RawTextHelpFormatter) sub_parser = parser.add_subparsers(title='command', \ help='represents the action from: generate, get_status\n\n', \ dest='command') # Add Command Parsers members = inspect.getmembers(sys.modules[__name__]) for name, cls in members: if name != "Cmd" and name.endswith("Cmd"): cls.add_args(sub_parser) # Parse and Process Arguments try: args = parser.parse_args() cluster_conf_path = args.cluster_conf_path[0] cluster_conf = MappedConf(cluster_conf_path) log_path = os.path.join(cluster_conf.get(CLUSTER_CONF_LOG_KEY), \ f'utils/{Conf.machine_id}/support') log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('support_bundle', log_path, level=log_level, backup_count=5, \ file_size_in_mb=5) out = args.func(args) if out is not None and len(out) > 0: print(out) return 0 except Exception as e: sys.stderr.write("%s\n\n" % str(e)) sys.stderr.write("%s\n" % traceback.format_exc()) return errno.EINVAL
def get_installed_version(resource_id: str, conf_url: str): """ Get current deployed versions on the node. Parameters: resource_id - Name of the node (as in gconf). conf_url - Global Configuration URL. """ node_id = None version_info = {} version_conf = MappedConf(conf_url) # Get list of all the nodes in the cluster. node_list = Release._get_node_list(version_conf) for node in node_list: if resource_id == version_conf.get(const.NODE_NAME_KEY % node): node_id = node if node_id is None: raise SetupError(errno.EINVAL, "Invalid Resource Id %s." % resource_id) # Get version details of all the components of a node num_components = int( version_conf.get(const.NUM_COMPONENTS_KEY % node_id)) for component in range(0, num_components): _name = version_conf.get(const.COMPONENT_NAME_KEY % (node_id, component)) _version = version_conf.get(const.COMPONENT_VERSION_KEY % (node_id, component)) if _version is not None: version_info[_name] = _version else: raise SetupError( INTERNAL_ERROR, "No installed version found for component %s" % _name) # get cluster release version release_name = version_conf.get(const.RELEASE_NAME_KEY) release_version = version_conf.get(const.RELEASE_VERSION_KEY) version_info[release_name] = release_version return version_info if len(version_info) > 1 else 0
# along with this program. If not, see <https://www.gnu.org/licenses/>. # For any questions about this software or licensing, # please email [email protected] or [email protected]. import os import unittest from cortx.utils.conf_store import Conf from cortx.utils.discovery import Discovery from cortx.utils.conf_store import MappedConf from cortx.utils.kv_store import KvStoreFactory from cortx.utils.discovery.error import DiscoveryError # Load cortx common config store_type = "json" cluster_conf = MappedConf('yaml:///etc/cortx/cluster.conf') local_storage_path = cluster_conf.get('cortx>common>storage>local') config_url = "%s://%s" % ( store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf')) common_config = KvStoreFactory.get_instance(config_url) common_config.load() # Load mock data test_dir = os.path.dirname(os.path.realpath(__file__)) health_store_path = os.path.join(test_dir, 'solution/lr2/health.json') manifest_store_path = os.path.join(test_dir, 'solution/lr2/manifest.json') mock_health_data_url = "%s://%s" % (store_type, health_store_path) mock_manifest_data_url = "%s://%s" % (store_type, manifest_store_path) mock_health = "mock-health" mock_manifest = "mock-manifest" Conf.load(mock_health, mock_health_data_url)
web.post('/AuditLog/message/', \ AuditLogRequestHandler.send), web.post('/AuditLog/webhook/', \ AuditLogRequestHandler.send_webhook_info) ]) super().run_app(web, message_server_port) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Utils server CLI', formatter_class=RawTextHelpFormatter) parser.add_argument('-c', '--config', dest='cluster_conf',\ help="Cluster config file path for Support Bundle") args=parser.parse_args() cluster_conf_url = args.cluster_conf cluster_conf = MappedConf(cluster_conf_url) # Get the log path log_dir = cluster_conf.get(CLUSTER_CONF_LOG_KEY) if not log_dir: raise UtilsServerError(errno.EINVAL, "Fail to initialize logger."+\ " Unable to find log_dir path entry") utils_log_path = os.path.join(log_dir, f'utils/{Conf.machine_id}/utils_server') # Get the log level log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('utils_server', utils_log_path, level=log_level, backup_count=5, \ file_size_in_mb=5) message_bus_backend = cluster_conf.get('cortx>utils>message_bus_backend') message_server_endpoints = cluster_conf.get( f'cortx>external>{message_bus_backend}>endpoints') message_server_port = cluster_conf.get('cortx>utils>message_server_port', 28300) cluster_id = cluster_conf.get('cluster>id')
import os import re import time import errno import psutil from datetime import datetime from cortx.utils import const from cortx.utils.conf_store import MappedConf from cortx.utils.kv_store import KvStoreFactory from cortx.utils.discovery.error import DiscoveryError from cortx.utils.discovery.resource import Resource, ResourceFactory # Load cortx common config store_type = "json" cluster_conf = MappedConf(const.CLUSTER_CONF) local_storage_path = cluster_conf.get('cortx>common>storage>local') config_url = "%s://%s" % ( store_type, os.path.join(local_storage_path, 'utils/conf/cortx.conf')) common_config = KvStoreFactory.get_instance(config_url) common_config.load() # Load Discovery request status tracker try: os.makedirs(common_config.get(["discovery>resource_map>location"])[0], exist_ok=True) except PermissionError as err: raise DiscoveryError(errno.EACCES, "Failed to create default store directory. %s" % err) requests_url = "%s://%s" % (store_type, os.path.join(
def config_apply(solution_config_url: str, cortx_conf_url: str = None, force_override: bool = False): """ Description: Parses input config and store in CORTX config location Parameters: [IN] Solution Config URL [OUT] CORTX Config URL """ if Log.logger is None: CortxProvisionerLog.initialize(const.SERVICE_NAME, const.TMP_LOG_PATH) if cortx_conf_url is None: cortx_conf_url = CortxProvisioner._cortx_conf_url cortx_conf = MappedConf(CortxProvisioner._tmp_cortx_conf_url) # Load same config again if force_override is True try: cs_option = {"fail_reload": False} if force_override else {"skip_reload": True} Log.info('Applying config %s' % solution_config_url) Conf.load(CortxProvisioner._solution_index, solution_config_url, **cs_option) except ConfError as e: Log.error(f'Unable to load {solution_config_url} url, Error:{e}') # Secrets path from config file if cortx_conf.get('cortx>common>storage>local'): CortxProvisioner._secrets_path = cortx_conf.get('cortx>common>storage>local')+CortxProvisioner._rel_secret_path # source code for encrypting and storing secret key if Conf.get(CortxProvisioner._solution_index, 'cluster') is not None: CortxProvisioner.apply_cluster_config(cortx_conf, CortxProvisioner.cortx_release) if Conf.get(CortxProvisioner._solution_index, 'cortx') is not None: # generating cipher key cipher_key = None cluster_id = Conf.get(CortxProvisioner._solution_index, 'cluster>id') if cluster_id is None: cluster_id = cortx_conf.get('cluster>id') if cluster_id is None: raise CortxProvisionerError(errno.EINVAL, 'Cluster ID not specified') cipher_key = Cipher.gen_key(cluster_id, 'cortx') if cipher_key is None: raise CortxProvisionerError(errno.EINVAL, 'Cipher key not specified') for key in Conf.get_keys(CortxProvisioner._solution_index): # using path /etc/cortx/solution/secret to confirm secret if key.endswith('secret'): secret_val = Conf.get(CortxProvisioner._solution_index, key) val = None with open(os.path.join(CortxProvisioner._secrets_path, secret_val), 'rb') as secret: val = secret.read() if val is None: raise CortxProvisionerError(errno.EINVAL, f'Could not find the Secret in {CortxProvisioner._secrets_path}') val = Cipher.encrypt(cipher_key, val) # decoding the byte string in val variable Conf.set(CortxProvisioner._solution_index, key, val.decode('utf-8')) CortxProvisioner.apply_cortx_config(cortx_conf, CortxProvisioner.cortx_release) # Adding array count key in conf cortx_conf.add_num_keys() Conf.save(cortx_conf._conf_idx)
async def init(bundle_obj, node_id, config_url, **kwargs): """ Initializes the Process of Support Bundle Generation for Every Component. command: cli Command Object :type: command return: None """ cluster_conf = MappedConf(config_url) log_path = os.path.join(cluster_conf.get(CLUSTER_CONF_LOG_KEY), \ f'utils/{Conf.machine_id}/support') log_level = cluster_conf.get('utils>log_level', 'INFO') Log.init('support_bundle_node', log_path, level=log_level, \ backup_count=5, file_size_in_mb=5) bundle_id = bundle_obj.bundle_id node_name = bundle_obj.node_name comment = bundle_obj.comment components_list = bundle_obj.components services_dict = bundle_obj.services Log.info(f"components:{components_list}") bundle_path = bundle_obj.bundle_path duration = kwargs.get('duration') size_limit = kwargs.get('size_limit') binlogs = kwargs.get('binlogs') coredumps = kwargs.get('coredumps') stacktrace = kwargs.get('stacktrace') Log.debug((f"{const.SB_BUNDLE_ID}: {bundle_id}, {const.SB_NODE_NAME}: " f"{node_name}, {const.SB_COMMENT}: {comment}, " f"{const.SB_COMPONENTS}: {components_list}, {const.SOS_COMP}")) # Read support_bundle.Yaml and Check's If It Exists. cmd_setup_file = os.path.join( cluster_conf.get('install_path', DEFAULT_INSTALL_PATH), const.SUPPORT_YAML) try: support_bundle_config = Yaml(cmd_setup_file).load() except Exception as e: Log.error(f"Internal error while parsing YAML file {cmd_setup_file}{e}") if not support_bundle_config: Log.error(f"No such file {cmd_setup_file}. ERROR:{ERROR}") # Start Execution for each Component Command. command_files_info = support_bundle_config.get('COMPONENTS') # OS Logs are specifically generated hence here Even # When All is Selected O.S. Logs Will Be Skipped. for each_component in components_list: services = services_dict[each_component] components_commands = [] components_files = command_files_info[each_component] for file_path in components_files: if not os.path.exists(file_path): Log.error(f"'{file_path}' file does not exist!") continue try: file_data = Yaml(file_path).load() except Exception as e: Log.error(f"Internal error while parsing YAML file {file_path}{e}") file_data = None break if file_data: components_commands = file_data.get( const.SUPPORT_BUNDLE.lower(), []) else: Log.error(f"Support.yaml file is empty: {file_path}") break if components_commands: component, return_code = await(\ ComponentsBundle._exc_components_cmd(\ components_commands, f'{bundle_id}_{node_id}_{each_component}', f'{bundle_path}{os.sep}', each_component, node_name, comment, config_url, services, binlogs, coredumps, stacktrace, duration, size_limit)) if return_code != 0: Log.error( f"Bundle generation failed for component - '{component}'") else: Log.info( f"Bundle generation started for component - '{component}'") tar_file_name = os.path.join(bundle_path, \ f'{bundle_id}_{node_id}.tar.gz') ComponentsBundle._create_summary_file(bundle_id, node_name, \ comment, bundle_path) try: Log.debug(f"Generating tar.gz file on path {tar_file_name} " f"from {bundle_path}") Tar(tar_file_name).dump([bundle_path]) bundle_status = f"Successfully generated SB at path:{bundle_path}" except Exception as e: bundle_status = f"Failed to generate tar file. ERROR:{e}" Log.error(f"Could not generate tar file {e}") finally: if os.path.exists(bundle_path): for each_dir in os.listdir(bundle_path): comp_dir = os.path.join(bundle_path, each_dir) if os.path.isdir(comp_dir): shutil.rmtree(comp_dir) if os.path.exists(os.path.join(bundle_path, 'summary.yaml')): os.remove(os.path.join(bundle_path, 'summary.yaml')) Log.info("Support bundle generation completed.") # Update Status in ConfStor Conf.load(const.SB_INDEX, 'json://' + const.FILESTORE_PATH, fail_reload=False) Conf.set(const.SB_INDEX, f'{node_id}>{bundle_id}>status', bundle_status) Conf.save(const.SB_INDEX)
class SetupCmd(object): """Base class for setup commands.""" ldap_root_user = None rootdn_passwd = None cluster_id = None machine_id = None _preqs_conf_file = "openldapsetup_prereqs.json" ha_service_map = {} sgiam_user_key = 'cluster_config>sgiam_user' sgiam_pass_key = 'cluster_config>sgiam_password' rootdn_user_key = 'cluster_config>rootdn_user' rootdn_pass_key = 'cluster_config>rootdn_password' cluster_id_key = 'cluster_config>cluster_id' Log.init('OpenldapProvisioning', '/var/log/cortx/utils/openldap', level='DEBUG') cluster_conf = MappedConf(CLUSTER_CONF) util_install_path = cluster_conf.get('install_path') openldap_prov_config = path.join(util_install_path, "cortx/utils/conf", "openldap_prov_config.yaml") openldap_config_file = path.join(util_install_path, "cortx/utils/conf", "openldap_config.yaml") utils_tmp_dir = path.join(util_install_path, "cortx/utils/tmp") def __init__(self, config: str): """Constructor.""" if config is None: return if not config.strip(): sys.stderr.write( f'config url:[{config}] must be a valid url path\n') raise Exception('empty config URL path') self.endpoint = None self._url = config Conf.load('prov_index', self._url) Conf.load('openldap_keys_index', f'yaml://{self.openldap_prov_config}') # machine_id will be used to read confstore keys with open('/etc/machine-id') as f: self.machine_id = f.read().strip() self.cluster_id = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_CLUSTER_ID_KEY').replace( "machine-id", self.machine_id)) @property def url(self) -> str: return self._url def get_confkey(self, key: str): return Conf.get('openldap_keys_index', key) def get_confvalue(self, key: str): return Conf.get('prov_index', key) def read_endpoint_value(self): if self.endpoint is None: self.endpoint = self.get_confvalue( self.get_confkey('TEST>CONFSTORE_ENDPOINT_KEY')) def read_ldap_credentials(self): """Read ldap credentials (rootdn, sgiam) from the openldap_config file.""" try: # Load the openldap config file index_id = 'openldap_config_file_read_index' Conf.load(index_id, f'yaml://{self.openldap_config_file}') # Read the cluster id from openldap_config file self.cluster_id = Conf.get(index_id, f'{self.cluster_id_key}') cipher_key = Cipher.generate_key( self.cluster_id, self.get_confkey('CONFSTORE_OPENLDAP_CONST_KEY')) # rootdn username/password self.ldap_root_user = Conf.get(index_id, f'{self.rootdn_user_key}') encrypted_rootdn_pass = Conf.get(index_id, f'{self.rootdn_pass_key}') if encrypted_rootdn_pass != None: self.rootdn_passwd = Cipher.decrypt( cipher_key, bytes(str(encrypted_rootdn_pass), 'utf-8')) except Exception as e: Log.error(f'read ldap credentials failed, error: {e}') raise e def update_cluster_id(self): """Update 'cluster_id' to openldap_config file.""" try: if path.isfile(f'{self.openldap_config_file}') == False: Log.error(f'{self.openldap_config_file} must be present') raise Exception(f'{self.openldap_config_file} must be present') else: key = 'cluster_config>cluster_id' index_id = 'openldap_config_file_cluster_id_index' Conf.load(index_id, f'yaml://{self.openldap_config_file}') Conf.set(index_id, f'{key}', f'{self.cluster_id}') Conf.save(index_id) updated_cluster_id = Conf.get(index_id, f'{key}') if updated_cluster_id != self.cluster_id: Log.error( f'failed to set {key}: {self.cluster_id} in {self.openldap_config_file} ' ) raise Exception( f'failed to set {key}: {self.cluster_id} in {self.openldap_config_file} ' ) except Exception as e: Log.error(f'Update cluster id failed, error: {e}') raise Exception(f'exception: {e}') def update_ldap_credentials(self): """Update ldap credentials (rootdn, sgiam) to openldap_config file.""" try: # Load the openldap config file index_id = 'openldap_config_file_write_index' Conf.load(index_id, f'yaml://{self.openldap_config_file}') # get the rootdn credentials from provisoner config file # set the rootdn credentials in to openldap_config file ldap_root_user = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_ROOTDN_USER_KEY')) encrypted_rootdn_pass = self.get_confvalue( self.get_confkey('CONFIG>CONFSTORE_ROOTDN_PASSWD_KEY')) if encrypted_rootdn_pass is None: Log.error('rootdn password cannot be None.') raise Exception('rootdn password cannot be None.') Conf.set(index_id, f'{self.rootdn_user_key}', f'{ldap_root_user}') Conf.set(index_id, f'{self.rootdn_pass_key}', f'{encrypted_rootdn_pass}') # save openldap config file Conf.save(index_id) except Exception as e: Log.error(f'update rootdn credentials failed, error: {e}') raise Exception(f'update rootdn credentials failed, error: {e}') def restart_services(self, services_list): """Restart services specified as parameter.""" for service_name in services_list: try: # if service name not found in the ha_service_map then use systemctl service_name = self.ha_service_map[service_name] cmd = ['cortx', 'restart', f'{service_name}'] except KeyError: cmd = ['/bin/systemctl', 'restart', f'{service_name}'] handler = SimpleProcess(cmd) res_op, res_err, res_rc = handler.run() if res_rc != 0: raise Exception( f"{cmd} failed with err: {res_err}, out: {res_op}, ret: {res_rc}" ) def validate_pre_requisites(self, rpms: list = None, pip3s: list = None, services: list = None, files: list = None): """Validate pre requisites using cortx-py-utils validator.""" sys.stdout.write(f'Validations running from {self._preqs_conf_file}\n') if pip3s: PkgV().validate('pip3s', pip3s) if services: ServiceV().validate('isrunning', services) if rpms: PkgV().validate('rpms', rpms) if files: PathV().validate('exists', files) def phase_prereqs_validate(self, phase_name: str): """Validate pre requisites using cortx-py-utils validator for the 'phase_name'.""" if not os.path.isfile(self._preqs_conf_file): raise FileNotFoundError( f'pre-requisite json file: {self._preqs_conf_file} not found') Conf.load('preqsConfFileIndex', f'json://{self._preqs_conf_file}') try: prereqs_block = Conf.get('preqsConfFileIndex', f'{phase_name}') if prereqs_block is not None: self.validate_pre_requisites( rpms=Conf.get('preqsConfFileIndex', f'{phase_name}>rpms'), services=Conf.get('preqsConfFileIndex', f'{phase_name}>services'), pip3s=Conf.get('preqsConfFileIndex', f'{phase_name}>pip3s'), files=Conf.get('preqsConfFileIndex', f'{phase_name}>files')) except Exception as e: raise OpenldapPROVError( f'ERROR: {phase_name} prereqs validations failed, exception: {e} \n' ) def extract_yardstick_list(self, phase_name: str): """Extract keylist to be used as yardstick for validating keys of each phase.""" # The openldap prov config file has below pairs : # "Key Constant" : "Actual Key" # Example of "Key Constant" : # CONFSTORE_SITE_COUNT_KEY # PREPARE # CONFIG>CONFSTORE_LDAPADMIN_USER_KEY # INIT # Example of "Actual Key" : # cluster>cluster-id>site>storage_set_count # cortx>software>openldap>sgiam>user # # When we call get_all_keys on openldap prov config # file, it returns all the "Key Constant", # which will contain PHASE(name) as the root # attribute (except for unsolicited keys). # To get "Actual Key" from each "Key Constant", # we need to call get_confkey on every such key. # # Note that for each of these "Key Constant", # there may not exist an "Actual Key" because # some phases do not have any "Actual Key". # Example of such cases - # POST_INSTALL # PREPARE # For such examples, we skip and continue with # remaining keys. prov_keys_list = Conf.get_keys('openldap_keys_index') # We have all "Key Constant" in prov_keys_list, # now extract "Actual Key" if it exists and # depending on phase and hierarchy, decide # whether it should be added to the yardstick # list for the phase passed here. yardstick_list = [] prev_phase = True next_phase = False for key in prov_keys_list: # If PHASE is not relevant, skip the key. # Or set flag as appropriate. For test, # reset and cleanup, do not inherit keys # from previous phases. if next_phase: break if key.find(phase_name) == 0: prev_phase = False else: if (phase_name == "TEST" or phase_name == "RESET" or phase_name == "CLEANUP"): continue if not prev_phase: next_phase = True break value = self.get_confkey(key) # If value does not exist which can be the # case for certain phases as mentioned above, # skip the value. if value is None: continue yardstick_list.append(value) return yardstick_list def phase_keys_validate(self, arg_file: str, phase_name: str): # Setting the desired values before we begin if self.machine_id is not None: machine_id_val = self.machine_id if self.cluster_id is not None: cluster_id_val = self.cluster_id # The 'storage_set_count' is read using # below hard-coded key which is the max # array size for storage set. storage_set_count_key = "cluster>cluster-id>site>storage_set_count" if self.cluster_id is not None: storage_set_count_key = storage_set_count_key.replace( "cluster-id", cluster_id_val) storage_set_count_str = self.get_confvalue(storage_set_count_key) if storage_set_count_str is not None: storage_set_val = int(storage_set_count_str) else: storage_set_val = 0 # Set phase name to upper case required for inheritance phase_name = phase_name.upper() try: # Extract keys from yardstick file for current phase considering inheritance yardstick_list = self.extract_yardstick_list(phase_name) # Set argument file confstore Conf.load('argument_file_index', arg_file) # Extract keys from argument file arg_keys_list = Conf.get_keys() # Since get_all_keys misses out listing entries inside # an array, the below code is required to fetch such # array entries. The result will be stored in a full # list which will be complete and will be used to verify # keys required for each phase. full_arg_keys_list = [] for key in arg_keys_list: if ((key.find('[') != -1) and (key.find(']') != -1)): storage_set = self.get_confvalue(key) base_key = key for set_key in storage_set: key = base_key + ">" + set_key full_arg_keys_list.append(key) else: full_arg_keys_list.append(key) # Below algorithm uses tokenization # of both yardstick and argument key # based on delimiter to generate # smaller key-tokens. Then check if # (A) all the key-tokens are pairs of # pre-defined token. e.g., # if key_yard is machine-id, then # key_arg must have corresponding # value of machine_id_val. # OR # (B) both the key-tokens from key_arg # and key_yard are the same. list_match_found = True key_match_found = False for key_yard in yardstick_list: key_yard_token_list = re.split('>|\[|\]', key_yard) key_match_found = False for key_arg in full_arg_keys_list: if key_match_found is False: key_arg_token_list = re.split('>|\[|\]', key_arg) if len(key_yard_token_list) == len(key_arg_token_list): for key_x, key_y in zip(key_yard_token_list, key_arg_token_list): key_match_found = False if key_x == "machine-id": if key_y != machine_id_val: break elif key_x == "cluster-id": if key_y != cluster_id_val: break elif key_x == "storage-set-count": if int(key_y) >= storage_set_val: break elif key_x != key_y: break key_match_found = True if key_match_found is False: list_match_found = False break if list_match_found is False: raise Exception(f'No match found for {key_yard}') sys.stdout.write("Validation complete\n") except Exception as e: raise Exception(f'ERROR : Validating keys failed, exception {e}\n')
def _validate_provisioning_status(cortx_conf: MappedConf, node_id: str, apply_phase: str): """Validate provisioning.""" ret_code = 0 recent_phase = cortx_conf.get(f'node>{node_id}>provisioning>phase') recent_status = cortx_conf.get(f'node>{node_id}>provisioning>status') msg = f'Recent phase for this node is {recent_phase} and ' + \ f'recent status is {recent_status}. ' # {apply_phase: {recent_phase: {recent_status: [boolean_result,rc]}}} validations_checks = { ProvisionerStages.DEPLOYMENT.value: { ProvisionerStages.DEPLOYMENT.value: { ProvisionerStatus.DEFAULT.value: [True, 0], ProvisionerStatus.ERROR.value: [True, 0], ProvisionerStatus.PROGRESS.value: [True, 0], ProvisionerStatus.SUCCESS.value: [False, 0] }, ProvisionerStages.UPGRADE.value: { ProvisionerStatus.DEFAULT.value: [True, 0], ProvisionerStatus.ERROR.value: [True, 0], ProvisionerStatus.PROGRESS.value: [True, 0], ProvisionerStatus.SUCCESS.value: [True, 0] } }, ProvisionerStages.UPGRADE.value: { ProvisionerStages.DEPLOYMENT.value: { ProvisionerStatus.DEFAULT.value: [False, 1], ProvisionerStatus.ERROR.value: [False, 1], ProvisionerStatus.PROGRESS.value: [False, 1], ProvisionerStatus.SUCCESS.value: [True, 0] }, ProvisionerStages.UPGRADE.value: { ProvisionerStatus.DEFAULT.value: [True, 0], ProvisionerStatus.ERROR.value: [True, 0], ProvisionerStatus.PROGRESS.value: [True, 0], ProvisionerStatus.SUCCESS.value: [True, 0] } } } if recent_phase is None and recent_status is None: Log.info(msg + f'Performing {apply_phase} on this node.') return True, ret_code if (not validations_checks.get(apply_phase) or not validations_checks.get(apply_phase).get(recent_phase) or not validations_checks.get(apply_phase).get( recent_phase).get(recent_status)): Log.error('Invalid phase or status.') ret_code = 1 return False, ret_code validate_result = validations_checks.get(apply_phase).get( recent_phase).get(recent_status) if validate_result[1] != 0: Log.error(msg + f'{apply_phase} is not possible on this node.') if apply_phase == ProvisionerStages.UPGRADE.value: # Reset status. recent_status = cortx_conf.set( f'node>{node_id}>provisioning>status', ProvisionerStatus.DEFAULT.value) else: Log.info(msg) return validate_result[0], validate_result[1]
def _add_version_info(cortx_conf: MappedConf, node_id): """Add version in confstore.""" version = CortxProvisioner.cortx_release.get_release_version() cortx_conf.set('cortx>common>release>version', version) cortx_conf.set(f'node>{node_id}>provisioning>version', version)