def validate(self, phase: str): """Perform validations for phase.""" try: Conf.load(phase, f'json://{self._preqs_conf_file}') prereqs_block = Conf.get(phase, f'{phase}') if prereqs_block: pip3s = Conf.get(phase, f'{phase}>pip3s') if pip3s: PkgV().validate('pip3s', pip3s) rpms = Conf.get(phase, f'{phase}>rpms') if rpms: PkgV().validate('rpms', rpms) services = Conf.get(phase, f'{phase}>services') if services: for service in services: pid = os.popen('pidof ' + service).read() if pid is None: Log.debug( 'Validation failed for service %s in %s phase' % (service, phase)) raise Exception( 'Validation failed for service %s in %s phase' % (service, phase)) files = Conf.get(phase, f'{phase}>files') if files: PathV().validate('exists', files) Log.debug("%s - pre-requisite validation complete" % phase) except Exception: Log.debug("%s - pre-requisite validation failed" % phase) raise Exception("prereqs validation failed") return 0
def validate_dependencies(setup): """Validate pre-requisites software packages.""" pip3_3ps_packages_main = { "cryptography": "2.8", "jsonschema": "3.2.0", "pika": "1.1.0", "pyinotify": "0.9.6", "python-daemon": "2.2.4", "requests": "2.25.1", "zope.component": "4.6.2", "zope.event": "4.5.0", "zope.interface": "5.2.0" } rpm_3ps_packages = { "hdparm": "9.43", "ipmitool": "1.8.18", "lshw": "B.02.18", "python3": "3.6.8", "python36-dbus": "1.2.4", "python36-gobject": "3.22.0", "python36-paramiko": "2.1.1", "python36-psutil": "5.6.7", "shadow-utils": "4.6", "smartmontools": "7.0", "systemd-python36": "1.0.0", "udisks2": "2.8.4" } ssu_dependency_rpms = [ "sg3_utils", "gemhpi", "pull_sea_logs", "python-hpi", "zabbix-agent-lib", "zabbix-api-gescheit", "zabbix-xrtx-lib", "python-openhpi-baselib", "zabbix-collector" ] ssu_required_process = [ "openhpid", "dcs-collectord" ] vm_dependency_rpms = [] pkg_validator = PkgV() pkg_validator.validate_pip3_pkgs(host=socket.getfqdn(), pkgs=pip3_3ps_packages_main, skip_version_check=False) pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(), pkgs=rpm_3ps_packages, skip_version_check=False) # Check for sspl required processes and misc dependencies if # setup/role is other than cortx if setup == "ssu": pkg_validator.validate("rpms", ssu_dependency_rpms) ServiceV().validate("isrunning", ssu_required_process) elif setup == "vm" or setup == "gw" or setup == "cmu": # No dependency currently. Keeping this section as it # may be needed in future. pkg_validator.validate("rpms", vm_dependency_rpms)
def validate(): """Check for required packages are installed.""" # python 3rd party package dependency pip3_3ps_packages_test = {"Flask": "1.1.1"} pkg_validator = PkgV() pkg_validator.validate_pip3_pkgs(host=None, pkgs=pip3_3ps_packages_test, skip_version_check=False)
def validate_pre_requisites(self, rpms: list = None, pip3s: list = None, services: list = None): try: if pip3s: PkgV().validate('pip3s', pip3s) if services: ServiceV().validate('isrunning', services) if rpms: PkgV().validate('rpms', rpms) except Exception as e: print(f"{e}, config:{self.__preqs_conf_file}") return False return True
def check_dependencies(self): # Check for dependency rpms and required processes active state based # on role if self.role == "ssu": PkgV().validate("rpms", self.SSU_DEPENDENCY_RPMS) ServiceV().validate("isrunning", self.SSU_REQUIRED_PROCESSES, is_process=True) elif self.role == "vm" or self.role == "gw" or self.role == "cmu": # No dependency currently. Keeping this section as it may be # needed in future. PkgV().validate("rpms", self.VM_DEPENDENCY_RPMS)
def validate_pre_requisites(self, rpms: list = None, pip3s: list = None, services: list = None, files: list = None): """Validate pre requisites using cortx-py-utils validator.""" sys.stdout.write(f'Validations running from {self._preqs_conf_file}\n') if pip3s: PkgV().validate('pip3s', pip3s) if services: ServiceV().validate('isrunning', services) if rpms: PkgV().validate('rpms', rpms) if files: PathV().validate('exists', files)
def _validate_kafka_installation(): """validates kafka is installed and kafka user and group are present """ # check kafka package installed try: PkgV().validate('rpms', ['kafka']) except Exception as e: raise KafkaSetupError(e.rc, e) # check kafak user exists try: kafka_user = get_user_by_name('kafka') kafka_group = get_group_by_name('kafka') if kafka_group.gr_gid != kafka_user.pw_gid: raise Exception except Exception as e: # create kafka user and group cmds = [ "adduser kafka", "usermod -aG wheel kafka", "groupadd --force kafka", "usermod --append --groups kafka kafka"] for cmd in cmds: _, err, rc = SimpleProcess(cmd).run() # rc 9 if kafka user already exists & 12 if kafka user created if rc not in (0, 9, 12): raise KafkaSetupError(rc, \ "Failed in creating kafka user and group", err)
def post_install(): """ Performs post install operations """ # check whether zookeeper and kafka are running ServiceV().validate('isrunning', ['kafka-zookeeper.service', \ 'kafka.service']) # Check required python packages utils_path = Utils._get_utils_path() with open(f"{utils_path}/conf/python_requirements.txt") as file: req_pack = [] for package in file.readlines(): pack = package.strip().split('==') req_pack.append(f"{pack[0]} ({pack[1]})") try: with open(f"{utils_path}/conf/python_requirements.ext.txt" ) as extfile: for package in extfile.readlines(): pack = package.strip().split('==') req_pack.append(f"{pack[0]} ({pack[1]})") except Exception: Log.info("Not found: " + f"{utils_path}/conf/python_requirements.ext.txt") PkgV().validate(v_type='pip3s', args=req_pack) return 0
def test_pip3_installed(self): """Check if pip3 pkg installed.""" try: pkg = ['toml'] PkgV().validate('pip3s', pkg) except Exception as e: self.fail("{}".format(e))
def validate(self): """Validate test command arguments.""" if not self.args.config: msg = "%s - Argument validation failure. %s" % ( self.name, "Global config is required.") logger.error(msg) raise SetupError(errno.EINVAL, msg) if not self.args.plan: msg = "%s - Argument validation failure. Test plan is needed" % ( self.name) logger.error(msg) raise SetupError(errno.EINVAL, msg) result = PkgV().validate("rpms", "sspl-test") if result == -1: msg = "'sspl-test' rpm pkg not found." logger.error(msg) raise SetupError(1, msg) # Service restart is required for coverage. # Hence it can be enabled only with test plans # which are present in TEST_REQ_SERVICE_RESTART list. if self.args.coverage and self.args.plan[ 0] not in TEST_REQ_SERVICE_RESTART: msg = "Code coverage can not be enabled for %s test plan." \ % self.args.plan[0] logger.error(msg) raise SetupError(errno.EINVAL, "%s - Argument validation failure. %s", self.name, msg) logger.info("%s - Validation done" % self.name)
def _validate_kafka_installation(): """Validates kafka is installed and kafka user and group are present.""" # check kafka package installed try: PkgV().validate('rpms', ['kafka']) except Exception as e: Log.error(f"Kafka rpm missing: {e}") raise KafkaSetupError(e.rc, e) # check kafak user exists try: kafka_user = get_user_by_name('kafka') kafka_group = get_group_by_name('kafka') if kafka_group.gr_gid != kafka_user.pw_gid: raise Exception except Exception as e: Log.error(f"Kafka user/group missing: {e}") # create kafka user and group cmds = [ "adduser kafka", "usermod -aG wheel kafka", "groupadd --force kafka", "usermod --append --groups kafka kafka" ] Log.info("Creating Kafka user and group") for cmd in cmds: _, err, rc = SimpleProcess(cmd).run() # rc 9 if kafka user already exists & 12 if kafka user created if rc not in (0, 9, 12): Log.debug(f"Failed in running command :{cmd}") Log.error(f"Failed in creating kafka user/group:{err}") raise KafkaSetupError(rc,\ "Failed in creating kafka user and group", err)
def validate(self, phase: str): """ Perform validations. Raises exceptions if validation fails. """ if phase == "post_install": # Perform python pkg validations. pip3_packages = { "elasticsearch": "7.12.0", "elasticsearch-dsl": "7.3.0" } # Validate pre-requisites software packages. rpm_packages = {"opendistroforelasticsearch": "1.12.0"} try: PkgV().validate_pip3_pkgs(host=socket.getfqdn(), pkgs=pip3_packages, skip_version_check=False) PkgV().validate_rpm_pkgs(host=socket.getfqdn(), pkgs=rpm_packages, skip_version_check=False) PkgV().validate("rpms", ["java-1.8.0-openjdk-headless"]) except VError as e: msg = f"Validation failed for {phase} phase with error {e}." Log.error(msg) raise elif phase == "config": if os.path.exists(self.opendistro_security_plugin): msg = ('Opendistro_security plugin path ' f'{self.opendistro_security_plugin} is not deleted.') Log.error(msg) raise Exception(msg) elif phase == "init": Log.info("No validation needed for init phase.") elif phase == "prepare": Log.info("No validation needed for prepare phase.") elif phase == "cleanup": Log.info("No validation needed for cleanup phase.") elif phase == "pre_upgrade": Log.info("No validation needed for pre_upgrade phase.") elif phase == "post_upgrade": Log.info("No validation needed for post_upgrade phase.") return 0
def validate_pre_requisites(self, rpms: list = None, pip3s: list = None, services: list = None, files: list = None): """Validate pre requisites using cortx-py-utils validator.""" try: if pip3s: PkgV().validate('pip3s', pip3s) if services: ServiceV().validate('isrunning', services) if rpms: PkgV().validate('rpms', rpms) if files: PathV().validate('exists', files) except Exception as e: sys.stderr.write('ERROR: post_install validations failed.\n') sys.stderr.write(f"{e}, config:{self._preqs_conf_file}\n") raise e
def validate(self): """Check for required packages are installed.""" # RPM dependency rpm_deps = {"cortx-sspl-test": None} # python 3rd party package dependency pip3_3ps_packages_test = {"Flask": "1.1.1"} pkg_validator = PkgV() pkg_validator.validate_pip3_pkgs(host=socket.getfqdn(), pkgs=pip3_3ps_packages_test, skip_version_check=False) pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(), pkgs=rpm_deps, skip_version_check=True) # Load global, sspl and test configs Conf.load(SSPL_CONFIG_INDEX, sspl_config_path) Conf.load(SSPL_TEST_CONFIG_INDEX, sspl_test_config_path) # Take copy of supplied config passed to sspl_test and load it with open(self.sspl_test_gc_copy_file, "w") as f: f.write("") self.sspl_test_gc_copy_url = "yaml://%s" % self.sspl_test_gc_copy_file Conf.load(SSPL_TEST_GLOBAL_CONFIG, self.sspl_test_gc_copy_url) Conf.load("global_config", self.sspl_test_gc_url) Conf.copy("global_config", SSPL_TEST_GLOBAL_CONFIG) # Validate input configs machine_id = Utility.get_machine_id() self.node_type = Conf.get(SSPL_TEST_GLOBAL_CONFIG, "server_node>%s>type" % machine_id) enclosure_id = Conf.get( SSPL_TEST_GLOBAL_CONFIG, "server_node>%s>storage>enclosure_id" % machine_id) self.enclosure_type = Conf.get( SSPL_TEST_GLOBAL_CONFIG, "storage_enclosure>%s>type" % enclosure_id)
def validate(self): """Validate test command arguments.""" if not self.args.config: raise SetupError(errno.EINVAL, "%s - Argument validation failure. %s", self.name, "Global config is required.") if not self.args.plan: raise SetupError( errno.EINVAL, "%s - Argument validation failure. Test plan is needed", self.name) result = PkgV().validate("rpms", "sspl-test") if result == -1: raise SetupError(1, "'sspl-test' rpm pkg not found.")
def validate(self, phase: str): """Perform validations for phase.""" try: Conf.load(phase, f'json://{self._preqs_conf_file}') prereqs_block = Conf.get(phase, f'{phase}') if prereqs_block: pip3s = Conf.get(phase, f'{phase}>pip3s') if pip3s: PkgV().validate('pip3s', pip3s) rpms = Conf.get(phase, f'{phase}>rpms') if rpms: PkgV().validate('rpms', rpms) services = Conf.get(phase, f'{phase}>services') if services: ServiceV().validate('isrunning', services) files = Conf.get(phase, f'{phase}>files') if files: PathV().validate('exists', files) Log.debug("%s - pre-requisite validation complete" % phase) except Exception: Log.debug("%s - pre-requisite validation failed" % phase) raise Exception("prereqs validation failed") return 0
def validate(self): """Check below requirements are met in setup. 1. required python 3rd party packages are installed 2. required rpm dependencies are installed 3. product specified in global config is supported by SSPL """ # SSPL python 3rd party package dependencies pip3_3ps_packages_main = { "cryptography": "2.8", "jsonschema": "3.2.0", "pika": "1.1.0", "pyinotify": "0.9.6", "python-daemon": "2.2.4", "requests": "2.25.1", "zope.component": "4.6.2", "zope.event": "4.5.0", "zope.interface": "5.2.0" } # SSPL 3rd party RPM dependencies rpm_3ps_packages = { "hdparm": "9.43", "ipmitool": "1.8.18", "lshw": "B.02.18", "python3": "3.6.8", "python36-dbus": "1.2.4", "python36-gobject": "3.22.0", "python36-paramiko": "2.1.1", "python36-psutil": "5.6.7", "rabbitmq-server": "3.8.9", "shadow-utils": "4.6", "smartmontools": "7.0", "systemd-python36": "1.0.0", "udisks2": "2.8.4" } pkg_validator = PkgV() pkg_validator.validate_pip3_pkgs(host=None, pkgs=pip3_3ps_packages_main, skip_version_check=False) pkg_validator.validate_rpm_pkgs(host=None, pkgs=rpm_3ps_packages, skip_version_check=False) self.PRODUCT_NAME = Conf.get(GLOBAL_CONFIG_INDEX, 'release>product') # Validate product if not self.PRODUCT_NAME: raise SetupError(1, "%s - validation failure. %s", self.name, "Product not found in global config copy.") if not enabled_products: raise SetupError(1, "No enabled products!") if self.PRODUCT_NAME not in enabled_products: raise SetupError( 1, "Product '%s' is not in enabled products list: %s", self.PRODUCT_NAME, enabled_products)
def process(self): """ Process post_install command. """ Log.info("Processing post_install command") try: if os.path.exists(const.CONFIG_DIR): shutil.rmtree(const.CONFIG_DIR) os.makedirs(const.CONFIG_DIR, exist_ok=True) # Create a directory and copy config file PostInstallCmd.copy_file(const.SOURCE_CONFIG_FILE, const.HA_CONFIG_FILE) PostInstallCmd.copy_file( f"{const.SOURCE_CONFIG_PATH}/{const.CM_CONTROLLER_INDEX}.json", const.CM_CONTROLLER_SCHEMA) PostInstallCmd.copy_file(const.SOURCE_ALERT_FILTER_RULES_FILE, const.ALERT_FILTER_RULES_FILE) PostInstallCmd.copy_file(const.SOURCE_CLI_SCHEMA_FILE, const.CLI_SCHEMA_FILE) PostInstallCmd.copy_file(const.SOURCE_SERVICE_FILE, const.SYSTEM_SERVICE_FILE) self._execute.run_cmd("systemctl daemon-reload") Log.info(f"{self.name}: Copied HA configs file.") # Pre-requisite checks are done here. # Make sure that cortx necessary packages have been installed if PostInstallCmd.DEV_CHECK != True: PkgV().validate('rpms', const.CORTX_CLUSTER_PACKAGES) Log.info("Found required cluster packages installed.") # Check user and group groups = [ g.gr_name for g in grp.getgrall() if const.CLUSTER_USER in g.gr_mem ] gid = pwd.getpwnam(const.CLUSTER_USER).pw_gid groups.append(grp.getgrgid(gid).gr_name) if not const.USER_GROUP_HACLIENT in groups: Log.error("hacluster is not a part of the haclient group") raise HaPrerequisiteException("post_install command failed") else: Log.info("hacluster is a part of the haclient group") except Exception as e: Log.error(f"Failed prerequisite with Error: {e}") raise HaPrerequisiteException("post_install command failed") Log.info("post_install command is successful")
def post_install(post_install_template: str): """ Performs post install operations """ # check whether zookeeper and kafka are running ServiceV().validate('isrunning', ['kafka-zookeeper.service', \ 'kafka.service']) # Check required python packages install_path = Utils._get_from_conf_file('install_path') utils_path = install_path + '/cortx/utils' with open(f"{utils_path}/conf/python_requirements.txt") as file: req_pack = [] for package in file.readlines(): pack = package.strip().split('==') req_pack.append(f"{pack[0]} ({pack[1]})") try: with open(f"{utils_path}/conf/python_requirements.ext.txt" ) as extfile: for package in extfile.readlines(): pack = package.strip().split('==') req_pack.append(f"{pack[0]} ({pack[1]})") except Exception: Log.info("Not found: " + f"{utils_path}/conf/python_requirements.ext.txt") PkgV().validate(v_type='pip3s', args=req_pack) default_sb_path = '/var/log/cortx/support_bundle' Utils._set_to_conf_file('support>local_path', default_sb_path) os.makedirs(default_sb_path, exist_ok=True) post_install_template_index = 'post_install_index' Conf.load(post_install_template_index, post_install_template) machine_id = Conf.machine_id key_list = [ f'server_node>{machine_id}>hostname', f'server_node>{machine_id}>name' ] ConfKeysV().validate('exists', post_install_template_index, key_list) #set cluster nodename:hostname mapping to cluster.conf (needed for Support Bundle) Conf.load('cluster', 'json:///etc/cortx/cluster.conf', skip_reload=True) Utils._copy_cluster_map(post_install_template_index) return 0
def validate(self, stage): if stage == "post_install": PkgV().validate('rpms', ['consul']) elif stage == "config": keys = [ f"server_node>{Conf.machine_id}>network>data>private_interfaces", f"server_node>{Conf.machine_id}>network>data>private_fqdn", "cortx>software>consul>config_path", "cortx>software>consul>data_path", ] for key in keys: value = Conf.get(self.index, key) if not value: raise ConfError( errno.EINVAL, "Consul Setup config validation falied. %s key not found", key) max_retry = 3 server_node_fqdns = [ Conf.get( self.index, f"server_node>{machine_id}>network>data>private_fqdn") for machine_id in Conf.get(self.index, "server_node").keys() ] for i in range(max_retry): try: NetworkV().validate("connectivity", server_node_fqdns) break except VError: if i == (max_retry - 1): raise time.sleep(0.5) elif stage == "cleanup": keys = [ "cortx>software>consul>config_path", "cortx>software>consul>data_path", ] for key in keys: value = Conf.get(self.index, key) if not value: raise ConfError( errno.EINVAL, "Consul Setup config validation falied. %s key not found", key)
def validate(self): if self.args: try: if "--config" in self.args: conf_index = self.args.index("--config") global_config = self.args[conf_index + 1] # Provision for --config <global_config_url> for future use-case. # Conf.load('global_config', global_config) if "--plan" in self.args: plan_index = self.args.index("--plan") if self.args[plan_index + 1] not in self.sspl_test_plans: raise SetupError( 1, "Invalid plan type specified. Please check usage") except Exception as ex: raise SetupError(1, "%s - validation failure. %s", self.name, str(ex)) result = PkgV().validate("rpms", "sspl-test") if result == -1: raise SetupError(1, "'sspl-test' rpm pkg not found.")
def process(self): """ Process post_install command. """ Log.info("Processing post_install command") try: # Create a directory and copy config file if os.path.exists(const.HA_CONFIG_FILE): os.remove(const.HA_CONFIG_FILE) os.makedirs(const.CONFIG_DIR, exist_ok=True) shutil.copyfile(const.SOURCE_CONFIG_FILE, const.HA_CONFIG_FILE) Log.info(f"{self.name}: Copied HA config file.") # Pre-requisite checks are done here. # Make sure the pacemaker, corosync and pcs packages have been installed PkgV().validate('rpms', const.PCS_CLUSTER_PACKAGES) Log.info("Found required cluster packages installed.") except Exception as e: Log.error(f"Failed prerequisite with Error: {e}") raise HaPrerequisiteException("post_install command failed") Log.info("post_install command is successful")
def test(plan: str): """ Perform configuration testing """ # Runs cortx-py-utils unittests as per test plan try: Log.info("Validating cortx-py-utils-test rpm") PkgV().validate('rpms', ['cortx-py-utils-test']) utils_path = Utils._get_utils_path() import cortx.utils.test as test_dir plan_path = os.path.join(os.path.dirname(test_dir.__file__), \ 'plans/', plan + '.pln') Log.info("Running test plan: %s", plan) cmd = "%s/bin/run_test -t %s" % (utils_path, plan_path) _output, _err, _rc = SimpleProcess(cmd).run(realtime_output=True) if _rc != 0: Log.error("Py-utils Test Failed") raise TestFailed("Py-utils Test Failed. \n Output : %s "\ "\n Error : %s \n Return Code : %s" %(_output, _err, _rc)) except VError as ve: Log.error("Failed at package Validation: %s", ve) raise SetupError(errno.EINVAL, "Failed at package Validation:"\ " %s", ve) return 0
def validate(self): """Check for required packages are installed.""" # RPM dependency rpm_deps = {"cortx-sspl-test": None} # python 3rd party package dependency pip3_packages_dep = {"Flask": "1.1.1", "coverage": "5.5"} if not self.coverage_enabled: pip3_packages_dep.pop("coverage") # Validate pip3 python pkg with required version. for pkg, version in pip3_packages_dep.items(): installed_pkg = None uninstalled_pkg = False try: pkg_req = Requirement.parse(f"{pkg}=={version}") installed_pkg = working_set.find(pkg_req) except VersionConflict: cmd = f'pip3 uninstall -y {pkg}' _, err, ret = SimpleProcess(cmd).run() if ret: raise TestException( "Failed to uninstall the pip3 pkg: %s(v%s), " "due to an Error: %s" % (pkg, version, err)) uninstalled_pkg = True except Exception as err: raise TestException("Failed at verification of pip3 pkg: %s, " "due to an Error: %s" % (pkg, err)) if not installed_pkg or uninstalled_pkg: cmd = f'pip3 install {pkg}=={version}' _, err, ret = SimpleProcess(cmd).run() if ret: raise TestException( "Failed to install the pip3 pkg: %s(v%s), " "due to an Error: %s" % (pkg, version, err)) logger.info(f"Ensured Package Dependency: {pkg}(v{version}).") # Validate rpm dependencies pkg_validator = PkgV() pkg_validator.validate_rpm_pkgs(host=socket.getfqdn(), pkgs=rpm_deps, skip_version_check=True) # Load global, sspl and test configs Conf.load(SSPL_CONFIG_INDEX, sspl_config_path) Conf.load(SSPL_TEST_CONFIG_INDEX, sspl_test_config_path) # Take copy of supplied config passed to sspl_test and load it with open(self.sspl_test_gc_copy_file, "w") as f: f.write("") self.sspl_test_gc_copy_url = "yaml://%s" % self.sspl_test_gc_copy_file Conf.load(SSPL_TEST_GLOBAL_CONFIG, self.sspl_test_gc_copy_url) Conf.load("global_config", self.sspl_test_gc_url) Conf.copy("global_config", SSPL_TEST_GLOBAL_CONFIG) # Validate input configs machine_id = Utility.get_machine_id() self.node_type = Conf.get(SSPL_TEST_GLOBAL_CONFIG, "server_node>%s>type" % machine_id) enclosure_id = Conf.get( SSPL_TEST_GLOBAL_CONFIG, "server_node>%s>storage>enclosure_id" % machine_id) self.enclosure_type = Conf.get( SSPL_TEST_GLOBAL_CONFIG, "storage_enclosure>%s>type" % enclosure_id)
def test_neg_rpm_installed(self): """Check if neagtive rpm pkg installed.""" neg_pkg = ['lvm2-2.02.186-7.el7'] self.assertRaises(VError, PkgV().validate, 'rpms', neg_pkg)
def test_remote_rpm_installed(self): """Check if rpm pkg installed.""" PkgV().validate('rpms', self.pkg, self.host)
def test_remote_pip3_installed(self): """Check if pip3 pkg installed.""" pkg = ["toml"] PkgV().validate('pip3s', pkg, self.host)