def parse_server_config(log, config, config_fpath, host_dict): """ Parse server config. """ server_config = utils.config_value(config, barrele_constant.BRL_SERVER) if server_config is None: log.cl_error( "can NOT find [%s] in the config file, " "please correct file [%s]", barrele_constant.BRL_SERVER, config_fpath) return None hostname = utils.config_value(server_config, barrele_constant.BRL_HOSTNAME) if hostname is None: log.cl_error( "can NOT find [%s] in the config of server, " "please correct file [%s]", barrele_constant.BRL_HOSTNAME, config_fpath) return None data_path = utils.config_value(server_config, barrele_constant.BRL_DATA_PATH) if data_path is None: log.cl_debug("no [%s] configured, using default value [%s]", barrele_constant.BRL_DATA_PATH, BARRELE_DATA_DIR) data_path = BARRELE_DATA_DIR ssh_identity_file = utils.config_value( server_config, barrele_constant.BRL_SSH_IDENTITY_FILE) host = ssh_host.get_or_add_host_to_dict(log, host_dict, hostname, ssh_identity_file) if host is None: return None return barrele_server.BarreleServer(host, data_path)
def read_release_info_file(log, release_info_fpath): """ Get Coral version from release info file. Return the release name and release date. """ yaml_content = lyaml.read_yaml_file(log, release_info_fpath) if yaml_content is None: return None, None release_name = utils.config_value(yaml_content, constant.CORAL_STR_RELEASE_NAME) if release_name is None: log.cl_error("can NOT find [%s] in version file [%s]", constant.CORAL_STR_RELEASE_NAME, release_info_fpath) return None, None release_date_str = utils.config_value(yaml_content, constant.CORAL_STR_RELEASE_DATE) if release_date_str is None: log.cl_error("can NOT find [%s] in version file [%s]", constant.CORAL_STR_RELEASE_DATE, release_info_fpath) return None, None try: release_date = int(release_date_str) except: log.cl_error("invalid release date [%s] in version file [%s]", release_date_str, release_info_fpath) return None, None return release_name, release_date
def get_version_from_version_file(log, version_file): """ Get Coral version from version file in ISO dir. Return CoralVersionInfo. """ yaml_content = lyaml.read_yaml_file(log, version_file) if yaml_content is None: return None release_name = utils.config_value(yaml_content, constant.CORAL_STR_RELEASE_NAME) if release_name is None: log.cl_error("can NOT find [%s] in version file [%s]", constant.CORAL_STR_RELEASE_NAME, version_file) return None target_cpu = utils.config_value(yaml_content, constant.CORAL_STR_TARGET_CPU) if target_cpu is None: log.cl_error("can NOT find [%s] in version file [%s]", constant.CORAL_STR_TARGET_CPU, version_file) return None distro_short = utils.config_value(yaml_content, constant.CORAL_STR_DISTRO_SHORT) if distro_short is None: log.cl_error("can NOT find [%s] in version file [%s]", constant.CORAL_STR_DISTRO_SHORT, version_file) return None release_date_str = utils.config_value(yaml_content, constant.CORAL_STR_RELEASE_DATE) if release_date_str is None: log.cl_error("can NOT find [%s] in version file [%s]", constant.CORAL_STR_RELEASE_DATE, version_file) return None try: release_date = int(release_date_str) except: log.cl_error("invalid release date [%s] in version file [%s]", release_date_str, version_file) return None coral_version_info = CoralVersionInfo(release_name, distro_short, target_cpu, release_date) return coral_version_info
def barrele_init_instance(log, workspace, config, config_fpath, log_to_file, logdir_is_default): """ Parse the config and init the instance """ # pylint: disable=too-many-locals,too-many-branches,too-many-statements collect_interval = utils.config_value( config, barrele_constant.BRL_COLLECT_INTERVAL) if collect_interval is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [%s]", barrele_constant.BRL_COLLECT_INTERVAL, config_fpath, BARRELE_COLLECT_INTERVAL) collect_interval = BARRELE_COLLECT_INTERVAL continuous_query_periods = utils.config_value( config, barrele_constant.BRL_CONTINUOUS_QUERY_PERIODS) if continuous_query_periods is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [%s]", barrele_constant.BRL_CONTINUOUS_QUERY_PERIODS, config_fpath, BARRELE_CONTINUOUS_QUERY_PERIODS) continuous_query_periods = BARRELE_CONTINUOUS_QUERY_PERIODS jobstat_pattern = utils.config_value(config, barrele_constant.BRL_JOBSTAT_PATTERN) if jobstat_pattern is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [%s]", barrele_constant.BRL_JOBSTAT_PATTERN, config_fpath, barrele_constant.BARRELE_JOBSTAT_PATTERN_UNKNOWN) jobstat_pattern = barrele_constant.BARRELE_JOBSTAT_PATTERN_UNKNOWN if jobstat_pattern not in barrele_constant.BARRELE_JOBSTAT_PATTERNS: log.cl_error("unsupported jobstat_pattern [%s], supported: %s", jobstat_pattern, barrele_constant.BARRELE_JOBSTAT_PATTERNS) return None lustre_fallback_version_name = \ utils.config_value(config, barrele_constant.BRL_LUSTRE_FALLBACK_VERSION) if lustre_fallback_version_name is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [%s]", barrele_constant.BRL_LUSTRE_FALLBACK_VERSION, config_fpath, BARRELE_LUSTRE_FALLBACK_VERSION) lustre_fallback_version_name = BARRELE_LUSTRE_FALLBACK_VERSION if lustre_fallback_version_name not in lustre_version.LUSTRE_VERSION_DICT: log.cl_error( "unsupported Lustre version [%s] is configured in the " "config file [%s]", lustre_fallback_version_name, config_fpath) return None lustre_fallback_version = \ lustre_version.LUSTRE_VERSION_DICT[lustre_fallback_version_name] enable_lustre_exp_mdt = utils.config_value( config, barrele_constant.BRL_ENABLE_LUSTRE_EXP_MDT) if enable_lustre_exp_mdt is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [False]", barrele_constant.BRL_ENABLE_LUSTRE_EXP_MDT, config_fpath) enable_lustre_exp_mdt = False enable_lustre_exp_ost = utils.config_value( config, barrele_constant.BRL_ENABLE_LUSTRE_EXP_OST) if enable_lustre_exp_ost is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [False]", barrele_constant.BRL_ENABLE_LUSTRE_EXP_OST, config_fpath) enable_lustre_exp_ost = False agent_configs = utils.config_value(config, barrele_constant.BRL_AGENTS) if agent_configs is None: log.cl_error( "can NOT find [%s] in the config file, " "please correct file [%s]", barrele_constant.BRL_AGENTS, config_fpath) return None host_dict = {} barreleye_server = parse_server_config(log, config, config_fpath, host_dict) if barreleye_server is None: log.cl_error("failed to parse server config") return None agent_dict = {} for agent_config in agent_configs: hostname_config = utils.config_value(agent_config, barrele_constant.BRL_HOSTNAME) if hostname_config is None: log.cl_error( "can NOT find [%s] in the config of SSH host " "[%s], please correct file [%s]", barrele_constant.BRL_HOSTNAME, hostname_config, config_fpath) return None hostnames = cmd_general.parse_list_string(log, hostname_config) if hostnames is None: log.cl_error("[%s] as [%s] is invalid in the config file [%s]", hostname_config, barrele_constant.BRL_HOSTNAME, config_fpath) return None ssh_identity_file = utils.config_value( agent_config, barrele_constant.BRL_SSH_IDENTITY_FILE) enable_disk = utils.config_value(agent_config, barrele_constant.BRL_ENABLE_DISK) if enable_disk is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [False]", barrele_constant.BRL_ENABLE_DISK, config_fpath) enable_disk = False enable_infiniband = utils.config_value( agent_config, barrele_constant.BRL_ENABLE_INFINIBAND) if enable_infiniband is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [False]", barrele_constant.BRL_ENABLE_INFINIBAND, config_fpath) enable_infiniband = False enable_lustre_client = utils.config_value( agent_config, barrele_constant.BRL_ENABLE_LUSTRE_CLIENT) if enable_lustre_client is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [False]", barrele_constant.BRL_ENABLE_LUSTRE_CLIENT, config_fpath) enable_lustre_client = False enable_lustre_mds = utils.config_value( agent_config, barrele_constant.BRL_ENABLE_LUSTRE_MDS) if enable_lustre_mds is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [True]", barrele_constant.BRL_ENABLE_LUSTRE_MDS, config_fpath) enable_lustre_mds = True enable_lustre_oss = utils.config_value( agent_config, barrele_constant.BRL_ENABLE_LUSTRE_OSS) if enable_lustre_oss is None: log.cl_debug( "no [%s] is configured in the config file [%s], " "using default value [True]", barrele_constant.BRL_ENABLE_LUSTRE_OSS, config_fpath) enable_lustre_oss = True for hostname in hostnames: if hostname in agent_dict: log.cl_error( "agent of host [%s] is configured for multiple times", hostname) return None host = ssh_host.get_or_add_host_to_dict(log, host_dict, hostname, ssh_identity_file) if host is None: return None agent = barrele_agent.BarreleAgent( host, barreleye_server, enable_disk=enable_disk, enable_lustre_oss=enable_lustre_oss, enable_lustre_mds=enable_lustre_mds, enable_lustre_client=enable_lustre_client, enable_infiniband=enable_infiniband) agent_dict[hostname] = agent local_host = ssh_host.get_local_host() instance = BarreleInstance(workspace, config, config_fpath, log_to_file, logdir_is_default, local_host, collect_interval, continuous_query_periods, jobstat_pattern, lustre_fallback_version, enable_lustre_exp_mdt, enable_lustre_exp_ost, host_dict, agent_dict, barreleye_server) return instance