Пример #1
0
def parse_server_config(log, config, config_fpath, host_dict):
    """
    Parse server config.
    """
    server_config = utils.config_value(config, barrele_constant.BRL_SERVER)
    if server_config is None:
        log.cl_error(
            "can NOT find [%s] in the config file, "
            "please correct file [%s]", barrele_constant.BRL_SERVER,
            config_fpath)
        return None

    hostname = utils.config_value(server_config, barrele_constant.BRL_HOSTNAME)
    if hostname is None:
        log.cl_error(
            "can NOT find [%s] in the config of server, "
            "please correct file [%s]", barrele_constant.BRL_HOSTNAME,
            config_fpath)
        return None

    data_path = utils.config_value(server_config,
                                   barrele_constant.BRL_DATA_PATH)
    if data_path is None:
        log.cl_debug("no [%s] configured, using default value [%s]",
                     barrele_constant.BRL_DATA_PATH, BARRELE_DATA_DIR)
        data_path = BARRELE_DATA_DIR

    ssh_identity_file = utils.config_value(
        server_config, barrele_constant.BRL_SSH_IDENTITY_FILE)

    host = ssh_host.get_or_add_host_to_dict(log, host_dict, hostname,
                                            ssh_identity_file)
    if host is None:
        return None
    return barrele_server.BarreleServer(host, data_path)
Пример #2
0
def read_release_info_file(log, release_info_fpath):
    """
    Get Coral version from release info file. Return the release name and
    release date.
    """
    yaml_content = lyaml.read_yaml_file(log, release_info_fpath)
    if yaml_content is None:
        return None, None

    release_name = utils.config_value(yaml_content,
                                      constant.CORAL_STR_RELEASE_NAME)
    if release_name is None:
        log.cl_error("can NOT find [%s] in version file [%s]",
                     constant.CORAL_STR_RELEASE_NAME, release_info_fpath)
        return None, None

    release_date_str = utils.config_value(yaml_content,
                                          constant.CORAL_STR_RELEASE_DATE)
    if release_date_str is None:
        log.cl_error("can NOT find [%s] in version file [%s]",
                     constant.CORAL_STR_RELEASE_DATE, release_info_fpath)
        return None, None

    try:
        release_date = int(release_date_str)
    except:
        log.cl_error("invalid release date [%s] in version file [%s]",
                     release_date_str, release_info_fpath)
        return None, None

    return release_name, release_date
Пример #3
0
def get_version_from_version_file(log, version_file):
    """
    Get Coral version from version file in ISO dir. Return CoralVersionInfo.
    """
    yaml_content = lyaml.read_yaml_file(log, version_file)
    if yaml_content is None:
        return None

    release_name = utils.config_value(yaml_content,
                                      constant.CORAL_STR_RELEASE_NAME)
    if release_name is None:
        log.cl_error("can NOT find [%s] in version file [%s]",
                     constant.CORAL_STR_RELEASE_NAME, version_file)
        return None

    target_cpu = utils.config_value(yaml_content,
                                    constant.CORAL_STR_TARGET_CPU)
    if target_cpu is None:
        log.cl_error("can NOT find [%s] in version file [%s]",
                     constant.CORAL_STR_TARGET_CPU, version_file)
        return None

    distro_short = utils.config_value(yaml_content,
                                      constant.CORAL_STR_DISTRO_SHORT)
    if distro_short is None:
        log.cl_error("can NOT find [%s] in version file [%s]",
                     constant.CORAL_STR_DISTRO_SHORT, version_file)
        return None

    release_date_str = utils.config_value(yaml_content,
                                          constant.CORAL_STR_RELEASE_DATE)
    if release_date_str is None:
        log.cl_error("can NOT find [%s] in version file [%s]",
                     constant.CORAL_STR_RELEASE_DATE, version_file)
        return None

    try:
        release_date = int(release_date_str)
    except:
        log.cl_error("invalid release date [%s] in version file [%s]",
                     release_date_str, version_file)
        return None

    coral_version_info = CoralVersionInfo(release_name, distro_short,
                                          target_cpu, release_date)

    return coral_version_info
Пример #4
0
def barrele_init_instance(log, workspace, config, config_fpath, log_to_file,
                          logdir_is_default):
    """
    Parse the config and init the instance
    """
    # pylint: disable=too-many-locals,too-many-branches,too-many-statements
    collect_interval = utils.config_value(
        config, barrele_constant.BRL_COLLECT_INTERVAL)
    if collect_interval is None:
        log.cl_debug(
            "no [%s] is configured in the config file [%s], "
            "using default value [%s]", barrele_constant.BRL_COLLECT_INTERVAL,
            config_fpath, BARRELE_COLLECT_INTERVAL)
        collect_interval = BARRELE_COLLECT_INTERVAL

    continuous_query_periods = utils.config_value(
        config, barrele_constant.BRL_CONTINUOUS_QUERY_PERIODS)
    if continuous_query_periods is None:
        log.cl_debug(
            "no [%s] is configured in the config file [%s], "
            "using default value [%s]",
            barrele_constant.BRL_CONTINUOUS_QUERY_PERIODS, config_fpath,
            BARRELE_CONTINUOUS_QUERY_PERIODS)
        continuous_query_periods = BARRELE_CONTINUOUS_QUERY_PERIODS

    jobstat_pattern = utils.config_value(config,
                                         barrele_constant.BRL_JOBSTAT_PATTERN)
    if jobstat_pattern is None:
        log.cl_debug(
            "no [%s] is configured in the config file [%s], "
            "using default value [%s]", barrele_constant.BRL_JOBSTAT_PATTERN,
            config_fpath, barrele_constant.BARRELE_JOBSTAT_PATTERN_UNKNOWN)
        jobstat_pattern = barrele_constant.BARRELE_JOBSTAT_PATTERN_UNKNOWN
    if jobstat_pattern not in barrele_constant.BARRELE_JOBSTAT_PATTERNS:
        log.cl_error("unsupported jobstat_pattern [%s], supported: %s",
                     jobstat_pattern,
                     barrele_constant.BARRELE_JOBSTAT_PATTERNS)
        return None

    lustre_fallback_version_name = \
        utils.config_value(config,
                           barrele_constant.BRL_LUSTRE_FALLBACK_VERSION)
    if lustre_fallback_version_name is None:
        log.cl_debug(
            "no [%s] is configured in the config file [%s], "
            "using default value [%s]",
            barrele_constant.BRL_LUSTRE_FALLBACK_VERSION, config_fpath,
            BARRELE_LUSTRE_FALLBACK_VERSION)
        lustre_fallback_version_name = BARRELE_LUSTRE_FALLBACK_VERSION

    if lustre_fallback_version_name not in lustre_version.LUSTRE_VERSION_DICT:
        log.cl_error(
            "unsupported Lustre version [%s] is configured in the "
            "config file [%s]", lustre_fallback_version_name, config_fpath)
        return None

    lustre_fallback_version = \
        lustre_version.LUSTRE_VERSION_DICT[lustre_fallback_version_name]

    enable_lustre_exp_mdt = utils.config_value(
        config, barrele_constant.BRL_ENABLE_LUSTRE_EXP_MDT)
    if enable_lustre_exp_mdt is None:
        log.cl_debug(
            "no [%s] is configured in the config file [%s], "
            "using default value [False]",
            barrele_constant.BRL_ENABLE_LUSTRE_EXP_MDT, config_fpath)
        enable_lustre_exp_mdt = False

    enable_lustre_exp_ost = utils.config_value(
        config, barrele_constant.BRL_ENABLE_LUSTRE_EXP_OST)
    if enable_lustre_exp_ost is None:
        log.cl_debug(
            "no [%s] is configured in the config file [%s], "
            "using default value [False]",
            barrele_constant.BRL_ENABLE_LUSTRE_EXP_OST, config_fpath)
        enable_lustre_exp_ost = False

    agent_configs = utils.config_value(config, barrele_constant.BRL_AGENTS)
    if agent_configs is None:
        log.cl_error(
            "can NOT find [%s] in the config file, "
            "please correct file [%s]", barrele_constant.BRL_AGENTS,
            config_fpath)
        return None

    host_dict = {}
    barreleye_server = parse_server_config(log, config, config_fpath,
                                           host_dict)
    if barreleye_server is None:
        log.cl_error("failed to parse server config")
        return None

    agent_dict = {}
    for agent_config in agent_configs:
        hostname_config = utils.config_value(agent_config,
                                             barrele_constant.BRL_HOSTNAME)
        if hostname_config is None:
            log.cl_error(
                "can NOT find [%s] in the config of SSH host "
                "[%s], please correct file [%s]",
                barrele_constant.BRL_HOSTNAME, hostname_config, config_fpath)
            return None

        hostnames = cmd_general.parse_list_string(log, hostname_config)
        if hostnames is None:
            log.cl_error("[%s] as [%s] is invalid in the config file [%s]",
                         hostname_config, barrele_constant.BRL_HOSTNAME,
                         config_fpath)
            return None

        ssh_identity_file = utils.config_value(
            agent_config, barrele_constant.BRL_SSH_IDENTITY_FILE)

        enable_disk = utils.config_value(agent_config,
                                         barrele_constant.BRL_ENABLE_DISK)
        if enable_disk is None:
            log.cl_debug(
                "no [%s] is configured in the config file [%s], "
                "using default value [False]",
                barrele_constant.BRL_ENABLE_DISK, config_fpath)
            enable_disk = False

        enable_infiniband = utils.config_value(
            agent_config, barrele_constant.BRL_ENABLE_INFINIBAND)
        if enable_infiniband is None:
            log.cl_debug(
                "no [%s] is configured in the config file [%s], "
                "using default value [False]",
                barrele_constant.BRL_ENABLE_INFINIBAND, config_fpath)
            enable_infiniband = False

        enable_lustre_client = utils.config_value(
            agent_config, barrele_constant.BRL_ENABLE_LUSTRE_CLIENT)
        if enable_lustre_client is None:
            log.cl_debug(
                "no [%s] is configured in the config file [%s], "
                "using default value [False]",
                barrele_constant.BRL_ENABLE_LUSTRE_CLIENT, config_fpath)
            enable_lustre_client = False

        enable_lustre_mds = utils.config_value(
            agent_config, barrele_constant.BRL_ENABLE_LUSTRE_MDS)
        if enable_lustre_mds is None:
            log.cl_debug(
                "no [%s] is configured in the config file [%s], "
                "using default value [True]",
                barrele_constant.BRL_ENABLE_LUSTRE_MDS, config_fpath)
            enable_lustre_mds = True

        enable_lustre_oss = utils.config_value(
            agent_config, barrele_constant.BRL_ENABLE_LUSTRE_OSS)
        if enable_lustre_oss is None:
            log.cl_debug(
                "no [%s] is configured in the config file [%s], "
                "using default value [True]",
                barrele_constant.BRL_ENABLE_LUSTRE_OSS, config_fpath)
            enable_lustre_oss = True

        for hostname in hostnames:
            if hostname in agent_dict:
                log.cl_error(
                    "agent of host [%s] is configured for multiple times",
                    hostname)
                return None
            host = ssh_host.get_or_add_host_to_dict(log, host_dict, hostname,
                                                    ssh_identity_file)
            if host is None:
                return None

            agent = barrele_agent.BarreleAgent(
                host,
                barreleye_server,
                enable_disk=enable_disk,
                enable_lustre_oss=enable_lustre_oss,
                enable_lustre_mds=enable_lustre_mds,
                enable_lustre_client=enable_lustre_client,
                enable_infiniband=enable_infiniband)
            agent_dict[hostname] = agent

    local_host = ssh_host.get_local_host()
    instance = BarreleInstance(workspace, config, config_fpath, log_to_file,
                               logdir_is_default, local_host, collect_interval,
                               continuous_query_periods, jobstat_pattern,
                               lustre_fallback_version, enable_lustre_exp_mdt,
                               enable_lustre_exp_ost, host_dict, agent_dict,
                               barreleye_server)
    return instance