def resource_catalog_from_config(config, logger=utilities.NullLogger, default_allowed_vos=None):
    """
    Create a ResourceCatalog from the subcluster entries in a config
    :type config: ConfigParser.ConfigParser
    :type logger: logging.Logger or None
    :rtype: ResourceCatalog
    """
    assert isinstance(config, ConfigParser.ConfigParser)
    from osg_configure.modules import resourcecatalog

    rc = resourcecatalog.ResourceCatalog()

    subclusters_without_max_wall_time = []
    for section in config.sections():
        prefix = None
        if section.lower().startswith('subcluster'):
            prefix = 'subcluster'
        elif section.lower().startswith('resource entry'):
            prefix = 'resource entry'
        else:
            continue

        check_section(config, section)

        rcentry = resourcecatalog.RCEntry()

        subcluster = section[len(prefix):].lstrip()

        rcentry.name = config.get(section, 'name')
        rcentry.cpus = config.getint(section, 'cores_per_node')
        rcentry.memory = config.getint(section, 'ram_mb')
        rcentry.allowed_vos = utilities.config_safe_get(config, section, 'allowed_vos',
                                                        default=default_allowed_vos)
        max_wall_time = utilities.config_safe_get(config, section, 'max_wall_time')
        if not max_wall_time:
            rcentry.max_wall_time = 1440
            subclusters_without_max_wall_time.append(subcluster)
        else:
            rcentry.max_wall_time = max_wall_time.strip()
        rcentry.queue = utilities.config_safe_get(config, section, 'queue')

        # The ability to specify extra requirements is disabled until admins demand it
        # rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
        rcentry.extra_requirements = None
        rcentry.extra_transforms = utilities.config_safe_get(config, section, 'extra_transforms')

        rc.add_rcentry(rcentry)
    # end for section in config.sections()

    if subclusters_without_max_wall_time:
        logger.warning("No max_wall_time specified for some subclusters; defaulting to 1440."
                       "\nAdd 'max_wall_time=1440' to the following subcluster(s) to clear this warning:"
                       "\n%s" % ", ".join(subclusters_without_max_wall_time))

    return rc
Example #2
0
def resource_catalog_from_config(config,
                                 logger=utilities.NullLogger,
                                 default_allowed_vos=None):
    """
    Create a ResourceCatalog from the subcluster entries in a config
    :type config: ConfigParser.ConfigParser
    :type logger: logging.Logger or None
    :rtype: ResourceCatalog
    """
    assert isinstance(config, ConfigParser.ConfigParser)
    from osg_configure.modules import resourcecatalog

    rc = resourcecatalog.ResourceCatalog()

    subclusters_without_max_wall_time = []
    for section in config.sections():
        if section.lower().startswith('subcluster'):
            check_section(config, section)

            rcentry = resourcecatalog.RCEntry()

            subcluster = section[len('subcluster'):].lstrip()

            rcentry.name = config.get(section, 'name')
            rcentry.cpus = config.getint(section, 'cores_per_node')
            rcentry.memory = config.getint(section, 'ram_mb')
            rcentry.allowed_vos = utilities.config_safe_get(
                config, section, 'allowed_vos', default=default_allowed_vos)
            max_wall_time = utilities.config_safe_get(config, section,
                                                      'max_wall_time')
            if not max_wall_time:
                rcentry.max_wall_time = 1440
                subclusters_without_max_wall_time.append(subcluster)
            else:
                rcentry.max_wall_time = max_wall_time.strip()
            rcentry.queue = utilities.config_safe_get(config, section, 'queue')

            # The ability to specify extra requirements is disabled until admins demand it
            # rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
            rcentry.extra_requirements = None
            rcentry.extra_transforms = utilities.config_safe_get(
                config, section, 'extra_transforms')

            rc.add_rcentry(rcentry)
    # end for section in config.sections()

    if subclusters_without_max_wall_time:
        logger.warning(
            "No max_wall_time specified for some subclusters; defaulting to 1440."
            "\nAdd 'max_wall_time=1440' to the following subcluster(s) to clear this warning:"
            "\n%s" % ", ".join(subclusters_without_max_wall_time))

    return rc
 def _set_default_servers(self, configuration):
     group = utilities.config_safe_get(configuration, 'Site Information', 'group')
     for key in ['bdii_servers', 'ce_collectors']:
         if group == 'OSG-ITB':
             self.options[key].default_value = self._itb_defaults[key]
         else:
             self.options[key].default_value = self._production_defaults[key]
Example #4
0
 def _set_default_servers(self, configuration):
     group = utilities.config_safe_get(configuration, 'Site Information',
                                       'group')
     if group == 'OSG-ITB':
         self.options[
             'ce_collectors'].default_value = self._itb_default_ce_collectors
     else:
         self.options[
             'ce_collectors'].default_value = self._production_default_ce_collectors
Example #5
0
def check_section(config, section):
    """
    Check attributes related to a subcluster and make sure that they are consistent
    """
    if "changeme" in section.lower():
        msg = "You have a section named '%s', you must change this name.\n" % section
        raise exceptions.SettingError(msg)

    for option, value in ENTRIES.items():
        status, kind = value
        entry = check_entry(config, section, option, status, kind)
        if option in BANNED_ENTRIES and entry == BANNED_ENTRIES[option]:
            raise exceptions.SettingError("Value for %s in section %s is " \
                                          "a default or banned entry (%s); " \
                                          "you must change this value." % \
                                          (option, section, BANNED_ENTRIES[option]))
        if entry is None:
            continue

        try:
            range_min, range_max = ENTRY_RANGES[option]

            if not (range_min <= entry <= range_max):
                msg = (
                    "Value for %(option)s in section %(section)s is outside allowed range"
                    ", %(range_min)d-%(range_max)d" % locals())
                raise exceptions.SettingError(msg)
        except KeyError:
            pass

    # Special case: Pilot sections either need "os" specified or "require_singularity=true"
    if is_pilot(section):
        require_singularity = utilities.config_safe_getboolean(
            config, section, "require_singularity", True)
        os = utilities.config_safe_get(config, section, "os", None)

        if not require_singularity and not os:
            msg = "'os' must be specified in section %s if 'require_singularity' is false" % section
            raise exceptions.SettingError(msg)
Example #6
0
def resource_catalog_from_config(config, default_allowed_vos=None):
    """
    Create a ResourceCatalog from the subcluster entries in a config
    :type config: ConfigParser.ConfigParser
    :rtype: ResourceCatalog
    """
    logger = logging.getLogger(__name__)
    assert isinstance(config, ConfigParser.ConfigParser)
    from osg_configure.modules import resourcecatalog

    rc = resourcecatalog.ResourceCatalog()

    # list of section names of all subcluster sections
    subcluster_sections = [section for section in config.sections() if section.lower().startswith('subcluster')]
    subcluster_names = [config.get(section, 'name').strip() for section in subcluster_sections]

    sections_without_max_wall_time = []
    for section in config.sections():
        lsection = section.lower()
        if not (lsection.startswith('subcluster') or lsection.startswith('resource entry')):
            continue

        check_section(config, section)

        rcentry = resourcecatalog.RCEntry()
        rcentry.name = config.get(section, 'name')

        rcentry.cpus = utilities.config_safe_get(config, section, 'cpucount') or \
                       utilities.config_safe_get(config, section, 'cores_per_node')
        if not rcentry.cpus:
            raise exceptions.SettingError("cpucount / cores_per_node not found in section %s" % section)
        rcentry.cpus = int(rcentry.cpus)

        rcentry.memory = utilities.config_safe_get(config, section, 'maxmemory') or \
                         utilities.config_safe_get(config, section, 'ram_mb')
        if not rcentry.memory:
            raise exceptions.SettingError("maxmemory / ram_mb not found in section %s" % section)
        rcentry.memory = int(rcentry.memory)

        rcentry.allowed_vos = utilities.config_safe_get(config, section, 'allowed_vos', default="").strip()
        if not rcentry.allowed_vos:
            logger.error("No allowed_vos specified for section '%s'."
                         "\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos"
                         "\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on"
                         "\nthe user accounts available on your CE." % section)
            raise exceptions.SettingError("No allowed_vos for %s" % section)
        if rcentry.allowed_vos == "*":
            if default_allowed_vos:
                rcentry.allowed_vos = default_allowed_vos
            else:
                rcentry.allowed_vos = None

        max_wall_time = utilities.config_safe_get(config, section, 'max_wall_time')
        if not max_wall_time:
            rcentry.max_wall_time = 1440
            sections_without_max_wall_time.append(section)
        else:
            rcentry.max_wall_time = max_wall_time.strip()
        rcentry.queue = utilities.config_safe_get(config, section, 'queue')

        scs = utilities.config_safe_get(config, section, 'subclusters')
        if scs:
            scs = re.split(r'\s*,\s*', scs)
            for sc in scs:
                if sc not in subcluster_names:
                    raise exceptions.SettingError("Undefined subcluster '%s' mentioned in section '%s'" % (sc, section))
        rcentry.subclusters = scs

        rcentry.vo_tag = utilities.config_safe_get(config, section, 'vo_tag')

        # The ability to specify extra requirements is disabled until admins demand it
        # rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
        rcentry.extra_requirements = None
        rcentry.extra_transforms = utilities.config_safe_get(config, section, 'extra_transforms')

        rc.add_rcentry(rcentry)
    # end for section in config.sections()

    if sections_without_max_wall_time:
        logger.warning("No max_wall_time specified for some sections; defaulting to 1440."
                       "\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:"
                       "\n'%s'" % "', '".join(sections_without_max_wall_time))

    return rc
Example #7
0
    def configure(self, attributes):
        """Configure installation using attributes"""
        self.log("InfoServicesConfiguration.configure started")

        if self.ignored:
            self.log("%s configuration ignored" % self.config_section,
                     level=logging.WARNING)
            self.log('InfoServicesConfiguration.configure completed')
            return True

        if not self.enabled:
            self.log("Not enabled")
            self.log("InfoServicesConfiguration.configure completed")
            return True

        if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled:
            if classad is None:
                self.log(
                    "Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings."
                    "\nEnsure the 'classad' Python module is installed and accessible to Python scripts."
                    "\nIf using HTCondor from RPMs, install the 'condor-python' RPM."
                    "\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH."
                    "\nHTCondor version must be at least 8.2.0.",
                    level=logging.WARNING)
            else:
                if self.authorization_method == 'vomsmap':
                    error = False
                    for requiredfile in [BAN_MAPFILE, BAN_VOMS_MAPFILE]:
                        if not os.path.exists(requiredfile):
                            self.log(
                                "%s authorization requested but %s was not found."
                                "\nThis will cause all mappings to fail."
                                "\nPlease reinstall lcmaps >= 1.6.6-1.3 or create a blank %s yourself."
                                % (self.authorization_method, requiredfile,
                                   requiredfile),
                                level=logging.ERROR)
                            error = True
                    if error:
                        return False

                    default_allowed_vos = reversevomap.get_allowed_vos()
                else:
                    using_gums = self.authorization_method == 'xacml'
                    # HACK for SOFTWARE-2792
                    if using_gums:
                        self.misc_module.update_gums_client_location()
                    self._ensure_valid_user_vo_file()
                    default_allowed_vos = utilities.get_vos(
                        USER_VO_MAP_LOCATION)
                if not default_allowed_vos:
                    # UGLY: only issue the warning if the admin has requested autodetection for some of their SCs/REs
                    raise_warning = False
                    for section in self.subcluster_sections.sections():
                        if utilities.config_safe_get(self.subcluster_sections,
                                                     section, 'allowed_vos',
                                                     '').strip() == "*":
                            raise_warning = True
                    if raise_warning:
                        self.log(
                            "Could not determine default allowed VOs for subclusters/resource entries.",
                            level=logging.WARNING)
                        if self.authorization_method == 'vomsmap':
                            self.log(
                                "Install vo-client-lcmaps-voms to obtain default mappings for VOs, and/or create"
                                " your own mapfile at /etc/grid-security/voms-mapfile.",
                                level=logging.WARNING)
                        else:
                            self.log(
                                "Ensure %s exists and is non-empty, or fill out allowed_vos in all your"
                                " Subcluster and Resource Entry sections." %
                                USER_VO_MAP_LOCATION,
                                level=logging.WARNING)
                try:
                    self.resource_catalog = subcluster.resource_catalog_from_config(
                        self.subcluster_sections,
                        default_allowed_vos=default_allowed_vos)
                except exceptions.SettingError as err:
                    self.log("Error in info services configuration: %s" %
                             str(err),
                             level=logging.ERROR)
                    return False
                self._configure_ce_collector()

        self.log("InfoServicesConfiguration.configure completed")
        return True
Example #8
0
 def csg(section, option):
     return utilities.config_safe_get(configuration, section, option,
                                      None)
Example #9
0
 def safeget(option: str, default=None) -> str:
     return utilities.config_safe_get(config, section, option, default)
 def csg(section, option):
     return utilities.config_safe_get(configuration, section, option, None)