def check_entry(config, section, option, status, kind): """ Check entries to make sure that they conform to the correct range of values """ entry = None try: entry = str(config.get(section, option)).strip() except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, ConfigParser.InterpolationError): pass is_subcluster = section.lower().startswith('subcluster') if not entry: if (status == REQUIRED or (status == REQUIRED_FOR_SUBCLUSTER and is_subcluster) or (status == REQUIRED_FOR_RESOURCE_ENTRY and not is_subcluster)): raise exceptions.SettingError("Can't get value for mandatory setting %s in section %s." % \ (option, section)) else: return None if kind == STRING: # No parsing we can do for strings. return entry elif kind == POSITIVE_INT: try: entry = int(entry) if entry < 0: raise ValueError() except (TypeError, ValueError): raise exceptions.SettingError("Value of option `%s` in section " \ "`%s` should be a positive integer, but it is `%s`" % \ (option, section, entry)) return entry elif kind == POSITIVE_FLOAT: try: entry = float(entry) if entry < 0: raise ValueError() except (TypeError, ValueError): raise exceptions.SettingError("Value of option `%s` in section " \ "`%s` should be a positive float, but it is `%s`" % \ (option, section, entry)) return entry elif kind == BOOLEAN: entry = entry.lower() possible_vals = ['t', 'true', 'yes', 'y', 'enable', 'enabled', 'f', 'false', 'no', 'n', 'disable', 'disabled'] positive_vals = ['t', 'true', 'yes', 'y', 'enable', 'enabled'] if entry not in possible_vals: raise exceptions.SettingError("Value of option `%s` in section " \ "`%s` should be a boolean, but it is `%s`" % (option, section, entry)) return entry in positive_vals elif kind == LIST: regex = re.compile(r'\s*,*\s*') entry = regex.split(entry) return entry else: # Kind of entry isn't known... OK for now. return entry
def check_section(config, section): """ Check attributes related to a subcluster and make sure that they are consistent """ if section.lower().find('changeme') >= 0: msg = "You have a section named '%s', you must change this name.\n" % section raise exceptions.SettingError(msg) for option, value in ENTRIES.items(): status, kind = value entry = check_entry(config, section, option, status, kind) if option in BANNED_ENTRIES and entry == BANNED_ENTRIES[option]: raise exceptions.SettingError("Value for %s in section %s is " \ "a default or banned entry (%s); " \ "you must change this value." % \ (option, section, BANNED_ENTRIES[option])) if entry is None: continue try: range_min, range_max = ENTRY_RANGES[option] if not (range_min <= entry <= range_max): msg = ("Value for %(option)s in section %(section)s is outside allowed range" ", %(range_min)d-%(range_max)d" % locals()) if option == 'HEPSPEC': msg += '. The conversion factor from HEPSPEC to SI2K is 250' raise exceptions.SettingError(msg) except KeyError: pass
def _auto_configure(self, configuration): """ Method to configure info services without an info services section on a CE """ self.enabled = True if configuration.has_option('Site Information', 'group'): group = configuration.get('Site Information', 'group') else: self.log( 'No group defined in Site Information, this is required on a CE', level=logging.ERROR) raise exceptions.SettingError( 'In Site Information, group needs to be set') if group == 'OSG': bdii_servers = self._production_defaults['bdii_servers'] elif group == 'OSG-ITB': bdii_servers = self._itb_defaults['bdii_servers'] else: self.log('Group must be OSG or OSG-ITB', level=logging.ERROR) raise exceptions.SettingError( 'In Site Information, group needs to be OSG or OSG-ITB') self.options['bdii_servers'].value = bdii_servers self.bdii_servers = self._parse_servers(bdii_servers)
def _parse_configuration(self, configuration): """ The meat of parse_configuration, runs after we've checked that GIP is enabled and we have the right RPMs installed. """ # The following is to set the user that gip files need to belong to # This can be overridden by setting the 'user' option in the [GIP] section self.gip_user = '******' if configuration.has_option(self.config_section, 'batch'): batch_opt = configuration.get(self.config_section, 'batch').lower() if (not utilities.blank(batch_opt) and batch_opt not in self._valid_batch_opt): msg = "The batch setting in %s must be a valid option " \ "(e.g. %s), %s was given" % (self.config_section, ",".join(self._valid_batch_opt), batch_opt) self.log(msg, level=logging.ERROR) raise exceptions.SettingError(msg) if utilities.ce_installed(): self._parse_configuration_ce(configuration) # Check for the presence of the classic SE has_classic_se = True try: has_classic_se = configuration.getboolean("GIP", "advertise_gsiftp") # pylint: disable-msg=W0702 # pylint: disable-msg=W0703 # pylint: disable-msg=W0704 except Exception: pass has_se = False for section in configuration.sections(): if not section.lower().startswith('se'): continue has_se = True self.check_se(configuration, section) if not has_se and not has_classic_se: try: self._check_entry(configuration, "GIP", "se_name", REQUIRED, STRING) except: msg = "There is no `SE` section, the old-style SE" + \ "setup in GIP is not configured, and there is no classic SE. " + \ " At least one must be configured. Please see the configuration" \ " documentation." raise exceptions.SettingError(msg) if configuration.has_option(self.config_section, 'user'): username = configuration.get(self.config_section, 'user') if not validation.valid_user(username): err_msg = "%s is not a valid account on this system" % username self.log(err_msg, section=self.config_section, option='user', level=logging.ERROR) raise exceptions.SettingError(err_msg) self.gip_user = username
def get_option(config, section, option=None): """ Get an option from a config file with optional defaults and mandatory options. Arguments config -- a ConfigParser object to query section -- the ini section the option is located in option -- an Option object to information on the option to retrieve """ if option is None: raise exceptions.SettingError('No option passed to get_option') if config.has_option(section, option.name): try: if not utilities.blank(config.get(section, option.name)): if option.opt_type == bool: option.value = config.getboolean(section, option.name) elif option.opt_type == int: option.value = config.getint(section, option.name) elif option.opt_type == float: option.value = config.getfloat(section, option.name) else: option.value = config.get(section, option.name) else: # if option is blank and there's a default for the option # return the default if possible, otherwise raise an exception # if the option is mandatory if (option.required == Option.MANDATORY and option.default_value is None): raise exceptions.SettingError("Can't get value for %s in %s " \ "section and no default given" % \ (option.name, section)) option.value = option.default_value except ValueError: error_mesg = "%s in %s section is of the wrong type" % (option.name, section) raise exceptions.SettingError(error_mesg) elif option.required == Option.MANDATORY: err_mesg = "Can't get value for %s in %s section" % (option.name, section) raise exceptions.SettingError(err_mesg) else: option.value = option.default_value
def set_status(self, configuration): """ Check the enable option and then set the appropriate attributes based on that. Returns False if the section is not enabled or set to ignore """ try: if not configuration.has_option(self.config_section, 'enabled'): self.logger.debug("%s not enabled" % self.config_section) self.enabled = False return False elif configuration.get(self.config_section, 'enabled').lower() == 'ignore': self.logger.debug("%s will be ignored" % self.config_section) self.ignored = True self.enabled = False return False elif not configuration.getboolean(self.config_section, 'enabled'): self.logger.debug("%s not enabled" % self.config_section) self.enabled = False return False else: self.enabled = True return True except configparser.NoOptionError: raise exceptions.SettingError("Can't get value for enable option " "in %s section" % self.config_section)
def _auto_configure(self, configuration): """ Configure gratia for a ce which does not have the gratia section """ self.enabled = True if configuration.has_option('Site Information', 'resource'): resource = configuration.get('Site Information', 'resource') self.options['resource'].value = resource elif configuration.has_option('Site Information', 'site_name'): resource = configuration.get('Site Information', 'site_name') self.options['resource'].value = resource else: self.log( 'No site_name or resource defined in Site Information, this' ' is required on a CE', level=logging.ERROR) raise exceptions.SettingError( 'In Site Information, ' 'site_name or resource needs to be set') if configuration.has_option('Site Information', 'group'): group = configuration.get('Site Information', 'group') else: self.log( 'No group defined in Site Information, this is required on a CE', level=logging.ERROR) raise exceptions.SettingError( 'In Site Information, group needs to be set') if group == 'OSG': probes = self._production_defaults['probes'] elif group == 'OSG-ITB': probes = self._itb_defaults['probes'] else: raise exceptions.SettingError('In Site Information, group must be ' 'OSG or OSG-ITB') self.options['probes'].value = probes self._parse_probes(probes) return True
def _parse_configuration_ce(self, configuration): # All CEs must advertise subclusters has_sc = self.check_subclusters(configuration) if not has_sc: try: self._check_entry(configuration, "GIP", "sc_number", REQUIRED, POSITIVE_INT) except (TypeError, ValueError, exceptions.SettingError): msg = "There is no `subcluster` section and the old-style subcluster" + \ "setup in GIP is not configured. " + \ " Please see the configuration documentation." raise exceptions.SettingError(msg)
def check_section(config, section): """ Check attributes related to a subcluster and make sure that they are consistent """ if "changeme" in section.lower(): msg = "You have a section named '%s', you must change this name.\n" % section raise exceptions.SettingError(msg) for option, value in ENTRIES.items(): status, kind = value entry = check_entry(config, section, option, status, kind) if option in BANNED_ENTRIES and entry == BANNED_ENTRIES[option]: raise exceptions.SettingError("Value for %s in section %s is " \ "a default or banned entry (%s); " \ "you must change this value." % \ (option, section, BANNED_ENTRIES[option])) if entry is None: continue try: range_min, range_max = ENTRY_RANGES[option] if not (range_min <= entry <= range_max): msg = ( "Value for %(option)s in section %(section)s is outside allowed range" ", %(range_min)d-%(range_max)d" % locals()) raise exceptions.SettingError(msg) except KeyError: pass # Special case: Pilot sections either need "os" specified or "require_singularity=true" if is_pilot(section): require_singularity = utilities.config_safe_getboolean( config, section, "require_singularity", True) os = utilities.config_safe_get(config, section, "os", None) if not require_singularity and not os: msg = "'os' must be specified in section %s if 'require_singularity' is false" % section raise exceptions.SettingError(msg)
def get_options(self, configuration, **kwargs): """ Populate self.options based on contents of ConfigParser object, warns if unknown options are found arguments: configuration - a ConfigParser object keyword arguments: ignore_options - a list of option names that should be ignored when checking for unknown options """ self.check_config(configuration) for option in self.options.values(): self.log("Getting value for %s" % option.name) try: configfile.get_option(configuration, self.config_section, option) self.log("Got %s" % option.value) except configparser.Error as err: self.log("Syntax error in configuration: %s" % err, option=option.name, section=self.config_section, level=logging.ERROR, exception=False) raise exceptions.SettingError(str(err)) except Exception: self.log("Received exception when parsing option", option=option.name, section=self.config_section, level=logging.ERROR, exception=False) raise # check and warn if unknown options found known_options = list(self.options.keys()) known_options.extend(kwargs.get('ignore_options', [])) temp = utilities.get_set_membership( configuration.options(self.config_section), known_options, configuration.defaults().keys()) for option in temp: self.log("Found unknown option", option=option, section=self.config_section, level=logging.WARNING)
def section_disabled(configuration, section): """ Check the enable option for a specified section Returns False if the section is not enabled or set to ignore """ try: if not configuration.has_option(section, 'enabled'): return True elif configuration.get(section, 'enabled').lower() == 'ignore': return True elif not configuration.getboolean(section, 'enabled'): return True else: return False except configparser.NoOptionError: raise exceptions.SettingError("Can't get value for enable option " "in %s section" % section)
def _parse_servers(cls, servers): """ Take a list of servers and parse it into a list of (server, subscription_type) tuples """ server_list = {} if servers.lower() == 'ignore': # if the server list is set to ignore, then don't use any servers # this allows cemon to be send ress information but not bdii or vice versa return server_list server_regex = re.compile(r'(.*)\[(.*)\]') for entry in servers.split(','): match = server_regex.match(entry) if match is None: raise exceptions.SettingError('Invalid subscription: %s' % entry) server_list[match.group(1).strip()] = match.group(2) return server_list
def resource_catalog_from_config(config, default_allowed_vos=None): """ Create a ResourceCatalog from the subcluster entries in a config :type config: ConfigParser.ConfigParser :rtype: ResourceCatalog """ logger = logging.getLogger(__name__) assert isinstance(config, ConfigParser.ConfigParser) from osg_configure.modules import resourcecatalog rc = resourcecatalog.ResourceCatalog() # list of section names of all subcluster sections subcluster_sections = [section for section in config.sections() if section.lower().startswith('subcluster')] subcluster_names = [config.get(section, 'name').strip() for section in subcluster_sections] sections_without_max_wall_time = [] for section in config.sections(): lsection = section.lower() if not (lsection.startswith('subcluster') or lsection.startswith('resource entry')): continue check_section(config, section) rcentry = resourcecatalog.RCEntry() rcentry.name = config.get(section, 'name') rcentry.cpus = utilities.config_safe_get(config, section, 'cpucount') or \ utilities.config_safe_get(config, section, 'cores_per_node') if not rcentry.cpus: raise exceptions.SettingError("cpucount / cores_per_node not found in section %s" % section) rcentry.cpus = int(rcentry.cpus) rcentry.memory = utilities.config_safe_get(config, section, 'maxmemory') or \ utilities.config_safe_get(config, section, 'ram_mb') if not rcentry.memory: raise exceptions.SettingError("maxmemory / ram_mb not found in section %s" % section) rcentry.memory = int(rcentry.memory) rcentry.allowed_vos = utilities.config_safe_get(config, section, 'allowed_vos', default="").strip() if not rcentry.allowed_vos: logger.error("No allowed_vos specified for section '%s'." "\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos" "\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on" "\nthe user accounts available on your CE." % section) raise exceptions.SettingError("No allowed_vos for %s" % section) if rcentry.allowed_vos == "*": if default_allowed_vos: rcentry.allowed_vos = default_allowed_vos else: rcentry.allowed_vos = None max_wall_time = utilities.config_safe_get(config, section, 'max_wall_time') if not max_wall_time: rcentry.max_wall_time = 1440 sections_without_max_wall_time.append(section) else: rcentry.max_wall_time = max_wall_time.strip() rcentry.queue = utilities.config_safe_get(config, section, 'queue') scs = utilities.config_safe_get(config, section, 'subclusters') if scs: scs = re.split(r'\s*,\s*', scs) for sc in scs: if sc not in subcluster_names: raise exceptions.SettingError("Undefined subcluster '%s' mentioned in section '%s'" % (sc, section)) rcentry.subclusters = scs rcentry.vo_tag = utilities.config_safe_get(config, section, 'vo_tag') # The ability to specify extra requirements is disabled until admins demand it # rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements') rcentry.extra_requirements = None rcentry.extra_transforms = utilities.config_safe_get(config, section, 'extra_transforms') rc.add_rcentry(rcentry) # end for section in config.sections() if sections_without_max_wall_time: logger.warning("No max_wall_time specified for some sections; defaulting to 1440." "\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:" "\n'%s'" % "', '".join(sections_without_max_wall_time)) return rc
def parse_configuration(self, configuration): """ Try to get configuration information from ConfigParser or SafeConfigParser object given by configuration and write recognized settings to attributes dict """ self.log('InfoServicesConfiguration.parse_configuration started') self.check_config(configuration) if not configuration.has_section(self.config_section): self.enabled = False self.log("%s section not in config file" % self.config_section) self.log('InfoServicesConfiguration.parse_configuration completed') return if not self.set_status(configuration): self.log('InfoServicesConfiguration.parse_configuration completed') return True self._set_default_servers(configuration) self.get_options(configuration, ignore_options=[ 'itb-ress-servers', 'itb-bdii-servers', 'osg-ress-servers', 'osg-bdii-servers', 'ress_servers', 'enabled', 'bdii_servers' ]) self.ce_collectors = self._parse_ce_collectors( self.options['ce_collectors'].value) self.misc_module.parse_configuration(configuration) def csg(section, option): return utilities.config_safe_get(configuration, section, option, None) def csgbool(section, option): return utilities.config_safe_getboolean(configuration, section, option, False) # We get some values for HTCondor-CE from the Site Information section self.osg_resource = csg('Site Information', 'resource') self.osg_resource_group = csg('Site Information', 'resource_group') # and the enabled batch systems from their respective sections self.enabled_batch_systems = [ bs for bs in BATCH_SYSTEMS if csgbool(bs, 'enabled') ] self.htcondor_gateway_enabled = csgbool('Gateway', 'htcondor_gateway_enabled') self.authorization_method = csg('Misc Services', 'authorization_method') self.subcluster_sections = ConfigParser.SafeConfigParser() self.gums_host = csg('Misc Services', 'gums_host') for section in configuration.sections(): if section.lower().startswith('subcluster') or section.lower( ).startswith('resource entry'): self.subcluster_sections.add_section(section) for key, value in configuration.items(section): self.subcluster_sections.set(section, key, value) if utilities.ce_installed( ) and not subcluster.check_config(configuration): self.log( "On a CE but no valid 'Subcluster' or 'Resource Entry' sections defined." " This is required to advertise the capabilities of your cluster to the central collector." " Jobs may not be sent to this CE.", level=logging.ERROR) raise exceptions.SettingError( "No Subcluster or Resource Entry sections") # Check resource catalog # This is a bit clunky to parse it here and not use the result in # configure(), but at this point we don't have a way of knowing what # default_allowed_vos should be. if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled and classad is not None: subcluster.resource_catalog_from_config(self.subcluster_sections, default_allowed_vos=None) self.log('InfoServicesConfiguration.parse_configuration completed')
def resource_catalog_from_config( config: ConfigParser, default_allowed_vos: str = None) -> ResourceCatalog: """ Create a ResourceCatalog from the subcluster entries in a config :param default_allowed_vos: The allowed_vos to use if the user specified "*" """ logger = logging.getLogger(__name__) assert isinstance(config, ConfigParser) from osg_configure.modules.resourcecatalog import ResourceCatalog, RCEntry def safeget(option: str, default=None) -> str: return utilities.config_safe_get(config, section, option, default) def safegetbool(option: str, default=None) -> bool: return utilities.config_safe_getboolean(config, section, option, default) rc = ResourceCatalog() # list of section names of all subcluster sections subcluster_sections = [ section for section in config.sections() if is_subcluster(section) ] subcluster_names = [ rce_section_get_name(config, section) for section in subcluster_sections ] sections_without_max_wall_time = [] for section in config.sections(): if not (is_subcluster(section) or is_resource_entry(section) or is_pilot(section)): continue check_section(config, section) rcentry = RCEntry() rcentry.name = rce_section_get_name(config, section) rcentry.cpus = (safeget("cpucount") or safeget("cores_per_node") or CPUCOUNT_DEFAULT) rcentry.cpus = int(rcentry.cpus) rcentry.memory = (safeget("maxmemory") or safeget("ram_mb") or RAM_MB_DEFAULT) rcentry.memory = int(rcentry.memory) rcentry.allowed_vos = utilities.split_comma_separated_list( safeget("allowed_vos", default="").strip()) if not rcentry.allowed_vos or not rcentry.allowed_vos[0]: logger.error( "No allowed_vos specified for section '%s'." "\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos" "\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on" "\nthe user accounts available on your CE." % section) raise exceptions.SettingError("No allowed_vos for %s" % section) if rcentry.allowed_vos == ["*"]: if default_allowed_vos: rcentry.allowed_vos = default_allowed_vos else: rcentry.allowed_vos = [] max_wall_time = safeget("max_wall_time") if not max_wall_time: rcentry.max_wall_time = 1440 sections_without_max_wall_time.append(section) else: rcentry.max_wall_time = max_wall_time.strip() rcentry.queue = safeget("queue") scs = utilities.split_comma_separated_list(safeget("subclusters", "")) if scs: for sc in scs: if sc not in subcluster_names: raise exceptions.SettingError( "Undefined subcluster '%s' mentioned in section '%s'" % (sc, section)) rcentry.subclusters = scs else: rcentry.subclusters = None rcentry.vo_tag = safeget("vo_tag") # The ability to specify extra requirements is disabled until admins demand it # rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements') rcentry.extra_requirements = None rcentry.extra_transforms = safeget("extra_transforms") rcentry.gpus = safeget("gpucount") if is_pilot(section): rcentry.max_pilots = safeget("max_pilots") rcentry.whole_node = safegetbool("whole_node", False) if rcentry.whole_node: rcentry.cpus = None rcentry.memory = None rcentry.require_singularity = safegetbool("require_singularity", True) rcentry.os = safeget("os") rcentry.send_tests = safegetbool("send_tests", True) rcentry.is_pilot = True rc.add_rcentry(rcentry) # end for section in config.sections() if sections_without_max_wall_time: logger.warning( "No max_wall_time specified for some sections; defaulting to 1440." "\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:" "\n'%s'" % "', '".join(sections_without_max_wall_time)) return rc
def check_se(self, config, section): """ Check attributes currently stored and make sure that they are consistent """ self.log('GipConfiguration.check_se started') attributes_ok = True enabled = True try: if config.has_option(section, 'enabled'): enabled = config.getboolean(section, 'enabled') # pylint: disable-msg=W0703 except Exception: enabled = False if not enabled: # if section is disabled, we can exit return attributes_ok if section.lower().find('changeme') >= 0: msg = "You have a section named 'SE CHANGEME', but it is not turned off.\n" msg += "'SE CHANGEME' is an example; you must change it if it is enabled." raise exceptions.SettingError(msg) for option, value in SE_ENTRIES.items(): status, kind = value entry = self._check_entry(config, section, option, status, kind) if option in SE_BANNED_ENTRIES and entry == SE_BANNED_ENTRIES[ option]: raise exceptions.SettingError("Value for %s in section %s is " \ "a default or banned entry (%s); " \ "you must change this value." % \ (option, section, SE_BANNED_ENTRIES[option])) if entry is None: continue # Validate the mount point information if option == 'mount_point': regex = re.compile(r"/(/*[A-Za-z0-9_\-]/*)*$") err_info = {'input': value} if len(entry) != 2: err_info['reason'] = "Only one path was specified!" msg = MOUNT_POINT_ERROR % err_info raise exceptions.SettingError(msg) if not regex.match(entry[0]): err_info['reason'] = "First path does not pass validation" msg = MOUNT_POINT_ERROR % err_info raise exceptions.SettingError(msg) if not regex.match(entry[1]): err_info['reason'] = "Second path does not pass validation" msg = MOUNT_POINT_ERROR % err_info raise exceptions.SettingError(msg) if option == 'srm_endpoint': regex = re.compile( r'([A-Za-z]+)://([A-Za-z0-9_\-.]+):([0-9]+)/(.+)') match = regex.match(entry) if not match or match.groups()[3].find('?SFN=') >= 0: msg = "Given SRM endpoint is not valid! It must be of the form " + \ "srm://<hostname>:<port>/<path>. The hostname, port, and path " + \ "must be present. The path should not contain the string '?SFN='" raise exceptions.SettingError(msg) elif option == 'allowed_vos': user_vo_map = None if config.has_option('Install Locations', 'user_vo_map'): user_vo_map = config.get('Install Locations', 'user_vo_map') vo_list = utilities.get_vos(user_vo_map) for vo in entry: if vo not in vo_list: msg = "The vo %s is explicitly listed in the allowed_vos list in " % vo msg += "section %s, but is not in the list of allowed VOs." % section if vo_list: msg += " The list of allowed VOs are: %s." % ', '.join( vo_list) else: msg += " There are no allowed VOs detected; contact the experts!" raise exceptions.SettingError(msg) self.log('GipConfiguration.check_se completed') return attributes_ok