def resource_catalog_from_config(config, logger=utilities.NullLogger, default_allowed_vos=None):
"""
Create a ResourceCatalog from the subcluster entries in a config
:type config: ConfigParser.ConfigParser
:type logger: logging.Logger or None
:rtype: ResourceCatalog
"""
assert isinstance(config, ConfigParser.ConfigParser)
from osg_configure.modules import resourcecatalog
rc = resourcecatalog.ResourceCatalog()
subclusters_without_max_wall_time = []
for section in config.sections():
prefix = None
if section.lower().startswith('subcluster'):
prefix = 'subcluster'
elif section.lower().startswith('resource entry'):
prefix = 'resource entry'
else:
continue
check_section(config, section)
rcentry = resourcecatalog.RCEntry()
subcluster = section[len(prefix):].lstrip()
rcentry.name = config.get(section, 'name')
rcentry.cpus = config.getint(section, 'cores_per_node')
rcentry.memory = config.getint(section, 'ram_mb')
rcentry.allowed_vos = utilities.config_safe_get(config, section, 'allowed_vos',
default=default_allowed_vos)
max_wall_time = utilities.config_safe_get(config, section, 'max_wall_time')
if not max_wall_time:
rcentry.max_wall_time = 1440
subclusters_without_max_wall_time.append(subcluster)
else:
rcentry.max_wall_time = max_wall_time.strip()
rcentry.queue = utilities.config_safe_get(config, section, 'queue')
# The ability to specify extra requirements is disabled until admins demand it
# rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
rcentry.extra_requirements = None
rcentry.extra_transforms = utilities.config_safe_get(config, section, 'extra_transforms')
rc.add_rcentry(rcentry)
# end for section in config.sections()
if subclusters_without_max_wall_time:
logger.warning("No max_wall_time specified for some subclusters; defaulting to 1440."
"\nAdd 'max_wall_time=1440' to the following subcluster(s) to clear this warning:"
"\n%s" % ", ".join(subclusters_without_max_wall_time))
return rc
def resource_catalog_from_config(config,
logger=utilities.NullLogger,
default_allowed_vos=None):
"""
Create a ResourceCatalog from the subcluster entries in a config
:type config: ConfigParser.ConfigParser
:type logger: logging.Logger or None
:rtype: ResourceCatalog
"""
assert isinstance(config, ConfigParser.ConfigParser)
from osg_configure.modules import resourcecatalog
rc = resourcecatalog.ResourceCatalog()
subclusters_without_max_wall_time = []
for section in config.sections():
if section.lower().startswith('subcluster'):
check_section(config, section)
rcentry = resourcecatalog.RCEntry()
subcluster = section[len('subcluster'):].lstrip()
rcentry.name = config.get(section, 'name')
rcentry.cpus = config.getint(section, 'cores_per_node')
rcentry.memory = config.getint(section, 'ram_mb')
rcentry.allowed_vos = utilities.config_safe_get(
config, section, 'allowed_vos', default=default_allowed_vos)
max_wall_time = utilities.config_safe_get(config, section,
'max_wall_time')
if not max_wall_time:
rcentry.max_wall_time = 1440
subclusters_without_max_wall_time.append(subcluster)
else:
rcentry.max_wall_time = max_wall_time.strip()
rcentry.queue = utilities.config_safe_get(config, section, 'queue')
# The ability to specify extra requirements is disabled until admins demand it
# rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
rcentry.extra_requirements = None
rcentry.extra_transforms = utilities.config_safe_get(
config, section, 'extra_transforms')
rc.add_rcentry(rcentry)
# end for section in config.sections()
if subclusters_without_max_wall_time:
logger.warning(
"No max_wall_time specified for some subclusters; defaulting to 1440."
"\nAdd 'max_wall_time=1440' to the following subcluster(s) to clear this warning:"
"\n%s" % ", ".join(subclusters_without_max_wall_time))
return rc
def _set_default_servers(self, configuration):
group = utilities.config_safe_get(configuration, 'Site Information', 'group')
for key in ['bdii_servers', 'ce_collectors']:
if group == 'OSG-ITB':
self.options[key].default_value = self._itb_defaults[key]
else:
self.options[key].default_value = self._production_defaults[key]
def _set_default_servers(self, configuration):
group = utilities.config_safe_get(configuration, 'Site Information',
'group')
if group == 'OSG-ITB':
self.options[
'ce_collectors'].default_value = self._itb_default_ce_collectors
else:
self.options[
'ce_collectors'].default_value = self._production_default_ce_collectors
def check_section(config, section):
"""
Check attributes related to a subcluster and make sure that they are consistent
"""
if "changeme" in section.lower():
msg = "You have a section named '%s', you must change this name.\n" % section
raise exceptions.SettingError(msg)
for option, value in ENTRIES.items():
status, kind = value
entry = check_entry(config, section, option, status, kind)
if option in BANNED_ENTRIES and entry == BANNED_ENTRIES[option]:
raise exceptions.SettingError("Value for %s in section %s is " \
"a default or banned entry (%s); " \
"you must change this value." % \
(option, section, BANNED_ENTRIES[option]))
if entry is None:
continue
try:
range_min, range_max = ENTRY_RANGES[option]
if not (range_min <= entry <= range_max):
msg = (
"Value for %(option)s in section %(section)s is outside allowed range"
", %(range_min)d-%(range_max)d" % locals())
raise exceptions.SettingError(msg)
except KeyError:
pass
# Special case: Pilot sections either need "os" specified or "require_singularity=true"
if is_pilot(section):
require_singularity = utilities.config_safe_getboolean(
config, section, "require_singularity", True)
os = utilities.config_safe_get(config, section, "os", None)
if not require_singularity and not os:
msg = "'os' must be specified in section %s if 'require_singularity' is false" % section
raise exceptions.SettingError(msg)
def resource_catalog_from_config(config, default_allowed_vos=None):
"""
Create a ResourceCatalog from the subcluster entries in a config
:type config: ConfigParser.ConfigParser
:rtype: ResourceCatalog
"""
logger = logging.getLogger(__name__)
assert isinstance(config, ConfigParser.ConfigParser)
from osg_configure.modules import resourcecatalog
rc = resourcecatalog.ResourceCatalog()
# list of section names of all subcluster sections
subcluster_sections = [section for section in config.sections() if section.lower().startswith('subcluster')]
subcluster_names = [config.get(section, 'name').strip() for section in subcluster_sections]
sections_without_max_wall_time = []
for section in config.sections():
lsection = section.lower()
if not (lsection.startswith('subcluster') or lsection.startswith('resource entry')):
continue
check_section(config, section)
rcentry = resourcecatalog.RCEntry()
rcentry.name = config.get(section, 'name')
rcentry.cpus = utilities.config_safe_get(config, section, 'cpucount') or \
utilities.config_safe_get(config, section, 'cores_per_node')
if not rcentry.cpus:
raise exceptions.SettingError("cpucount / cores_per_node not found in section %s" % section)
rcentry.cpus = int(rcentry.cpus)
rcentry.memory = utilities.config_safe_get(config, section, 'maxmemory') or \
utilities.config_safe_get(config, section, 'ram_mb')
if not rcentry.memory:
raise exceptions.SettingError("maxmemory / ram_mb not found in section %s" % section)
rcentry.memory = int(rcentry.memory)
rcentry.allowed_vos = utilities.config_safe_get(config, section, 'allowed_vos', default="").strip()
if not rcentry.allowed_vos:
logger.error("No allowed_vos specified for section '%s'."
"\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos"
"\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on"
"\nthe user accounts available on your CE." % section)
raise exceptions.SettingError("No allowed_vos for %s" % section)
if rcentry.allowed_vos == "*":
if default_allowed_vos:
rcentry.allowed_vos = default_allowed_vos
else:
rcentry.allowed_vos = None
max_wall_time = utilities.config_safe_get(config, section, 'max_wall_time')
if not max_wall_time:
rcentry.max_wall_time = 1440
sections_without_max_wall_time.append(section)
else:
rcentry.max_wall_time = max_wall_time.strip()
rcentry.queue = utilities.config_safe_get(config, section, 'queue')
scs = utilities.config_safe_get(config, section, 'subclusters')
if scs:
scs = re.split(r'\s*,\s*', scs)
for sc in scs:
if sc not in subcluster_names:
raise exceptions.SettingError("Undefined subcluster '%s' mentioned in section '%s'" % (sc, section))
rcentry.subclusters = scs
rcentry.vo_tag = utilities.config_safe_get(config, section, 'vo_tag')
# The ability to specify extra requirements is disabled until admins demand it
# rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
rcentry.extra_requirements = None
rcentry.extra_transforms = utilities.config_safe_get(config, section, 'extra_transforms')
rc.add_rcentry(rcentry)
# end for section in config.sections()
if sections_without_max_wall_time:
logger.warning("No max_wall_time specified for some sections; defaulting to 1440."
"\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:"
"\n'%s'" % "', '".join(sections_without_max_wall_time))
return rc
def configure(self, attributes):
"""Configure installation using attributes"""
self.log("InfoServicesConfiguration.configure started")
if self.ignored:
self.log("%s configuration ignored" % self.config_section,
level=logging.WARNING)
self.log('InfoServicesConfiguration.configure completed')
return True
if not self.enabled:
self.log("Not enabled")
self.log("InfoServicesConfiguration.configure completed")
return True
if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled:
if classad is None:
self.log(
"Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings."
"\nEnsure the 'classad' Python module is installed and accessible to Python scripts."
"\nIf using HTCondor from RPMs, install the 'condor-python' RPM."
"\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH."
"\nHTCondor version must be at least 8.2.0.",
level=logging.WARNING)
else:
if self.authorization_method == 'vomsmap':
error = False
for requiredfile in [BAN_MAPFILE, BAN_VOMS_MAPFILE]:
if not os.path.exists(requiredfile):
self.log(
"%s authorization requested but %s was not found."
"\nThis will cause all mappings to fail."
"\nPlease reinstall lcmaps >= 1.6.6-1.3 or create a blank %s yourself."
% (self.authorization_method, requiredfile,
requiredfile),
level=logging.ERROR)
error = True
if error:
return False
default_allowed_vos = reversevomap.get_allowed_vos()
else:
using_gums = self.authorization_method == 'xacml'
# HACK for SOFTWARE-2792
if using_gums:
self.misc_module.update_gums_client_location()
self._ensure_valid_user_vo_file()
default_allowed_vos = utilities.get_vos(
USER_VO_MAP_LOCATION)
if not default_allowed_vos:
# UGLY: only issue the warning if the admin has requested autodetection for some of their SCs/REs
raise_warning = False
for section in self.subcluster_sections.sections():
if utilities.config_safe_get(self.subcluster_sections,
section, 'allowed_vos',
'').strip() == "*":
raise_warning = True
if raise_warning:
self.log(
"Could not determine default allowed VOs for subclusters/resource entries.",
level=logging.WARNING)
if self.authorization_method == 'vomsmap':
self.log(
"Install vo-client-lcmaps-voms to obtain default mappings for VOs, and/or create"
" your own mapfile at /etc/grid-security/voms-mapfile.",
level=logging.WARNING)
else:
self.log(
"Ensure %s exists and is non-empty, or fill out allowed_vos in all your"
" Subcluster and Resource Entry sections." %
USER_VO_MAP_LOCATION,
level=logging.WARNING)
try:
self.resource_catalog = subcluster.resource_catalog_from_config(
self.subcluster_sections,
default_allowed_vos=default_allowed_vos)
except exceptions.SettingError as err:
self.log("Error in info services configuration: %s" %
str(err),
level=logging.ERROR)
return False
self._configure_ce_collector()
self.log("InfoServicesConfiguration.configure completed")
return True
def csg(section, option):
return utilities.config_safe_get(configuration, section, option,
None)
def safeget(option: str, default=None) -> str:
return utilities.config_safe_get(config, section, option, default)
def csg(section, option):
return utilities.config_safe_get(configuration, section, option, None)
