def testMulti(self):
(self.rc.add_rcentry(RCEntry(
name='sc1', cpus=1, memory=2000)).add_rcentry(
RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry(
RCEntry(name='sc3',
cpus=4,
memory=8000,
allowed_vos=split_comma_separated_list(
'osg ,,,atlas'))))
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testOutOfRange(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testResourceEntryWithPilot(self):
rce = RCEntry(name='glow.chtc.wisc.edu',
cpus=1,
memory=2500,
max_wall_time=1440,
allowed_vos=["glow"],
gpus=1,
max_pilots=1000,
whole_node=False,
queue="",
require_singularity=True,
os="rhel8",
send_tests=True,
is_pilot=True)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
rc = ResourceCatalog()
rc.add_rcentry(rce)
actual_string = rc.compose_text()
self.assertLongStringEqual(actual_string, expected_string)
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraRequirements(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def resource_catalog_from_config(
config: ConfigParser,
default_allowed_vos: str = None) -> ResourceCatalog:
"""
Create a ResourceCatalog from the subcluster entries in a config
:param default_allowed_vos: The allowed_vos to use if the user specified "*"
"""
logger = logging.getLogger(__name__)
assert isinstance(config, ConfigParser)
from osg_configure.modules.resourcecatalog import ResourceCatalog, RCEntry
def safeget(option: str, default=None) -> str:
return utilities.config_safe_get(config, section, option, default)
def safegetbool(option: str, default=None) -> bool:
return utilities.config_safe_getboolean(config, section, option,
default)
rc = ResourceCatalog()
# list of section names of all subcluster sections
subcluster_sections = [
section for section in config.sections() if is_subcluster(section)
]
subcluster_names = [
rce_section_get_name(config, section)
for section in subcluster_sections
]
sections_without_max_wall_time = []
for section in config.sections():
if not (is_subcluster(section) or is_resource_entry(section)
or is_pilot(section)):
continue
check_section(config, section)
rcentry = RCEntry()
rcentry.name = rce_section_get_name(config, section)
rcentry.cpus = (safeget("cpucount") or safeget("cores_per_node")
or CPUCOUNT_DEFAULT)
rcentry.cpus = int(rcentry.cpus)
rcentry.memory = (safeget("maxmemory") or safeget("ram_mb")
or RAM_MB_DEFAULT)
rcentry.memory = int(rcentry.memory)
rcentry.allowed_vos = utilities.split_comma_separated_list(
safeget("allowed_vos", default="").strip())
if not rcentry.allowed_vos or not rcentry.allowed_vos[0]:
logger.error(
"No allowed_vos specified for section '%s'."
"\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos"
"\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on"
"\nthe user accounts available on your CE." % section)
raise exceptions.SettingError("No allowed_vos for %s" % section)
if rcentry.allowed_vos == ["*"]:
if default_allowed_vos:
rcentry.allowed_vos = default_allowed_vos
else:
rcentry.allowed_vos = []
max_wall_time = safeget("max_wall_time")
if not max_wall_time:
rcentry.max_wall_time = 1440
sections_without_max_wall_time.append(section)
else:
rcentry.max_wall_time = max_wall_time.strip()
rcentry.queue = safeget("queue")
scs = utilities.split_comma_separated_list(safeget("subclusters", ""))
if scs:
for sc in scs:
if sc not in subcluster_names:
raise exceptions.SettingError(
"Undefined subcluster '%s' mentioned in section '%s'" %
(sc, section))
rcentry.subclusters = scs
else:
rcentry.subclusters = None
rcentry.vo_tag = safeget("vo_tag")
# The ability to specify extra requirements is disabled until admins demand it
# rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
rcentry.extra_requirements = None
rcentry.extra_transforms = safeget("extra_transforms")
rcentry.gpus = safeget("gpucount")
if is_pilot(section):
rcentry.max_pilots = safeget("max_pilots")
rcentry.whole_node = safegetbool("whole_node", False)
if rcentry.whole_node:
rcentry.cpus = None
rcentry.memory = None
rcentry.require_singularity = safegetbool("require_singularity",
True)
rcentry.os = safeget("os")
rcentry.send_tests = safegetbool("send_tests", True)
rcentry.is_pilot = True
rc.add_rcentry(rcentry)
# end for section in config.sections()
if sections_without_max_wall_time:
logger.warning(
"No max_wall_time specified for some sections; defaulting to 1440."
"\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:"
"\n'%s'" % "', '".join(sections_without_max_wall_time))
return rc
def testOutOfRange(self):
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testNoName(self):
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testNoName(self):
if not resourcecatalog: return
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)