def testResourceEntryWithPilot(self):
rce = RCEntry(name='glow.chtc.wisc.edu',
cpus=1,
memory=2500,
max_wall_time=1440,
allowed_vos=["glow"],
gpus=1,
max_pilots=1000,
whole_node=False,
queue="",
require_singularity=True,
os="rhel8",
send_tests=True,
is_pilot=True)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
rc = ResourceCatalog()
rc.add_rcentry(rce)
actual_string = rc.compose_text()
self.assertLongStringEqual(actual_string, expected_string)
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' %
(function.__name__, args, kwargs, exception.__name__))
def assertLongStringEqual(self, first, second, msg=None):
self.assertEqual(first.splitlines(), second.splitlines(), msg=msg)
def setUp(self):
if not resourcecatalog:
self.skipTest("No resourcecatalog")
self.rc = ResourceCatalog()
def testEmpty(self):
self.assertEqual(self.rc.compose_text().strip(),
"OSG_ResourceCatalog = {}")
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
(self.rc.add_rcentry(RCEntry(
name='sc1', cpus=1, memory=2000)).add_rcentry(
RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry(
RCEntry(name='sc3',
cpus=4,
memory=8000,
allowed_vos=split_comma_separated_list(
'osg ,,,atlas'))))
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraRequirements(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
config = configparser.SafeConfigParser()
config_string = r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
allowed_vos = osg, atlas
"""
config.read_string(config_string)
self.assertLongStringEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testResourceEntry(self):
# Test using the "Resource Entry" section name instead of "Subcluster"
# and also using some of the attributes ATLAS requested
config = configparser.SafeConfigParser()
config_string = r"""
[Resource Entry Valid]
name = red.unl.edu
maxmemory = 4000
cpucount = 4
queue = red
vo_tag = ANALYSIS
allowed_vos = osg, atlas
"""
config.read_string(config_string)
self.assertLongStringEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \
Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \
VOTag = "ANALYSIS"; \
] \
}""")
def testResourceEntryWithSubclusters(self):
config = configparser.SafeConfigParser()
config_file = get_test_config("subcluster/resourceentry_and_sc.ini")
config.read(config_file)
self.assertDoesNotRaise(exceptions.SettingError,
subcluster.resource_catalog_from_config,
config)
rc = subcluster.resource_catalog_from_config(config).compose_text()
self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc,
'\'subclusters\' attrib improperly transformed')
def testResourceEntryBad(self):
for config_filename in [
"subcluster/resourceentry_missing_cpucount.ini",
"subcluster/resourceentry_missing_memory.ini",
"subcluster/resourceentry_missing_queue.ini",
"subcluster/resourceentry_missing_sc.ini"
]:
config = configparser.SafeConfigParser()
config_file = get_test_config(config_filename)
config.read(config_file)
try:
self.assertRaises(exceptions.SettingError,
subcluster.resource_catalog_from_config,
config)
except AssertionError:
sys.stderr.write("Failed to raise error on " + config_filename)
raise
def testFullWithExtraTransforms(self):
config = configparser.SafeConfigParser()
config_string = r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
allowed_vos = osg, atlas
"""
config.read_string(config_string)
self.assertLongStringEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testResourceEntryWithPilot(self):
rce = RCEntry(name='glow.chtc.wisc.edu',
cpus=1,
memory=2500,
max_wall_time=1440,
allowed_vos=["glow"],
gpus=1,
max_pilots=1000,
whole_node=False,
queue="",
require_singularity=True,
os="rhel8",
send_tests=True,
is_pilot=True)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
rc = ResourceCatalog()
rc.add_rcentry(rce)
actual_string = rc.compose_text()
self.assertLongStringEqual(actual_string, expected_string)
def testPilot(self):
config = configparser.SafeConfigParser()
config_string = r"""
[Pilot glow.chtc.wisc.edu]
name = glow.chtc.wisc.edu
#ram_mb = 2500
#cpucount = 1
#max_wall_time = 1440
allowed_vos = glow
gpucount = 1
max_pilots = 1000
#whole_node = false
#queue =
#require_singularity = true
os = rhel8
#send_tests = true
"""
config.read_string(config_string)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
actual_string = subcluster.resource_catalog_from_config(
config).compose_text()
self.assertLongStringEqual(actual_string, expected_string)
def testPilotExample(self):
config_parser = configparser.SafeConfigParser()
config_file = get_test_config("subcluster/pilots_example.ini")
config_parser.read(config_file)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "icecube" }; \
CPUs = 1; \
GPUs = 2; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 2880; \
Memory = 8192; \
Name = "GPU"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
], \
[ \
AllowedVOs = { "atlas" }; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Name = "WholeNode"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = True; \
], \
[ \
AllowedVOs = { "osg", "cms" }; \
CPUs = 8; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 32768; \
Name = "default"; \
OS = "rhel6"; \
RequireSingularity = False; \
SendTests = True; \
WholeNode = False; \
] \
}
"""
actual_string = subcluster.resource_catalog_from_config(
config_parser).compose_text()
self.assertLongStringEqual(actual_string, expected_string)
def resource_catalog_from_config(
config: ConfigParser,
default_allowed_vos: str = None) -> ResourceCatalog:
"""
Create a ResourceCatalog from the subcluster entries in a config
:param default_allowed_vos: The allowed_vos to use if the user specified "*"
"""
logger = logging.getLogger(__name__)
assert isinstance(config, ConfigParser)
from osg_configure.modules.resourcecatalog import ResourceCatalog, RCEntry
def safeget(option: str, default=None) -> str:
return utilities.config_safe_get(config, section, option, default)
def safegetbool(option: str, default=None) -> bool:
return utilities.config_safe_getboolean(config, section, option,
default)
rc = ResourceCatalog()
# list of section names of all subcluster sections
subcluster_sections = [
section for section in config.sections() if is_subcluster(section)
]
subcluster_names = [
rce_section_get_name(config, section)
for section in subcluster_sections
]
sections_without_max_wall_time = []
for section in config.sections():
if not (is_subcluster(section) or is_resource_entry(section)
or is_pilot(section)):
continue
check_section(config, section)
rcentry = RCEntry()
rcentry.name = rce_section_get_name(config, section)
rcentry.cpus = (safeget("cpucount") or safeget("cores_per_node")
or CPUCOUNT_DEFAULT)
rcentry.cpus = int(rcentry.cpus)
rcentry.memory = (safeget("maxmemory") or safeget("ram_mb")
or RAM_MB_DEFAULT)
rcentry.memory = int(rcentry.memory)
rcentry.allowed_vos = utilities.split_comma_separated_list(
safeget("allowed_vos", default="").strip())
if not rcentry.allowed_vos or not rcentry.allowed_vos[0]:
logger.error(
"No allowed_vos specified for section '%s'."
"\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos"
"\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on"
"\nthe user accounts available on your CE." % section)
raise exceptions.SettingError("No allowed_vos for %s" % section)
if rcentry.allowed_vos == ["*"]:
if default_allowed_vos:
rcentry.allowed_vos = default_allowed_vos
else:
rcentry.allowed_vos = []
max_wall_time = safeget("max_wall_time")
if not max_wall_time:
rcentry.max_wall_time = 1440
sections_without_max_wall_time.append(section)
else:
rcentry.max_wall_time = max_wall_time.strip()
rcentry.queue = safeget("queue")
scs = utilities.split_comma_separated_list(safeget("subclusters", ""))
if scs:
for sc in scs:
if sc not in subcluster_names:
raise exceptions.SettingError(
"Undefined subcluster '%s' mentioned in section '%s'" %
(sc, section))
rcentry.subclusters = scs
else:
rcentry.subclusters = None
rcentry.vo_tag = safeget("vo_tag")
# The ability to specify extra requirements is disabled until admins demand it
# rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements')
rcentry.extra_requirements = None
rcentry.extra_transforms = safeget("extra_transforms")
rcentry.gpus = safeget("gpucount")
if is_pilot(section):
rcentry.max_pilots = safeget("max_pilots")
rcentry.whole_node = safegetbool("whole_node", False)
if rcentry.whole_node:
rcentry.cpus = None
rcentry.memory = None
rcentry.require_singularity = safegetbool("require_singularity",
True)
rcentry.os = safeget("os")
rcentry.send_tests = safegetbool("send_tests", True)
rcentry.is_pilot = True
rc.add_rcentry(rcentry)
# end for section in config.sections()
if sections_without_max_wall_time:
logger.warning(
"No max_wall_time specified for some sections; defaulting to 1440."
"\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:"
"\n'%s'" % "', '".join(sections_without_max_wall_time))
return rc
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__))
def setUp(self):
self.rc = ResourceCatalog()
def testEmpty(self):
self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}")
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
(self.rc
.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
.add_rcentry(RCEntry(name='sc2', cpus=2, memory=4000))
.add_rcentry(RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas')))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testNoName(self):
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testExtraRequirements(self):
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullResourceEntry(self):
# Same as testFull, but using the "Resource Entry" section name instead of "Subcluster"
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Resource Entry Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtraTransforms(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtras(self):
# Disable this test because the feature is disabled for now
return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_requirements = WantGPUs =?= 1
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' %
(function.__name__, args, kwargs, exception.__name__))
def setUp(self):
self.rc = ResourceCatalog()
def testEmpty(self):
self.assertEqual(self.rc.compose_text().strip(),
"OSG_ResourceCatalog = {}")
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
(self.rc.add_rcentry(RCEntry(
name='sc1', cpus=1, memory=2000)).add_rcentry(
RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry(
RCEntry(name='sc3',
cpus=4,
memory=8000,
allowed_vos='osg ,,,atlas')))
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testNoName(self):
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testExtraRequirements(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
""")
config.readfp(config_io)
self.assertEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtraTransforms(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtras(self):
# Disable this test because the feature is disabled for now
return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_requirements = WantGPUs =?= 1
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__))
def setUp(self):
if not resourcecatalog: return
self.rc = ResourceCatalog()
def testEmpty(self):
if not resourcecatalog: return
self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}")
def testSingle(self):
if not resourcecatalog: return
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
if not resourcecatalog: return
(self.rc
.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
.add_rcentry(RCEntry(name='sc2', cpus=2, memory=4000))
.add_rcentry(RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas')))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testNoName(self):
if not resourcecatalog: return
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testExtraRequirements(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testResourceEntry(self):
if not resourcecatalog: return
# Test using the "Resource Entry" section name instead of "Subcluster"
# and also using some of the attributes ATLAS requested
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Resource Entry Valid]
name = red.unl.edu
maxmemory = 4000
cpucount = 4
queue = red
vo_tag = ANALYSIS
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \
Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \
VOTag = "ANALYSIS"; \
] \
}""")
def testResourceEntryWithSubclusters(self):
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_file = get_test_config("subcluster/resourceentry_and_sc.ini")
config.read(config_file)
self.assertDoesNotRaise(exceptions.SettingError, subcluster.resource_catalog_from_config, config)
rc = subcluster.resource_catalog_from_config(config).compose_text()
self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc,
'\'subclusters\' attrib improperly transformed')
def testResourceEntryBad(self):
if not resourcecatalog: return
for config_filename in ["subcluster/resourceentry_missing_cpucount.ini",
"subcluster/resourceentry_missing_memory.ini",
"subcluster/resourceentry_missing_queue.ini",
"subcluster/resourceentry_missing_sc.ini"]:
config = ConfigParser.SafeConfigParser()
config_file = get_test_config(config_filename)
config.read(config_file)
try:
self.assertRaises(exceptions.SettingError, subcluster.resource_catalog_from_config, config)
except AssertionError:
sys.stderr.write("Failed to raise error on " + config_filename)
raise
def testFullWithExtraTransforms(self):
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtras(self):
# Disable this test because the feature is disabled for now
return
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_requirements = WantGPUs =?= 1
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")