def testResourceEntryWithPilot(self):
rce = RCEntry(name='glow.chtc.wisc.edu',
cpus=1,
memory=2500,
max_wall_time=1440,
allowed_vos=["glow"],
gpus=1,
max_pilots=1000,
whole_node=False,
queue="",
require_singularity=True,
os="rhel8",
send_tests=True,
is_pilot=True)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
rc = ResourceCatalog()
rc.add_rcentry(rce)
actual_string = rc.compose_text()
self.assertLongStringEqual(actual_string, expected_string)
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' %
(function.__name__, args, kwargs, exception.__name__))
def assertLongStringEqual(self, first, second, msg=None):
self.assertEqual(first.splitlines(), second.splitlines(), msg=msg)
def setUp(self):
if not resourcecatalog:
self.skipTest("No resourcecatalog")
self.rc = ResourceCatalog()
def testEmpty(self):
self.assertEqual(self.rc.compose_text().strip(),
"OSG_ResourceCatalog = {}")
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
(self.rc.add_rcentry(RCEntry(
name='sc1', cpus=1, memory=2000)).add_rcentry(
RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry(
RCEntry(name='sc3',
cpus=4,
memory=8000,
allowed_vos=split_comma_separated_list(
'osg ,,,atlas'))))
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraRequirements(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertLongStringEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
config = configparser.SafeConfigParser()
config_string = r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
allowed_vos = osg, atlas
"""
config.read_string(config_string)
self.assertLongStringEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testResourceEntry(self):
# Test using the "Resource Entry" section name instead of "Subcluster"
# and also using some of the attributes ATLAS requested
config = configparser.SafeConfigParser()
config_string = r"""
[Resource Entry Valid]
name = red.unl.edu
maxmemory = 4000
cpucount = 4
queue = red
vo_tag = ANALYSIS
allowed_vos = osg, atlas
"""
config.read_string(config_string)
self.assertLongStringEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \
Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \
VOTag = "ANALYSIS"; \
] \
}""")
def testResourceEntryWithSubclusters(self):
config = configparser.SafeConfigParser()
config_file = get_test_config("subcluster/resourceentry_and_sc.ini")
config.read(config_file)
self.assertDoesNotRaise(exceptions.SettingError,
subcluster.resource_catalog_from_config,
config)
rc = subcluster.resource_catalog_from_config(config).compose_text()
self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc,
'\'subclusters\' attrib improperly transformed')
def testResourceEntryBad(self):
for config_filename in [
"subcluster/resourceentry_missing_cpucount.ini",
"subcluster/resourceentry_missing_memory.ini",
"subcluster/resourceentry_missing_queue.ini",
"subcluster/resourceentry_missing_sc.ini"
]:
config = configparser.SafeConfigParser()
config_file = get_test_config(config_filename)
config.read(config_file)
try:
self.assertRaises(exceptions.SettingError,
subcluster.resource_catalog_from_config,
config)
except AssertionError:
sys.stderr.write("Failed to raise error on " + config_filename)
raise
def testFullWithExtraTransforms(self):
config = configparser.SafeConfigParser()
config_string = r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
allowed_vos = osg, atlas
"""
config.read_string(config_string)
self.assertLongStringEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testResourceEntryWithPilot(self):
rce = RCEntry(name='glow.chtc.wisc.edu',
cpus=1,
memory=2500,
max_wall_time=1440,
allowed_vos=["glow"],
gpus=1,
max_pilots=1000,
whole_node=False,
queue="",
require_singularity=True,
os="rhel8",
send_tests=True,
is_pilot=True)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
rc = ResourceCatalog()
rc.add_rcentry(rce)
actual_string = rc.compose_text()
self.assertLongStringEqual(actual_string, expected_string)
def testPilot(self):
config = configparser.SafeConfigParser()
config_string = r"""
[Pilot glow.chtc.wisc.edu]
name = glow.chtc.wisc.edu
#ram_mb = 2500
#cpucount = 1
#max_wall_time = 1440
allowed_vos = glow
gpucount = 1
max_pilots = 1000
#whole_node = false
#queue =
#require_singularity = true
os = rhel8
#send_tests = true
"""
config.read_string(config_string)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "glow" }; \
CPUs = 1; \
GPUs = 1; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 2500; \
Name = "glow.chtc.wisc.edu"; \
OS = "rhel8"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
] \
}"""
actual_string = subcluster.resource_catalog_from_config(
config).compose_text()
self.assertLongStringEqual(actual_string, expected_string)
def testPilotExample(self):
config_parser = configparser.SafeConfigParser()
config_file = get_test_config("subcluster/pilots_example.ini")
config_parser.read(config_file)
expected_string = r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "icecube" }; \
CPUs = 1; \
GPUs = 2; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 2880; \
Memory = 8192; \
Name = "GPU"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = False; \
], \
[ \
AllowedVOs = { "atlas" }; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Name = "WholeNode"; \
RequireSingularity = True; \
SendTests = True; \
WholeNode = True; \
], \
[ \
AllowedVOs = { "osg", "cms" }; \
CPUs = 8; \
IsPilotEntry = True; \
MaxPilots = 1000; \
MaxWallTime = 1440; \
Memory = 32768; \
Name = "default"; \
OS = "rhel6"; \
RequireSingularity = False; \
SendTests = True; \
WholeNode = False; \
] \
}
"""
actual_string = subcluster.resource_catalog_from_config(
config_parser).compose_text()
self.assertLongStringEqual(actual_string, expected_string)
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' %
(function.__name__, args, kwargs, exception.__name__))
def setUp(self):
self.rc = ResourceCatalog()
def testEmpty(self):
self.assertEqual(self.rc.compose_text().strip(),
"OSG_ResourceCatalog = {}")
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
(self.rc.add_rcentry(RCEntry(
name='sc1', cpus=1, memory=2000)).add_rcentry(
RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry(
RCEntry(name='sc3',
cpus=4,
memory=8000,
allowed_vos='osg ,,,atlas')))
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testNoName(self):
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testExtraRequirements(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc',
cpus=1,
memory=2000,
extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertEqual(
self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
""")
config.readfp(config_io)
self.assertEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtraTransforms(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtras(self):
# Disable this test because the feature is disabled for now
return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_requirements = WantGPUs =?= 1
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(
subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__))
def setUp(self):
self.rc = ResourceCatalog()
def testEmpty(self):
self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}")
def testSingle(self):
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
(self.rc
.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
.add_rcentry(RCEntry(name='sc2', cpus=2, memory=4000))
.add_rcentry(RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas')))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testNoName(self):
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testExtraRequirements(self):
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullResourceEntry(self):
# Same as testFull, but using the "Resource Entry" section name instead of "Subcluster"
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Resource Entry Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtraTransforms(self):
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtras(self):
# Disable this test because the feature is disabled for now
return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_requirements = WantGPUs =?= 1
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
class TestResourceCatalog(unittest.TestCase):
def assertDoesNotRaise(self, exception, function, *args, **kwargs):
try:
function(*args, **kwargs)
except exception:
self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__))
def setUp(self):
if not resourcecatalog: return
self.rc = ResourceCatalog()
def testEmpty(self):
if not resourcecatalog: return
self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}")
def testSingle(self):
if not resourcecatalog: return
self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testMulti(self):
if not resourcecatalog: return
(self.rc
.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000))
.add_rcentry(RCEntry(name='sc2', cpus=2, memory=4000))
.add_rcentry(RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas')))
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc1"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
CPUs = 2; \
Memory = 4000; \
Name = "sc2"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
], \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
Memory = 8000; \
Name = "sc3"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testNoName(self):
if not resourcecatalog: return
rce = RCEntry(name='', cpus=1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testOutOfRange(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=-1, memory=1)
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
rce.cpus = 1
rce.memory = 0
self.assertRaises(ValueError, self.rc.add_rcentry, rce)
def testZeroMaxWallTime(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0)
self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce)
def testExtraRequirements(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testExtraTransforms(self):
if not resourcecatalog: return
rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1')
self.rc.add_rcentry(rce)
self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \
[ \
CPUs = 1; \
Memory = 2000; \
Name = "sc"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFull(self):
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Valid]
name = red.unl.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \
] \
}""")
def testResourceEntry(self):
if not resourcecatalog: return
# Test using the "Resource Entry" section name instead of "Subcluster"
# and also using some of the attributes ATLAS requested
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Resource Entry Valid]
name = red.unl.edu
maxmemory = 4000
cpucount = 4
queue = red
vo_tag = ANALYSIS
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "red.unl.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \
Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \
VOTag = "ANALYSIS"; \
] \
}""")
def testResourceEntryWithSubclusters(self):
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_file = get_test_config("subcluster/resourceentry_and_sc.ini")
config.read(config_file)
self.assertDoesNotRaise(exceptions.SettingError, subcluster.resource_catalog_from_config, config)
rc = subcluster.resource_catalog_from_config(config).compose_text()
self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc,
'\'subclusters\' attrib improperly transformed')
def testResourceEntryBad(self):
if not resourcecatalog: return
for config_filename in ["subcluster/resourceentry_missing_cpucount.ini",
"subcluster/resourceentry_missing_memory.ini",
"subcluster/resourceentry_missing_queue.ini",
"subcluster/resourceentry_missing_sc.ini"]:
config = ConfigParser.SafeConfigParser()
config_file = get_test_config(config_filename)
config.read(config_file)
try:
self.assertRaises(exceptions.SettingError, subcluster.resource_catalog_from_config, config)
except AssertionError:
sys.stderr.write("Failed to raise error on " + config_filename)
raise
def testFullWithExtraTransforms(self):
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
def testFullWithExtras(self):
# Disable this test because the feature is disabled for now
return
if not resourcecatalog: return
config = ConfigParser.SafeConfigParser()
config_io = cStringIO.StringIO(r"""
[Subcluster Test]
name = glow.chtc.wisc.edu
node_count = 60
ram_mb = 4000
cpu_model = Opteron 275
cpu_vendor = AMD
cpu_speed_mhz = 2200
cpu_platform = x86_64
cpus_per_node = 2
cores_per_node = 4
inbound_network = FALSE
outbound_network = TRUE
HEPSPEC = 10
queue = blue
extra_requirements = WantGPUs =?= 1
extra_transforms = set_WantRHEL6 = 1
max_wall_time = 1440
allowed_vos = osg, atlas
""")
config.readfp(config_io)
self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(),
r"""OSG_ResourceCatalog = { \
[ \
AllowedVOs = { "osg", "atlas" }; \
CPUs = 4; \
MaxWallTime = 1440; \
Memory = 4000; \
Name = "glow.chtc.wisc.edu"; \
Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && WantGPUs =?= 1; \
Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \
] \
}""")
