def testResourceEntryWithPilot(self): rce = RCEntry(name='glow.chtc.wisc.edu', cpus=1, memory=2500, max_wall_time=1440, allowed_vos=["glow"], gpus=1, max_pilots=1000, whole_node=False, queue="", require_singularity=True, os="rhel8", send_tests=True, is_pilot=True) expected_string = r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "glow" }; \ CPUs = 1; \ GPUs = 1; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Memory = 2500; \ Name = "glow.chtc.wisc.edu"; \ OS = "rhel8"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = False; \ ] \ }""" rc = ResourceCatalog() rc.add_rcentry(rce) actual_string = rc.compose_text() self.assertLongStringEqual(actual_string, expected_string)
def setUp(self): if not resourcecatalog: self.skipTest("No resourcecatalog") self.rc = ResourceCatalog()
class TestResourceCatalog(unittest.TestCase): def assertDoesNotRaise(self, exception, function, *args, **kwargs): try: function(*args, **kwargs) except exception: self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__)) def assertLongStringEqual(self, first, second, msg=None): self.assertEqual(first.splitlines(), second.splitlines(), msg=msg) def setUp(self): if not resourcecatalog: self.skipTest("No resourcecatalog") self.rc = ResourceCatalog() def testEmpty(self): self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}") def testSingle(self): self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000)) self.assertLongStringEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testMulti(self): (self.rc.add_rcentry(RCEntry( name='sc1', cpus=1, memory=2000)).add_rcentry( RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry( RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos=split_comma_separated_list( 'osg ,,,atlas')))) self.assertLongStringEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ CPUs = 2; \ Memory = 4000; \ Name = "sc2"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ Memory = 8000; \ Name = "sc3"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testExtraRequirements(self): rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1') self.rc.add_rcentry(rce) self.assertLongStringEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testExtraTransforms(self): rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1') self.rc.add_rcentry(rce) self.assertLongStringEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFull(self): config = configparser.SafeConfigParser() config_string = r""" [Subcluster Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 allowed_vos = osg, atlas """ config.read_string(config_string) self.assertLongStringEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testResourceEntry(self): # Test using the "Resource Entry" section name instead of "Subcluster" # and also using some of the attributes ATLAS requested config = configparser.SafeConfigParser() config_string = r""" [Resource Entry Valid] name = red.unl.edu maxmemory = 4000 cpucount = 4 queue = red vo_tag = ANALYSIS allowed_vos = osg, atlas """ config.read_string(config_string) self.assertLongStringEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \ Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \ VOTag = "ANALYSIS"; \ ] \ }""") def testResourceEntryWithSubclusters(self): config = configparser.SafeConfigParser() config_file = get_test_config("subcluster/resourceentry_and_sc.ini") config.read(config_file) self.assertDoesNotRaise(exceptions.SettingError, subcluster.resource_catalog_from_config, config) rc = subcluster.resource_catalog_from_config(config).compose_text() self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc, '\'subclusters\' attrib improperly transformed') def testResourceEntryBad(self): for config_filename in [ "subcluster/resourceentry_missing_cpucount.ini", "subcluster/resourceentry_missing_memory.ini", "subcluster/resourceentry_missing_queue.ini", "subcluster/resourceentry_missing_sc.ini" ]: config = configparser.SafeConfigParser() config_file = get_test_config(config_filename) config.read(config_file) try: self.assertRaises(exceptions.SettingError, subcluster.resource_catalog_from_config, config) except AssertionError: sys.stderr.write("Failed to raise error on " + config_filename) raise def testFullWithExtraTransforms(self): config = configparser.SafeConfigParser() config_string = r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 allowed_vos = osg, atlas """ config.read_string(config_string) self.assertLongStringEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""") def testResourceEntryWithPilot(self): rce = RCEntry(name='glow.chtc.wisc.edu', cpus=1, memory=2500, max_wall_time=1440, allowed_vos=["glow"], gpus=1, max_pilots=1000, whole_node=False, queue="", require_singularity=True, os="rhel8", send_tests=True, is_pilot=True) expected_string = r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "glow" }; \ CPUs = 1; \ GPUs = 1; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Memory = 2500; \ Name = "glow.chtc.wisc.edu"; \ OS = "rhel8"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = False; \ ] \ }""" rc = ResourceCatalog() rc.add_rcentry(rce) actual_string = rc.compose_text() self.assertLongStringEqual(actual_string, expected_string) def testPilot(self): config = configparser.SafeConfigParser() config_string = r""" [Pilot glow.chtc.wisc.edu] name = glow.chtc.wisc.edu #ram_mb = 2500 #cpucount = 1 #max_wall_time = 1440 allowed_vos = glow gpucount = 1 max_pilots = 1000 #whole_node = false #queue = #require_singularity = true os = rhel8 #send_tests = true """ config.read_string(config_string) expected_string = r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "glow" }; \ CPUs = 1; \ GPUs = 1; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Memory = 2500; \ Name = "glow.chtc.wisc.edu"; \ OS = "rhel8"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = False; \ ] \ }""" actual_string = subcluster.resource_catalog_from_config( config).compose_text() self.assertLongStringEqual(actual_string, expected_string) def testPilotExample(self): config_parser = configparser.SafeConfigParser() config_file = get_test_config("subcluster/pilots_example.ini") config_parser.read(config_file) expected_string = r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "icecube" }; \ CPUs = 1; \ GPUs = 2; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 2880; \ Memory = 8192; \ Name = "GPU"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = False; \ ], \ [ \ AllowedVOs = { "atlas" }; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Name = "WholeNode"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = True; \ ], \ [ \ AllowedVOs = { "osg", "cms" }; \ CPUs = 8; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Memory = 32768; \ Name = "default"; \ OS = "rhel6"; \ RequireSingularity = False; \ SendTests = True; \ WholeNode = False; \ ] \ } """ actual_string = subcluster.resource_catalog_from_config( config_parser).compose_text() self.assertLongStringEqual(actual_string, expected_string)
def setUp(self): self.rc = ResourceCatalog()
def resource_catalog_from_config( config: ConfigParser, default_allowed_vos: str = None) -> ResourceCatalog: """ Create a ResourceCatalog from the subcluster entries in a config :param default_allowed_vos: The allowed_vos to use if the user specified "*" """ logger = logging.getLogger(__name__) assert isinstance(config, ConfigParser) from osg_configure.modules.resourcecatalog import ResourceCatalog, RCEntry def safeget(option: str, default=None) -> str: return utilities.config_safe_get(config, section, option, default) def safegetbool(option: str, default=None) -> bool: return utilities.config_safe_getboolean(config, section, option, default) rc = ResourceCatalog() # list of section names of all subcluster sections subcluster_sections = [ section for section in config.sections() if is_subcluster(section) ] subcluster_names = [ rce_section_get_name(config, section) for section in subcluster_sections ] sections_without_max_wall_time = [] for section in config.sections(): if not (is_subcluster(section) or is_resource_entry(section) or is_pilot(section)): continue check_section(config, section) rcentry = RCEntry() rcentry.name = rce_section_get_name(config, section) rcentry.cpus = (safeget("cpucount") or safeget("cores_per_node") or CPUCOUNT_DEFAULT) rcentry.cpus = int(rcentry.cpus) rcentry.memory = (safeget("maxmemory") or safeget("ram_mb") or RAM_MB_DEFAULT) rcentry.memory = int(rcentry.memory) rcentry.allowed_vos = utilities.split_comma_separated_list( safeget("allowed_vos", default="").strip()) if not rcentry.allowed_vos or not rcentry.allowed_vos[0]: logger.error( "No allowed_vos specified for section '%s'." "\nThe factory will not send jobs to these subclusters/resources. Specify the allowed_vos" "\nattribute as either a list of VOs, or a '*' to use an autodetected VO list based on" "\nthe user accounts available on your CE." % section) raise exceptions.SettingError("No allowed_vos for %s" % section) if rcentry.allowed_vos == ["*"]: if default_allowed_vos: rcentry.allowed_vos = default_allowed_vos else: rcentry.allowed_vos = [] max_wall_time = safeget("max_wall_time") if not max_wall_time: rcentry.max_wall_time = 1440 sections_without_max_wall_time.append(section) else: rcentry.max_wall_time = max_wall_time.strip() rcentry.queue = safeget("queue") scs = utilities.split_comma_separated_list(safeget("subclusters", "")) if scs: for sc in scs: if sc not in subcluster_names: raise exceptions.SettingError( "Undefined subcluster '%s' mentioned in section '%s'" % (sc, section)) rcentry.subclusters = scs else: rcentry.subclusters = None rcentry.vo_tag = safeget("vo_tag") # The ability to specify extra requirements is disabled until admins demand it # rcentry.extra_requirements = utilities.config_safe_get(config, section, 'extra_requirements') rcentry.extra_requirements = None rcentry.extra_transforms = safeget("extra_transforms") rcentry.gpus = safeget("gpucount") if is_pilot(section): rcentry.max_pilots = safeget("max_pilots") rcentry.whole_node = safegetbool("whole_node", False) if rcentry.whole_node: rcentry.cpus = None rcentry.memory = None rcentry.require_singularity = safegetbool("require_singularity", True) rcentry.os = safeget("os") rcentry.send_tests = safegetbool("send_tests", True) rcentry.is_pilot = True rc.add_rcentry(rcentry) # end for section in config.sections() if sections_without_max_wall_time: logger.warning( "No max_wall_time specified for some sections; defaulting to 1440." "\nAdd 'max_wall_time=1440' to the following section(s) to clear this warning:" "\n'%s'" % "', '".join(sections_without_max_wall_time)) return rc
class TestResourceCatalog(unittest.TestCase): def assertDoesNotRaise(self, exception, function, *args, **kwargs): try: function(*args, **kwargs) except exception: self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__)) def setUp(self): self.rc = ResourceCatalog() def testEmpty(self): self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}") def testSingle(self): self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000)) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testMulti(self): (self.rc .add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000)) .add_rcentry(RCEntry(name='sc2', cpus=2, memory=4000)) .add_rcentry(RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas'))) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ CPUs = 2; \ Memory = 4000; \ Name = "sc2"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ Memory = 8000; \ Name = "sc3"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testNoName(self): rce = RCEntry(name='', cpus=1, memory=1) self.assertRaises(ValueError, self.rc.add_rcentry, rce) def testOutOfRange(self): rce = RCEntry(name='sc', cpus=-1, memory=1) self.assertRaises(ValueError, self.rc.add_rcentry, rce) rce.cpus = 1 rce.memory = 0 self.assertRaises(ValueError, self.rc.add_rcentry, rce) def testZeroMaxWallTime(self): rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0) self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce) def testExtraRequirements(self): rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1') self.rc.add_rcentry(rce) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testExtraTransforms(self): rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1') self.rc.add_rcentry(rce) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFull(self): config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFullResourceEntry(self): # Same as testFull, but using the "Resource Entry" section name instead of "Subcluster" config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Resource Entry Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFullWithExtraTransforms(self): config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFullWithExtras(self): # Disable this test because the feature is disabled for now return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_requirements = WantGPUs =?= 1 extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""")
class TestResourceCatalog(unittest.TestCase): def assertDoesNotRaise(self, exception, function, *args, **kwargs): try: function(*args, **kwargs) except exception: self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__)) def setUp(self): self.rc = ResourceCatalog() def testEmpty(self): self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}") def testSingle(self): self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000)) self.assertEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testMulti(self): (self.rc.add_rcentry(RCEntry( name='sc1', cpus=1, memory=2000)).add_rcentry( RCEntry(name='sc2', cpus=2, memory=4000)).add_rcentry( RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas'))) self.assertEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ CPUs = 2; \ Memory = 4000; \ Name = "sc2"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ Memory = 8000; \ Name = "sc3"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testNoName(self): rce = RCEntry(name='', cpus=1, memory=1) self.assertRaises(ValueError, self.rc.add_rcentry, rce) def testOutOfRange(self): rce = RCEntry(name='sc', cpus=-1, memory=1) self.assertRaises(ValueError, self.rc.add_rcentry, rce) rce.cpus = 1 rce.memory = 0 self.assertRaises(ValueError, self.rc.add_rcentry, rce) def testZeroMaxWallTime(self): rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0) self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce) def testExtraRequirements(self): rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1') self.rc.add_rcentry(rce) self.assertEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testExtraTransforms(self): rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1') self.rc.add_rcentry(rce) self.assertEqual( self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFull(self): config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 """) config.readfp(config_io) self.assertEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFullWithExtraTransforms(self): config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 """) config.readfp(config_io) self.assertEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFullWithExtras(self): # Disable this test because the feature is disabled for now return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_requirements = WantGPUs =?= 1 extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 """) config.readfp(config_io) self.assertEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""")
def setUp(self): if not resourcecatalog: return self.rc = ResourceCatalog()
class TestResourceCatalog(unittest.TestCase): def assertDoesNotRaise(self, exception, function, *args, **kwargs): try: function(*args, **kwargs) except exception: self.fail('%s called with %r and %r raised %s' % (function.__name__, args, kwargs, exception.__name__)) def setUp(self): if not resourcecatalog: return self.rc = ResourceCatalog() def testEmpty(self): if not resourcecatalog: return self.assertEqual(self.rc.compose_text().strip(), "OSG_ResourceCatalog = {}") def testSingle(self): if not resourcecatalog: return self.rc.add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000)) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testMulti(self): if not resourcecatalog: return (self.rc .add_rcentry(RCEntry(name='sc1', cpus=1, memory=2000)) .add_rcentry(RCEntry(name='sc2', cpus=2, memory=4000)) .add_rcentry(RCEntry(name='sc3', cpus=4, memory=8000, allowed_vos='osg ,,,atlas'))) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc1"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ CPUs = 2; \ Memory = 4000; \ Name = "sc2"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ], \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ Memory = 8000; \ Name = "sc3"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testNoName(self): if not resourcecatalog: return rce = RCEntry(name='', cpus=1, memory=1) self.assertRaises(ValueError, self.rc.add_rcentry, rce) def testOutOfRange(self): if not resourcecatalog: return rce = RCEntry(name='sc', cpus=-1, memory=1) self.assertRaises(ValueError, self.rc.add_rcentry, rce) rce.cpus = 1 rce.memory = 0 self.assertRaises(ValueError, self.rc.add_rcentry, rce) def testZeroMaxWallTime(self): if not resourcecatalog: return rce = RCEntry(name='sc', cpus=1, memory=1, max_wall_time=0) self.assertDoesNotRaise(ValueError, self.rc.add_rcentry, rce) def testExtraRequirements(self): if not resourcecatalog: return rce = RCEntry(name='sc', cpus=1, memory=2000, extra_requirements='TARGET.WantGPUs =?= 1') self.rc.add_rcentry(rce) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && TARGET.WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testExtraTransforms(self): if not resourcecatalog: return rce = RCEntry(name='sc', cpus=1, memory=2000, extra_transforms='set_WantRHEL6 = 1') self.rc.add_rcentry(rce) self.assertEqual(self.rc.compose_text().strip(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 1; \ Memory = 2000; \ Name = "sc"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFull(self): if not resourcecatalog: return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 allowed_vos = osg, atlas """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""") def testResourceEntry(self): if not resourcecatalog: return # Test using the "Resource Entry" section name instead of "Subcluster" # and also using some of the attributes ATLAS requested config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Resource Entry Valid] name = red.unl.edu maxmemory = 4000 cpucount = 4 queue = red vo_tag = ANALYSIS allowed_vos = osg, atlas """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \ Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \ VOTag = "ANALYSIS"; \ ] \ }""") def testResourceEntryWithSubclusters(self): if not resourcecatalog: return config = ConfigParser.SafeConfigParser() config_file = get_test_config("subcluster/resourceentry_and_sc.ini") config.read(config_file) self.assertDoesNotRaise(exceptions.SettingError, subcluster.resource_catalog_from_config, config) rc = subcluster.resource_catalog_from_config(config).compose_text() self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc, '\'subclusters\' attrib improperly transformed') def testResourceEntryBad(self): if not resourcecatalog: return for config_filename in ["subcluster/resourceentry_missing_cpucount.ini", "subcluster/resourceentry_missing_memory.ini", "subcluster/resourceentry_missing_queue.ini", "subcluster/resourceentry_missing_sc.ini"]: config = ConfigParser.SafeConfigParser() config_file = get_test_config(config_filename) config.read(config_file) try: self.assertRaises(exceptions.SettingError, subcluster.resource_catalog_from_config, config) except AssertionError: sys.stderr.write("Failed to raise error on " + config_filename) raise def testFullWithExtraTransforms(self): if not resourcecatalog: return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 allowed_vos = osg, atlas """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""") def testFullWithExtras(self): # Disable this test because the feature is disabled for now return if not resourcecatalog: return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_requirements = WantGPUs =?= 1 extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 allowed_vos = osg, atlas """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""")