def testResourceEntry(self): # Test using the "Resource Entry" section name instead of "Subcluster" # and also using some of the attributes ATLAS requested config = configparser.SafeConfigParser() config_string = r""" [Resource Entry Valid] name = red.unl.edu maxmemory = 4000 cpucount = 4 queue = red vo_tag = ANALYSIS allowed_vos = osg, atlas """ config.read_string(config_string) self.assertLongStringEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs) && TARGET.VOTag == "ANALYSIS"; \ Transform = [ set_MaxMemory = RequestMemory; set_VOTag = "ANALYSIS"; set_remote_queue = "red"; set_xcount = RequestCPUs; ]; \ VOTag = "ANALYSIS"; \ ] \ }""")
def testFullWithExtraTransforms(self): config = configparser.SafeConfigParser() config_string = r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 allowed_vos = osg, atlas """ config.read_string(config_string) self.assertLongStringEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "osg", "atlas" }; \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && member(TARGET.VO, AllowedVOs); \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""")
def testFullWithExtras(self): # Disable this test because the feature is disabled for now return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_requirements = WantGPUs =?= 1 extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""")
def testFullResourceEntry(self): # Same as testFull, but using the "Resource Entry" section name instead of "Subcluster" config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Resource Entry Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 """) config.readfp(config_io) self.assertEqual(subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""")
def testFull(self): config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Valid] name = red.unl.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 """) config.readfp(config_io) self.assertEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "red.unl.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory; \ Transform = [ set_MaxMemory = RequestMemory; set_xcount = RequestCPUs; ]; \ ] \ }""")
def testResourceEntryWithSubclusters(self): if not resourcecatalog: return config = ConfigParser.SafeConfigParser() config_file = get_test_config("subcluster/resourceentry_and_sc.ini") config.read(config_file) self.assertDoesNotRaise(exceptions.SettingError, subcluster.resource_catalog_from_config, config) rc = subcluster.resource_catalog_from_config(config).compose_text() self.assertTrue('Subclusters = { "SC1", "Sub Cluster 2" }; \\' in rc, '\'subclusters\' attrib improperly transformed')
def configure(self, attributes): """Configure installation using attributes""" self.log("InfoServicesConfiguration.configure started") if self.ignored: self.log("%s configuration ignored" % self.config_section, level=logging.WARNING) self.log('InfoServicesConfiguration.configure completed') return True if not self.enabled: self.log("Not enabled") self.log("InfoServicesConfiguration.configure completed") return True if self.copy_host_cert_for_service_cert: if not self.create_missing_service_cert_key( SERVICECERT_PATH, SERVICEKEY_PATH, 'tomcat'): self.log("Could not create service cert/key", level=logging.ERROR) return False if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled: if classad is None: self.log( "Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings." "\nEnsure the 'classad' Python module is installed and accessible to Python scripts." "\nIf using HTCondor from RPMs, install the 'condor-python' RPM." "\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH." "\nHTCondor version must be at least 8.2.0.", level=logging.WARNING) else: using_gums = self.authorization_method == 'xacml' default_allowed_vos = None try: misc.ensure_valid_user_vo_file(using_gums, logger=self.logger) default_allowed_vos = utilities.get_vos( misc.USER_VO_MAP_LOCATION) except exceptions.ConfigureError, err: self.log("Could not determine allowed VOs: %s" % str(err), level=logging.WARNING) self.resource_catalog = subcluster.resource_catalog_from_config( self.subcluster_sections, logger=self.logger, default_allowed_vos=default_allowed_vos) self._configure_ce_collector()
def testPilotExample(self): config_parser = configparser.SafeConfigParser() config_file = get_test_config("subcluster/pilots_example.ini") config_parser.read(config_file) expected_string = r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "icecube" }; \ CPUs = 1; \ GPUs = 2; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 2880; \ Memory = 8192; \ Name = "GPU"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = False; \ ], \ [ \ AllowedVOs = { "atlas" }; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Name = "WholeNode"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = True; \ ], \ [ \ AllowedVOs = { "osg", "cms" }; \ CPUs = 8; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Memory = 32768; \ Name = "default"; \ OS = "rhel6"; \ RequireSingularity = False; \ SendTests = True; \ WholeNode = False; \ ] \ } """ actual_string = subcluster.resource_catalog_from_config( config_parser).compose_text() self.assertLongStringEqual(actual_string, expected_string)
def configure(self, attributes): """Configure installation using attributes""" self.log("InfoServicesConfiguration.configure started") if self.ignored: self.log("%s configuration ignored" % self.config_section, level=logging.WARNING) self.log('InfoServicesConfiguration.configure completed') return True if not self.enabled: self.log("Not enabled") self.log("InfoServicesConfiguration.configure completed") return True if self.copy_host_cert_for_service_cert: if not self.create_missing_service_cert_key(SERVICECERT_PATH, SERVICEKEY_PATH, 'tomcat'): self.log("Could not create service cert/key", level=logging.ERROR) return False if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled: if classad is None: self.log("Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings." "\nEnsure the 'classad' Python module is installed and accessible to Python scripts." "\nIf using HTCondor from RPMs, install the 'condor-python' RPM." "\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH." "\nHTCondor version must be at least 8.2.0.", level=logging.WARNING) else: using_gums = self.authorization_method == 'xacml' default_allowed_vos = None try: misc.ensure_valid_user_vo_file(using_gums, logger=self.logger) default_allowed_vos = utilities.get_vos(misc.USER_VO_MAP_LOCATION) except exceptions.ConfigureError, err: self.log("Could not determine allowed VOs: %s" % str(err), level=logging.WARNING) self.resource_catalog = subcluster.resource_catalog_from_config(self.subcluster_sections, logger=self.logger, default_allowed_vos=default_allowed_vos) self._configure_ce_collector()
def testPilot(self): config = configparser.SafeConfigParser() config_string = r""" [Pilot glow.chtc.wisc.edu] name = glow.chtc.wisc.edu #ram_mb = 2500 #cpucount = 1 #max_wall_time = 1440 allowed_vos = glow gpucount = 1 max_pilots = 1000 #whole_node = false #queue = #require_singularity = true os = rhel8 #send_tests = true """ config.read_string(config_string) expected_string = r"""OSG_ResourceCatalog = { \ [ \ AllowedVOs = { "glow" }; \ CPUs = 1; \ GPUs = 1; \ IsPilotEntry = True; \ MaxPilots = 1000; \ MaxWallTime = 1440; \ Memory = 2500; \ Name = "glow.chtc.wisc.edu"; \ OS = "rhel8"; \ RequireSingularity = True; \ SendTests = True; \ WholeNode = False; \ ] \ }""" actual_string = subcluster.resource_catalog_from_config( config).compose_text() self.assertLongStringEqual(actual_string, expected_string)
def testFullWithExtras(self): # Disable this test because the feature is disabled for now return config = ConfigParser.SafeConfigParser() config_io = cStringIO.StringIO(r""" [Subcluster Test] name = glow.chtc.wisc.edu node_count = 60 ram_mb = 4000 cpu_model = Opteron 275 cpu_vendor = AMD cpu_speed_mhz = 2200 cpu_platform = x86_64 cpus_per_node = 2 cores_per_node = 4 inbound_network = FALSE outbound_network = TRUE HEPSPEC = 10 queue = blue extra_requirements = WantGPUs =?= 1 extra_transforms = set_WantRHEL6 = 1 max_wall_time = 1440 """) config.readfp(config_io) self.assertEqual( subcluster.resource_catalog_from_config(config).compose_text(), r"""OSG_ResourceCatalog = { \ [ \ CPUs = 4; \ MaxWallTime = 1440; \ Memory = 4000; \ Name = "glow.chtc.wisc.edu"; \ Requirements = TARGET.RequestCPUs <= CPUs && TARGET.RequestMemory <= Memory && WantGPUs =?= 1; \ Transform = [ set_MaxMemory = RequestMemory; set_WantRHEL6 = 1; set_remote_queue = "blue"; set_xcount = RequestCPUs; ]; \ ] \ }""")
def parse_configuration(self, configuration): """ Try to get configuration information from ConfigParser or SafeConfigParser object given by configuration and write recognized settings to attributes dict """ self.log('InfoServicesConfiguration.parse_configuration started') self.check_config(configuration) if not configuration.has_section(self.config_section): self.enabled = False self.log("%s section not in config file" % self.config_section) self.log('InfoServicesConfiguration.parse_configuration completed') return if not self.set_status(configuration): self.log('InfoServicesConfiguration.parse_configuration completed') return True self._set_default_servers(configuration) self.get_options(configuration, ignore_options=[ 'itb-ress-servers', 'itb-bdii-servers', 'osg-ress-servers', 'osg-bdii-servers', 'ress_servers', 'enabled', 'bdii_servers' ]) self.ce_collectors = self._parse_ce_collectors( self.options['ce_collectors'].value) self.misc_module.parse_configuration(configuration) def csg(section, option): return utilities.config_safe_get(configuration, section, option, None) def csgbool(section, option): return utilities.config_safe_getboolean(configuration, section, option, False) # We get some values for HTCondor-CE from the Site Information section self.osg_resource = csg('Site Information', 'resource') self.osg_resource_group = csg('Site Information', 'resource_group') # and the enabled batch systems from their respective sections self.enabled_batch_systems = [ bs for bs in BATCH_SYSTEMS if csgbool(bs, 'enabled') ] self.htcondor_gateway_enabled = csgbool('Gateway', 'htcondor_gateway_enabled') self.authorization_method = csg('Misc Services', 'authorization_method') self.subcluster_sections = ConfigParser.SafeConfigParser() self.gums_host = csg('Misc Services', 'gums_host') for section in configuration.sections(): if section.lower().startswith('subcluster') or section.lower( ).startswith('resource entry'): self.subcluster_sections.add_section(section) for key, value in configuration.items(section): self.subcluster_sections.set(section, key, value) if utilities.ce_installed( ) and not subcluster.check_config(configuration): self.log( "On a CE but no valid 'Subcluster' or 'Resource Entry' sections defined." " This is required to advertise the capabilities of your cluster to the central collector." " Jobs may not be sent to this CE.", level=logging.ERROR) raise exceptions.SettingError( "No Subcluster or Resource Entry sections") # Check resource catalog # This is a bit clunky to parse it here and not use the result in # configure(), but at this point we don't have a way of knowing what # default_allowed_vos should be. if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled and classad is not None: subcluster.resource_catalog_from_config(self.subcluster_sections, default_allowed_vos=None) self.log('InfoServicesConfiguration.parse_configuration completed')
def configure(self, attributes): """Configure installation using attributes""" self.log("InfoServicesConfiguration.configure started") if self.ignored: self.log("%s configuration ignored" % self.config_section, level=logging.WARNING) self.log('InfoServicesConfiguration.configure completed') return True if not self.enabled: self.log("Not enabled") self.log("InfoServicesConfiguration.configure completed") return True if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled: if classad is None: self.log( "Cannot configure HTCondor CE info services: unable to import HTCondor Python bindings." "\nEnsure the 'classad' Python module is installed and accessible to Python scripts." "\nIf using HTCondor from RPMs, install the 'condor-python' RPM." "\nIf not, you may need to add the directory containing the Python bindings to PYTHONPATH." "\nHTCondor version must be at least 8.2.0.", level=logging.WARNING) else: if self.authorization_method == 'vomsmap': error = False for requiredfile in [BAN_MAPFILE, BAN_VOMS_MAPFILE]: if not os.path.exists(requiredfile): self.log( "%s authorization requested but %s was not found." "\nThis will cause all mappings to fail." "\nPlease reinstall lcmaps >= 1.6.6-1.3 or create a blank %s yourself." % (self.authorization_method, requiredfile, requiredfile), level=logging.ERROR) error = True if error: return False default_allowed_vos = reversevomap.get_allowed_vos() else: using_gums = self.authorization_method == 'xacml' # HACK for SOFTWARE-2792 if using_gums: self.misc_module.update_gums_client_location() self._ensure_valid_user_vo_file() default_allowed_vos = utilities.get_vos( USER_VO_MAP_LOCATION) if not default_allowed_vos: # UGLY: only issue the warning if the admin has requested autodetection for some of their SCs/REs raise_warning = False for section in self.subcluster_sections.sections(): if utilities.config_safe_get(self.subcluster_sections, section, 'allowed_vos', '').strip() == "*": raise_warning = True if raise_warning: self.log( "Could not determine default allowed VOs for subclusters/resource entries.", level=logging.WARNING) if self.authorization_method == 'vomsmap': self.log( "Install vo-client-lcmaps-voms to obtain default mappings for VOs, and/or create" " your own mapfile at /etc/grid-security/voms-mapfile.", level=logging.WARNING) else: self.log( "Ensure %s exists and is non-empty, or fill out allowed_vos in all your" " Subcluster and Resource Entry sections." % USER_VO_MAP_LOCATION, level=logging.WARNING) try: self.resource_catalog = subcluster.resource_catalog_from_config( self.subcluster_sections, default_allowed_vos=default_allowed_vos) except exceptions.SettingError as err: self.log("Error in info services configuration: %s" % str(err), level=logging.ERROR) return False self._configure_ce_collector() self.log("InfoServicesConfiguration.configure completed") return True