def testMissingAttribute(self): """ Test the parsing when attributes are missing, should get exceptions """ config_file = get_test_config("siteattributes/siteattributes2.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = siteinformation.SiteInformation(logger=global_logger) try: settings.parse_configuration(configuration) except Exception as e: self.fail("Received exception while parsing configuration: %s" % e) mandatory_on_all = ['group'] # ^ TODO OSG 3.5: add "resource" to this list mandatory_on_ce = [ 'host_name', 'sponsor', 'contact', 'email', 'city', 'country', 'longitude', 'latitude' ] mandatory = list(mandatory_on_all) if ce_installed(): mandatory += mandatory_on_ce for option in mandatory: config_file = get_test_config("siteattributes/siteattributes1.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) configuration.remove_option('Site Information', option) settings = siteinformation.SiteInformation(logger=global_logger) self.assertRaises(exceptions.SettingError, settings.parse_configuration, configuration)
def testOASISConfig(self): """ Test storage parsing when using OASIS configuration """ # StorageConfiguration is not enabled on non-ce installs if not utilities.ce_installed(): return config_file = get_test_config("storage/oasis.ini") configuration = configparser.SafeConfigParser() configuration.read(config_file) settings = storage.StorageConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception as e: self.fail("Received exception while parsing configuration: %s" % e) attributes = settings.get_attributes() variables = { 'OSG_STORAGE_ELEMENT': 'False', 'OSG_DEFAULT_SE': 'test.domain.org', 'OSG_GRID': '/etc', 'OSG_APP': '/cvmfs/oasis.opensciencegrid.org', 'OSG_DATA': 'UNAVAILABLE', 'OSG_SITE_READ': '/var', 'OSG_SITE_WRITE': '/usr' } for var in variables: self.assertTrue(var in attributes, "Attribute %s missing" % var) self.assertEqual(attributes[var], variables[var], "Wrong value obtained for %s, got %s but " \ "expected %s" % (var, attributes[var], variables[var])) self.assertTrue(settings.check_attributes(attributes))
def testBadHost(self): """ Test the check_attributes function when the squid proxy hostname is not valie """ if not ce_installed(): return True config_file = get_test_config("squid/squid_bad_host.ini") configuration = configparser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception as e: self.fail("Received exception while parsing configuration: %s" % e) attributes = settings.get_attributes() self.assertFalse(settings.check_attributes(attributes), "Did not notice invalid host") config_file = get_test_config("squid/squid_bad_host2.ini") configuration = configparser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception as e: self.fail("Received exception while parsing configuration") attributes = settings.get_attributes() self.assertFalse(settings.check_attributes(attributes), "Did not notice invalid host")
def _parse_configuration(self, configuration): """ The meat of parse_configuration, runs after we've checked that GIP is enabled and we have the right RPMs installed. """ # The following is to set the user that gip files need to belong to # This can be overridden by setting the 'user' option in the [GIP] section self.gip_user = '******' if configuration.has_option(self.config_section, 'batch'): batch_opt = configuration.get(self.config_section, 'batch').lower() if (not utilities.blank(batch_opt) and batch_opt not in self._valid_batch_opt): msg = "The batch setting in %s must be a valid option " \ "(e.g. %s), %s was given" % (self.config_section, ",".join(self._valid_batch_opt), batch_opt) self.log(msg, level=logging.ERROR) raise exceptions.SettingError(msg) if utilities.ce_installed(): self._parse_configuration_ce(configuration) # Check for the presence of the classic SE has_classic_se = True try: has_classic_se = configuration.getboolean("GIP", "advertise_gsiftp") # pylint: disable-msg=W0702 # pylint: disable-msg=W0703 # pylint: disable-msg=W0704 except Exception: pass has_se = False for section in configuration.sections(): if not section.lower().startswith('se'): continue has_se = True self.check_se(configuration, section) if not has_se and not has_classic_se: try: self._check_entry(configuration, "GIP", "se_name", REQUIRED, STRING) except: msg = "There is no `SE` section, the old-style SE" + \ "setup in GIP is not configured, and there is no classic SE. " + \ " At least one must be configured. Please see the configuration" \ " documentation." raise exceptions.SettingError(msg) if configuration.has_option(self.config_section, 'user'): username = configuration.get(self.config_section, 'user') if not validation.valid_user(username): err_msg = "%s is not a valid account on this system" % username self.log(err_msg, section=self.config_section, option='user', level=logging.ERROR) raise exceptions.SettingError(err_msg) self.gip_user = username
def _parse_configuration(self, configuration): """ The meat of parse_configuration, runs after we've checked that GIP is enabled and we have the right RPMs installed. """ # The following is to set the user that gip files need to belong to # This can be overridden by setting the 'user' option in the [GIP] section self.gip_user = '******' if configuration.has_option(self.config_section, 'batch'): batch_opt = configuration.get(self.config_section, 'batch').lower() if (not utilities.blank(batch_opt) and batch_opt not in self._valid_batch_opt): msg = "The batch setting in %s must be a valid option " \ "(e.g. %s), %s was given" % (self.config_section, ",".join(self._valid_batch_opt), batch_opt) self.log(msg, level=logging.ERROR) raise exceptions.SettingError(msg) if utilities.ce_installed(): self._parse_configuration_ce(configuration) # Check for the presence of the classic SE has_classic_se = True try: has_classic_se = configuration.getboolean("GIP", "advertise_gsiftp") # pylint: disable-msg=W0702 # pylint: disable-msg=W0703 # pylint: disable-msg=W0704 except Exception: pass has_se = False for section in configuration.sections(): if not section.lower().startswith('se'): continue has_se = True self.check_se(configuration, section) if not has_se and not has_classic_se: try: self._check_entry(configuration, "GIP", "se_name", REQUIRED, STRING) except: msg = "There is no `SE` section, the old-style SE" + \ "setup in GIP is not configured, and there is no classic SE. " + \ " At least one must be configured. Please see the configuration" \ " documentation." raise exceptions.SettingError(msg) if configuration.has_option(self.config_section, 'user'): username = configuration.get(self.config_section, 'user') if not validation.valid_user(username): err_msg = "%s is not a valid account on this system" % username self.log(err_msg, section=self.config_section, option='user', level=logging.ERROR) raise exceptions.SettingError(err_msg) self.gip_user = username
def write_htcondor_ce_sentinel(self): if self.htcondor_gateway_enabled and utilities.ce_installed(): contents = utilities.read_file( self.HTCONDOR_CE_CONFIG_FILE, default="# This file is managed by osg-configure\n") contents = utilities.add_or_replace_setting(contents, "OSG_CONFIGURED", "true", quote_value=False) utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
def testBadCache(self): """ Test the check_attributes function when cache size is not an integer """ if not ce_installed(): return True config_file = get_test_config("squid/squid_bad_cache.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) self.assertRaises(exceptions.SettingError, settings.parse_configuration, configuration)
def testBadCache(self): """ Test the check_attributes function when cache size is not an integer """ if not ce_installed(): return True config_file = get_test_config("squid/squid_bad_cache.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) self.assertRaises(exceptions.SettingError, settings.parse_configuration, configuration)
def testValidSettings(self): """ Test the check_attributes function to see if it oks good attributes """ if not ce_installed(): return True config_file = get_test_config("squid/valid_settings.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testValidSettings(self): """ Test the check_attributes function to see if it oks good attributes """ if not ce_installed(): return True config_file = get_test_config("squid/valid_settings.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def get_option(config, section, option): """ Get an option from a config file with optional defaults and mandatory options. Arguments config -- a ConfigParser object to query section -- the ini section the option is located in option -- an Option object to information on the option to retrieve """ is_required_option = (option.required == Option.MANDATORY or (option.required == Option.MANDATORY_ON_CE and utilities.ce_installed())) if config.has_option(section, option.name): try: if not utilities.blank(config.get(section, option.name)): if option.opt_type == bool: option.value = config.getboolean(section, option.name) elif option.opt_type == int: option.value = config.getint(section, option.name) elif option.opt_type == float: option.value = config.getfloat(section, option.name) else: option.value = config.get(section, option.name) else: # if option is blank and there's a default for the option # return the default if possible, otherwise raise an exception # if the option is mandatory if option.default_value is not None: option.value = option.default_value elif is_required_option: raise exceptions.SettingError("Can't get value for %s in %s " \ "section and no default given" % \ (option.name, section)) except ValueError: error_mesg = "%s in %s section is of the wrong type" % ( option.name, section) raise exceptions.SettingError(error_mesg) elif is_required_option: err_mesg = "Can't get value for %s in %s section" % (option.name, section) raise exceptions.SettingError(err_mesg) else: option.value = option.default_value
def testBlankLocation(self): """ Test the check_attributes function to see if it raises an error when location is left blank """ if not ce_installed(): return True config_file = get_test_config("squid/squid_blank_location.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testParsing2(self): """ Test storage parsing """ # StorageConfiguration is not enabled on non-ce installs if not utilities.ce_installed(): return config_file = get_test_config("storage/storage2.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = storage.StorageConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testParsingProductionDefaults(self): """ Test infoservices parsing to make sure it picks up production defaults """ # need to be on a CE to get CE defaults if not utilities.ce_installed(): return config_file = get_test_config("infoservices/prod_defaults.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = infoservices.InfoServicesConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testParsingITBDefaults(self): """ Test infoservices parsing to make sure it picks up ITB defaults """ # need to be on a CE to get CE defaults if not utilities.ce_installed(): return config_file = get_test_config("infoservices/itb_defaults.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = infoservices.InfoServicesConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testValidProductionDefaults(self): """ Test the check_attributes function to see if it oks the production defaults """ # need to be on a CE to get CE defaults if not utilities.ce_installed(): return config_file = get_test_config("infoservices/prod_defaults.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = infoservices.InfoServicesConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testBlankLocation(self): """ Test the check_attributes function to see if it raises an error when location is left blank """ if not ce_installed(): return True config_file = get_test_config("squid/squid_blank_location.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testBadPort(self): """ Test the check_attributes function when port for the squid proxy is not an integer """ if not ce_installed(): return True config_file = get_test_config("squid/squid_bad_port.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testBadPort(self): """ Test the check_attributes function when port for the squid proxy is not an integer """ if not ce_installed(): return True config_file = get_test_config("squid/squid_bad_port.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testValidProductionDefaults(self): """ Test the check_attributes function to see if it oks the production defaults """ # need to be on a CE to get CE defaults if not utilities.ce_installed(): return config_file = get_test_config("infoservices/prod_defaults.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = infoservices.InfoServicesConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception, e: self.fail("Received exception while parsing configuration: %s" % e)
def testParsing1(self): """ Test storage parsing """ # StorageConfiguration is not enabled on non-ce installs if not utilities.ce_installed(): return config_file = get_test_config("storage/storage1.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) settings = storage.StorageConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception as e: self.fail("Received exception while parsing configuration: %s" % e) attributes = settings.get_attributes() variables = { 'OSG_STORAGE_ELEMENT': 'True', 'OSG_DEFAULT_SE': 'test.domain.org', 'OSG_GRID': '/tmp', 'OSG_APP': '/tmp', 'OSG_DATA': '/var', 'OSG_WN_TMP': '/usr', 'OSG_SITE_READ': '/bin', 'OSG_SITE_WRITE': '/usr/bin' } for var in variables: self.assertTrue(attributes.has_key(var), "Attribute %s missing" % var) self.assertEqual(attributes[var], variables[var], "Wrong value obtained for %s, got %s but " \ "expected %s" % (var, attributes[var], variables[var])) if not validation.valid_directory('/tmp/etc'): # handle cases where this is not run under osg test framework os.mkdir('/tmp/etc') os.chmod('/tmp/etc', 0o777) self.assertTrue(settings.check_attributes(attributes)) os.rmdir('/tmp/etc') else: self.assertTrue(settings.check_attributes(attributes))
def testMissingAttribute(self): """ Test the check_attributes function """ # StorageConfiguration is not enabled on non-ce installs if not utilities.ce_installed(): return mandatory = ['se_available'] for option in mandatory: config_file = get_test_config("storage/storage1.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) configuration.remove_option('Storage', option) settings = storage.StorageConfiguration(logger=global_logger) self.assertRaises(exceptions.SettingError, settings.parse_configuration, configuration)
def configure(self, attributes): """Configure installation using attributes""" self.log('MiscConfiguration.configure started') if not self.enabled: self.log('Not enabled') self.log('MiscConfiguration.configure completed') return True # run fetch-crl script if not utilities.fetch_crl(): self.log("Error while running fetch-crl script", level=logging.WARNING) if not utilities.crls_exist(): self.log( "No CRLs exist and Fetch-CRL couldn't download any. GSI authentication is likely to fail. To attempt another download, run 'fetch-crl'.", level=logging.WARNING) if self.authorization_method == 'gridmap': self._set_lcmaps_callout(False) elif self.authorization_method == 'local-gridmap': self.log( "local-gridmap is a deprecated auth method -- use 'gridmap' instead", level=logging.WARNING) self._set_lcmaps_callout(False) elif self.authorization_method == 'vomsmap': self._set_lcmaps_callout(True) else: assert False, "Invalid authorization_method should have been caught in check_attributes()" if self.options['edit_lcmaps_db'].value: self._write_lcmaps_file() else: self.log("Not updating lcmaps.db because edit_lcmaps_db is false", level=logging.DEBUG) if self.htcondor_gateway_enabled and utilities.ce_installed(): self.write_gridmap_to_htcondor_ce_config() self.log('MiscConfiguration.configure completed') return True
def testMissingAttribute(self): """ Test the check_attributes function """ # StorageConfiguration is not enabled on non-ce installs if not utilities.ce_installed(): return mandatory = ['se_available'] for option in mandatory: config_file = get_test_config("storage/storage1.ini") configuration = ConfigParser.SafeConfigParser() configuration.read(config_file) configuration.remove_option('Storage', option) settings = storage.StorageConfiguration(logger=global_logger) self.assertRaises(exceptions.SettingError, settings.parse_configuration, configuration)
def testValidSettings(self): """ Test the check_attributes function to see if it oks good attributes """ if not ce_installed(): return True config_file = get_test_config("squid/valid_settings.ini") configuration = configparser.SafeConfigParser() configuration.read(config_file) settings = squid.SquidConfiguration(logger=global_logger) try: settings.parse_configuration(configuration) except Exception as e: self.fail("Received exception while parsing configuration: %s" % e) attributes = settings.get_attributes() self.assertTrue(settings.check_attributes(attributes), "Correct locations incorrectly flagged as missing")
def configure(self, attributes): """Configure installation using attributes""" self.log('MiscConfiguration.configure started') if not self.enabled: self.log('Not enabled') self.log('MiscConfiguration.configure completed') return True # run fetch-crl script if not utilities.fetch_crl(): self.log("Error while running fetch-crl script", level=logging.ERROR) raise exceptions.ConfigureError( 'fetch-crl returned non-zero exit code') if self.authorization_method == 'xacml': self._set_lcmaps_callout(True) self.update_gums_client_location() elif self.authorization_method == 'gridmap': self._set_lcmaps_callout(False) elif self.authorization_method == 'local-gridmap': self._set_lcmaps_callout(False) elif self.authorization_method == 'vomsmap': self._set_lcmaps_callout(True) else: assert False, "Invalid authorization_method should have been caught in check_attributes()" if self.options['edit_lcmaps_db'].value: self._write_lcmaps_file() else: self.log("Not updating lcmaps.db because edit_lcmaps_db is false", level=logging.DEBUG) if self.htcondor_gateway_enabled and utilities.ce_installed(): self.write_gridmap_to_htcondor_ce_config() self.log('MiscConfiguration.configure completed') return True
def parse_configuration(self, configuration): """ Try to get configuration information from ConfigParser or SafeConfigParser object given by configuration and write recognized settings to attributes dict """ self.log('InfoServicesConfiguration.parse_configuration started') self.check_config(configuration) if not configuration.has_section(self.config_section): self.enabled = False self.log("%s section not in config file" % self.config_section) self.log('InfoServicesConfiguration.parse_configuration completed') return if not self.set_status(configuration): self.log('InfoServicesConfiguration.parse_configuration completed') return True self._set_default_servers(configuration) self.get_options(configuration, ignore_options=[ 'itb-ress-servers', 'itb-bdii-servers', 'osg-ress-servers', 'osg-bdii-servers', 'ress_servers', 'enabled', 'bdii_servers' ]) self.ce_collectors = self._parse_ce_collectors( self.options['ce_collectors'].value) self.misc_module.parse_configuration(configuration) def csg(section, option): return utilities.config_safe_get(configuration, section, option, None) def csgbool(section, option): return utilities.config_safe_getboolean(configuration, section, option, False) # We get some values for HTCondor-CE from the Site Information section self.osg_resource = csg('Site Information', 'resource') self.osg_resource_group = csg('Site Information', 'resource_group') # and the enabled batch systems from their respective sections self.enabled_batch_systems = [ bs for bs in BATCH_SYSTEMS if csgbool(bs, 'enabled') ] self.htcondor_gateway_enabled = csgbool('Gateway', 'htcondor_gateway_enabled') self.authorization_method = csg('Misc Services', 'authorization_method') self.subcluster_sections = ConfigParser.SafeConfigParser() self.gums_host = csg('Misc Services', 'gums_host') for section in configuration.sections(): if section.lower().startswith('subcluster') or section.lower( ).startswith('resource entry'): self.subcluster_sections.add_section(section) for key, value in configuration.items(section): self.subcluster_sections.set(section, key, value) if utilities.ce_installed( ) and not subcluster.check_config(configuration): self.log( "On a CE but no valid 'Subcluster' or 'Resource Entry' sections defined." " This is required to advertise the capabilities of your cluster to the central collector." " Jobs may not be sent to this CE.", level=logging.ERROR) raise exceptions.SettingError( "No Subcluster or Resource Entry sections") # Check resource catalog # This is a bit clunky to parse it here and not use the result in # configure(), but at this point we don't have a way of knowing what # default_allowed_vos should be. if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled and classad is not None: subcluster.resource_catalog_from_config(self.subcluster_sections, default_allowed_vos=None) self.log('InfoServicesConfiguration.parse_configuration completed')