def testMissingAttribute(self):
"""
Test the parsing when attributes are missing, should get exceptions
"""
config_file = get_test_config("siteattributes/siteattributes2.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = siteinformation.SiteInformation(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
mandatory_on_all = ['group']
# ^ TODO OSG 3.5: add "resource" to this list
mandatory_on_ce = [
'host_name', 'sponsor', 'contact', 'email', 'city', 'country',
'longitude', 'latitude'
]
mandatory = list(mandatory_on_all)
if ce_installed():
mandatory += mandatory_on_ce
for option in mandatory:
config_file = get_test_config("siteattributes/siteattributes1.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
configuration.remove_option('Site Information', option)
settings = siteinformation.SiteInformation(logger=global_logger)
self.assertRaises(exceptions.SettingError,
settings.parse_configuration, configuration)
def testOASISConfig(self):
"""
Test storage parsing when using OASIS configuration
"""
# StorageConfiguration is not enabled on non-ce installs
if not utilities.ce_installed():
return
config_file = get_test_config("storage/oasis.ini")
configuration = configparser.SafeConfigParser()
configuration.read(config_file)
settings = storage.StorageConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
attributes = settings.get_attributes()
variables = {
'OSG_STORAGE_ELEMENT': 'False',
'OSG_DEFAULT_SE': 'test.domain.org',
'OSG_GRID': '/etc',
'OSG_APP': '/cvmfs/oasis.opensciencegrid.org',
'OSG_DATA': 'UNAVAILABLE',
'OSG_SITE_READ': '/var',
'OSG_SITE_WRITE': '/usr'
}
for var in variables:
self.assertTrue(var in attributes, "Attribute %s missing" % var)
self.assertEqual(attributes[var],
variables[var],
"Wrong value obtained for %s, got %s but " \
"expected %s" % (var, attributes[var], variables[var]))
self.assertTrue(settings.check_attributes(attributes))
def testBadHost(self):
"""
Test the check_attributes function when the squid proxy hostname is
not valie
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_bad_host.ini")
configuration = configparser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Did not notice invalid host")
config_file = get_test_config("squid/squid_bad_host2.ini")
configuration = configparser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Did not notice invalid host")
def _parse_configuration(self, configuration):
"""
The meat of parse_configuration, runs after we've checked that GIP is
enabled and we have the right RPMs installed.
"""
# The following is to set the user that gip files need to belong to
# This can be overridden by setting the 'user' option in the [GIP] section
self.gip_user = '******'
if configuration.has_option(self.config_section, 'batch'):
batch_opt = configuration.get(self.config_section, 'batch').lower()
if (not utilities.blank(batch_opt)
and batch_opt not in self._valid_batch_opt):
msg = "The batch setting in %s must be a valid option " \
"(e.g. %s), %s was given" % (self.config_section,
",".join(self._valid_batch_opt),
batch_opt)
self.log(msg, level=logging.ERROR)
raise exceptions.SettingError(msg)
if utilities.ce_installed():
self._parse_configuration_ce(configuration)
# Check for the presence of the classic SE
has_classic_se = True
try:
has_classic_se = configuration.getboolean("GIP",
"advertise_gsiftp")
# pylint: disable-msg=W0702
# pylint: disable-msg=W0703
# pylint: disable-msg=W0704
except Exception:
pass
has_se = False
for section in configuration.sections():
if not section.lower().startswith('se'):
continue
has_se = True
self.check_se(configuration, section)
if not has_se and not has_classic_se:
try:
self._check_entry(configuration, "GIP", "se_name", REQUIRED,
STRING)
except:
msg = "There is no `SE` section, the old-style SE" + \
"setup in GIP is not configured, and there is no classic SE. " + \
" At least one must be configured. Please see the configuration" \
" documentation."
raise exceptions.SettingError(msg)
if configuration.has_option(self.config_section, 'user'):
username = configuration.get(self.config_section, 'user')
if not validation.valid_user(username):
err_msg = "%s is not a valid account on this system" % username
self.log(err_msg,
section=self.config_section,
option='user',
level=logging.ERROR)
raise exceptions.SettingError(err_msg)
self.gip_user = username
def _parse_configuration(self, configuration):
"""
The meat of parse_configuration, runs after we've checked that GIP is
enabled and we have the right RPMs installed.
"""
# The following is to set the user that gip files need to belong to
# This can be overridden by setting the 'user' option in the [GIP] section
self.gip_user = '******'
if configuration.has_option(self.config_section, 'batch'):
batch_opt = configuration.get(self.config_section, 'batch').lower()
if (not utilities.blank(batch_opt) and
batch_opt not in self._valid_batch_opt):
msg = "The batch setting in %s must be a valid option " \
"(e.g. %s), %s was given" % (self.config_section,
",".join(self._valid_batch_opt),
batch_opt)
self.log(msg, level=logging.ERROR)
raise exceptions.SettingError(msg)
if utilities.ce_installed():
self._parse_configuration_ce(configuration)
# Check for the presence of the classic SE
has_classic_se = True
try:
has_classic_se = configuration.getboolean("GIP", "advertise_gsiftp")
# pylint: disable-msg=W0702
# pylint: disable-msg=W0703
# pylint: disable-msg=W0704
except Exception:
pass
has_se = False
for section in configuration.sections():
if not section.lower().startswith('se'):
continue
has_se = True
self.check_se(configuration, section)
if not has_se and not has_classic_se:
try:
self._check_entry(configuration, "GIP", "se_name", REQUIRED, STRING)
except:
msg = "There is no `SE` section, the old-style SE" + \
"setup in GIP is not configured, and there is no classic SE. " + \
" At least one must be configured. Please see the configuration" \
" documentation."
raise exceptions.SettingError(msg)
if configuration.has_option(self.config_section, 'user'):
username = configuration.get(self.config_section, 'user')
if not validation.valid_user(username):
err_msg = "%s is not a valid account on this system" % username
self.log(err_msg,
section=self.config_section,
option='user',
level=logging.ERROR)
raise exceptions.SettingError(err_msg)
self.gip_user = username
def write_htcondor_ce_sentinel(self):
if self.htcondor_gateway_enabled and utilities.ce_installed():
contents = utilities.read_file(
self.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
contents = utilities.add_or_replace_setting(contents,
"OSG_CONFIGURED",
"true",
quote_value=False)
utilities.atomic_write(self.HTCONDOR_CE_CONFIG_FILE, contents)
def testBadCache(self):
"""
Test the check_attributes function when cache size is not an integer
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_bad_cache.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
self.assertRaises(exceptions.SettingError,
settings.parse_configuration, configuration)
def testBadCache(self):
"""
Test the check_attributes function when cache size is not an integer
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_bad_cache.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
self.assertRaises(exceptions.SettingError,
settings.parse_configuration,
configuration)
def testValidSettings(self):
"""
Test the check_attributes function to see if it oks good attributes
"""
if not ce_installed():
return True
config_file = get_test_config("squid/valid_settings.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testValidSettings(self):
"""
Test the check_attributes function to see if it oks good attributes
"""
if not ce_installed():
return True
config_file = get_test_config("squid/valid_settings.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def get_option(config, section, option):
"""
Get an option from a config file with optional defaults and mandatory
options.
Arguments
config -- a ConfigParser object to query
section -- the ini section the option is located in
option -- an Option object to information on the option to retrieve
"""
is_required_option = (option.required == Option.MANDATORY
or (option.required == Option.MANDATORY_ON_CE
and utilities.ce_installed()))
if config.has_option(section, option.name):
try:
if not utilities.blank(config.get(section, option.name)):
if option.opt_type == bool:
option.value = config.getboolean(section, option.name)
elif option.opt_type == int:
option.value = config.getint(section, option.name)
elif option.opt_type == float:
option.value = config.getfloat(section, option.name)
else:
option.value = config.get(section, option.name)
else:
# if option is blank and there's a default for the option
# return the default if possible, otherwise raise an exception
# if the option is mandatory
if option.default_value is not None:
option.value = option.default_value
elif is_required_option:
raise exceptions.SettingError("Can't get value for %s in %s " \
"section and no default given" % \
(option.name, section))
except ValueError:
error_mesg = "%s in %s section is of the wrong type" % (
option.name, section)
raise exceptions.SettingError(error_mesg)
elif is_required_option:
err_mesg = "Can't get value for %s in %s section" % (option.name,
section)
raise exceptions.SettingError(err_mesg)
else:
option.value = option.default_value
def testBlankLocation(self):
"""
Test the check_attributes function to see if it raises an error when
location is left blank
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_blank_location.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testParsing2(self):
"""
Test storage parsing
"""
# StorageConfiguration is not enabled on non-ce installs
if not utilities.ce_installed():
return
config_file = get_test_config("storage/storage2.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = storage.StorageConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testParsingProductionDefaults(self):
"""
Test infoservices parsing to make sure it picks up production defaults
"""
# need to be on a CE to get CE defaults
if not utilities.ce_installed():
return
config_file = get_test_config("infoservices/prod_defaults.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = infoservices.InfoServicesConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testParsingITBDefaults(self):
"""
Test infoservices parsing to make sure it picks up ITB defaults
"""
# need to be on a CE to get CE defaults
if not utilities.ce_installed():
return
config_file = get_test_config("infoservices/itb_defaults.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = infoservices.InfoServicesConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testValidProductionDefaults(self):
"""
Test the check_attributes function to see if it oks the production defaults
"""
# need to be on a CE to get CE defaults
if not utilities.ce_installed():
return
config_file = get_test_config("infoservices/prod_defaults.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = infoservices.InfoServicesConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testBlankLocation(self):
"""
Test the check_attributes function to see if it raises an error when
location is left blank
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_blank_location.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testBadPort(self):
"""
Test the check_attributes function when port for the squid proxy is
not an integer
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_bad_port.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testBadPort(self):
"""
Test the check_attributes function when port for the squid proxy is
not an integer
"""
if not ce_installed():
return True
config_file = get_test_config("squid/squid_bad_port.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testValidProductionDefaults(self):
"""
Test the check_attributes function to see if it oks the production defaults
"""
# need to be on a CE to get CE defaults
if not utilities.ce_installed():
return
config_file = get_test_config("infoservices/prod_defaults.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = infoservices.InfoServicesConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testParsing1(self):
"""
Test storage parsing
"""
# StorageConfiguration is not enabled on non-ce installs
if not utilities.ce_installed():
return
config_file = get_test_config("storage/storage1.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = storage.StorageConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
attributes = settings.get_attributes()
variables = {
'OSG_STORAGE_ELEMENT': 'True',
'OSG_DEFAULT_SE': 'test.domain.org',
'OSG_GRID': '/tmp',
'OSG_APP': '/tmp',
'OSG_DATA': '/var',
'OSG_WN_TMP': '/usr',
'OSG_SITE_READ': '/bin',
'OSG_SITE_WRITE': '/usr/bin'
}
for var in variables:
self.assertTrue(attributes.has_key(var),
"Attribute %s missing" % var)
self.assertEqual(attributes[var],
variables[var],
"Wrong value obtained for %s, got %s but " \
"expected %s" % (var, attributes[var], variables[var]))
if not validation.valid_directory('/tmp/etc'):
# handle cases where this is not run under osg test framework
os.mkdir('/tmp/etc')
os.chmod('/tmp/etc', 0o777)
self.assertTrue(settings.check_attributes(attributes))
os.rmdir('/tmp/etc')
else:
self.assertTrue(settings.check_attributes(attributes))
def testMissingAttribute(self):
"""
Test the check_attributes function
"""
# StorageConfiguration is not enabled on non-ce installs
if not utilities.ce_installed():
return
mandatory = ['se_available']
for option in mandatory:
config_file = get_test_config("storage/storage1.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
configuration.remove_option('Storage', option)
settings = storage.StorageConfiguration(logger=global_logger)
self.assertRaises(exceptions.SettingError,
settings.parse_configuration, configuration)
def configure(self, attributes):
"""Configure installation using attributes"""
self.log('MiscConfiguration.configure started')
if not self.enabled:
self.log('Not enabled')
self.log('MiscConfiguration.configure completed')
return True
# run fetch-crl script
if not utilities.fetch_crl():
self.log("Error while running fetch-crl script",
level=logging.WARNING)
if not utilities.crls_exist():
self.log(
"No CRLs exist and Fetch-CRL couldn't download any. GSI authentication is likely to fail. To attempt another download, run 'fetch-crl'.",
level=logging.WARNING)
if self.authorization_method == 'gridmap':
self._set_lcmaps_callout(False)
elif self.authorization_method == 'local-gridmap':
self.log(
"local-gridmap is a deprecated auth method -- use 'gridmap' instead",
level=logging.WARNING)
self._set_lcmaps_callout(False)
elif self.authorization_method == 'vomsmap':
self._set_lcmaps_callout(True)
else:
assert False, "Invalid authorization_method should have been caught in check_attributes()"
if self.options['edit_lcmaps_db'].value:
self._write_lcmaps_file()
else:
self.log("Not updating lcmaps.db because edit_lcmaps_db is false",
level=logging.DEBUG)
if self.htcondor_gateway_enabled and utilities.ce_installed():
self.write_gridmap_to_htcondor_ce_config()
self.log('MiscConfiguration.configure completed')
return True
def testMissingAttribute(self):
"""
Test the check_attributes function
"""
# StorageConfiguration is not enabled on non-ce installs
if not utilities.ce_installed():
return
mandatory = ['se_available']
for option in mandatory:
config_file = get_test_config("storage/storage1.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
configuration.remove_option('Storage', option)
settings = storage.StorageConfiguration(logger=global_logger)
self.assertRaises(exceptions.SettingError,
settings.parse_configuration,
configuration)
def testValidSettings(self):
"""
Test the check_attributes function to see if it oks good attributes
"""
if not ce_installed():
return True
config_file = get_test_config("squid/valid_settings.ini")
configuration = configparser.SafeConfigParser()
configuration.read(config_file)
settings = squid.SquidConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
attributes = settings.get_attributes()
self.assertTrue(settings.check_attributes(attributes),
"Correct locations incorrectly flagged as missing")
def configure(self, attributes):
"""Configure installation using attributes"""
self.log('MiscConfiguration.configure started')
if not self.enabled:
self.log('Not enabled')
self.log('MiscConfiguration.configure completed')
return True
# run fetch-crl script
if not utilities.fetch_crl():
self.log("Error while running fetch-crl script",
level=logging.ERROR)
raise exceptions.ConfigureError(
'fetch-crl returned non-zero exit code')
if self.authorization_method == 'xacml':
self._set_lcmaps_callout(True)
self.update_gums_client_location()
elif self.authorization_method == 'gridmap':
self._set_lcmaps_callout(False)
elif self.authorization_method == 'local-gridmap':
self._set_lcmaps_callout(False)
elif self.authorization_method == 'vomsmap':
self._set_lcmaps_callout(True)
else:
assert False, "Invalid authorization_method should have been caught in check_attributes()"
if self.options['edit_lcmaps_db'].value:
self._write_lcmaps_file()
else:
self.log("Not updating lcmaps.db because edit_lcmaps_db is false",
level=logging.DEBUG)
if self.htcondor_gateway_enabled and utilities.ce_installed():
self.write_gridmap_to_htcondor_ce_config()
self.log('MiscConfiguration.configure completed')
return True
def parse_configuration(self, configuration):
"""
Try to get configuration information from ConfigParser or SafeConfigParser object given
by configuration and write recognized settings to attributes dict
"""
self.log('InfoServicesConfiguration.parse_configuration started')
self.check_config(configuration)
if not configuration.has_section(self.config_section):
self.enabled = False
self.log("%s section not in config file" % self.config_section)
self.log('InfoServicesConfiguration.parse_configuration completed')
return
if not self.set_status(configuration):
self.log('InfoServicesConfiguration.parse_configuration completed')
return True
self._set_default_servers(configuration)
self.get_options(configuration,
ignore_options=[
'itb-ress-servers', 'itb-bdii-servers',
'osg-ress-servers', 'osg-bdii-servers',
'ress_servers', 'enabled', 'bdii_servers'
])
self.ce_collectors = self._parse_ce_collectors(
self.options['ce_collectors'].value)
self.misc_module.parse_configuration(configuration)
def csg(section, option):
return utilities.config_safe_get(configuration, section, option,
None)
def csgbool(section, option):
return utilities.config_safe_getboolean(configuration, section,
option, False)
# We get some values for HTCondor-CE from the Site Information section
self.osg_resource = csg('Site Information', 'resource')
self.osg_resource_group = csg('Site Information', 'resource_group')
# and the enabled batch systems from their respective sections
self.enabled_batch_systems = [
bs for bs in BATCH_SYSTEMS if csgbool(bs, 'enabled')
]
self.htcondor_gateway_enabled = csgbool('Gateway',
'htcondor_gateway_enabled')
self.authorization_method = csg('Misc Services',
'authorization_method')
self.subcluster_sections = ConfigParser.SafeConfigParser()
self.gums_host = csg('Misc Services', 'gums_host')
for section in configuration.sections():
if section.lower().startswith('subcluster') or section.lower(
).startswith('resource entry'):
self.subcluster_sections.add_section(section)
for key, value in configuration.items(section):
self.subcluster_sections.set(section, key, value)
if utilities.ce_installed(
) and not subcluster.check_config(configuration):
self.log(
"On a CE but no valid 'Subcluster' or 'Resource Entry' sections defined."
" This is required to advertise the capabilities of your cluster to the central collector."
" Jobs may not be sent to this CE.",
level=logging.ERROR)
raise exceptions.SettingError(
"No Subcluster or Resource Entry sections")
# Check resource catalog
# This is a bit clunky to parse it here and not use the result in
# configure(), but at this point we don't have a way of knowing what
# default_allowed_vos should be.
if self.ce_collector_required_rpms_installed and self.htcondor_gateway_enabled and classad is not None:
subcluster.resource_catalog_from_config(self.subcluster_sections,
default_allowed_vos=None)
self.log('InfoServicesConfiguration.parse_configuration completed')
