def parse_configuration(self, configuration):
"""
Try to get configuration information from ConfigParser or SafeConfigParser
object given by configuration and write recognized settings to attributes
dict
"""
self.log('GipConfiguration.parse_configuration started')
if not utilities.rpm_installed('gip'):
self.log('GIP not installed, disabling GIP')
self.log('GipConfiguration.parse_configuration completed')
self.enabled = False
return
else:
self.enabled = True
# TODO Do we want to force the user to have a GIP section? Uncomment the following if not.
# if not configuration.has_section(self.config_section):
# self.log("%s section not in config file" % self.config_section)
# self.log('GipConfiguration.parse_configuration completed')
# self.enabled = False
# return
self.check_config(configuration)
self._parse_configuration(configuration)
self.log('GipConfiguration.parse_configuration completed')
def testParsing2(self):
"""
Test rsv parsing
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv2.ini")
options = settings.options
variables = {'gratia_probes': '',
'ce_hosts': 'my.host.com, my2.host.com',
'gridftp_hosts': 'my.host.com, my2.host.com',
'gridftp_dir': '/tmp',
'gums_hosts': 'my.host.com, my2.host.com',
'srm_hosts': 'my.host.com, my2.host.com',
'srm_dir': '/srm/dir, /srm/dir2',
'srm_webservice_path': 'srm/v2/server,',
'service_cert': '/etc/redhat-release',
'service_key': '/etc/redhat-release',
'service_proxy': '/tmp/rsvproxy',
'enable_gratia': False,
'enable_nagios': False}
for var in variables:
self.assertTrue(options.has_key(var),
"Option %s missing" % var)
self.assertEqual(options[var].value,
variables[var],
"Wrong value obtained for %s, got %s but " \
"expected %s" % (var, options[var].value, variables[var]))
def testMultipleHosts(self):
"""
Test rsv parsing
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/multiple_hosts.ini")
options = settings.options
variables = {
'ce_hosts': 'host1.site.com, host2.site.com',
'gridftp_hosts': 'host3.site.com, host4.site.com',
'gridftp_dir': '/tmp',
'gums_hosts': 'host8.site.com, host9.site.com',
'srm_hosts': 'host5.site.com, host6.site.com, host7.site.com:1234',
'srm_dir': '/srm/dir',
'srm_webservice_path': 'srm/v2/server',
'service_cert': './configs/rsv/rsv1.ini',
'service_key': './configs/rsv/rsv1.ini',
'service_proxy': '/tmp/rsvproxy',
'enable_gratia': True,
'enable_nagios': True
}
for var in variables:
self.assertTrue(options.has_key(var), "Option %s missing" % var)
self.assertEqual(options[var].value,
variables[var],
"Wrong value obtained for %s, got %s but " \
"expected %s" % (var, options[var].value, variables[var]))
def __init__(self, *args, **kwargs):
# pylint: disable-msg=W0142
super(InfoServicesConfiguration, self).__init__(*args, **kwargs)
self.logger = logging.getLogger(__name__)
self.log("InfoServicesConfiguration.__init__ started")
self.config_section = 'Info Services'
self.options = {
'ce_collectors':
configfile.Option(name='ce_collectors',
default_value='',
required=configfile.Option.OPTIONAL)
}
self._itb_default_ce_collectors = \
'collector-itb.opensciencegrid.org:%d' % HTCONDOR_CE_COLLECTOR_PORT
self._production_default_ce_collectors = \
'collector1.opensciencegrid.org:%d,collector2.opensciencegrid.org:%d' % (
HTCONDOR_CE_COLLECTOR_PORT, HTCONDOR_CE_COLLECTOR_PORT)
self.ce_collectors = []
self.ce_collector_required_rpms_installed = utilities.rpm_installed(
'htcondor-ce')
self.osg_resource = ""
self.osg_resource_group = ""
self.enabled_batch_systems = []
self.htcondor_gateway_enabled = None
self.resource_catalog = None
self.authorization_method = None
self.subcluster_sections = None
self.gums_host = None
self.misc_module = MiscConfiguration(*args, **kwargs)
self.log("InfoServicesConfiguration.__init__ completed")
def testServiceList(self):
"""
Test to make sure right services get returned
"""
# If rsv-core is not installed, settings.enabled will always be False
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv1.ini")
services = settings.enabled_services()
expected_services = set(['rsv', 'condor-cron'])
self.assertEqual(
services, expected_services,
"List of enabled services incorrect, " + "got %s but expected %s" %
(services, expected_services))
settings = self.load_settings_from_files("rsv/disabled.ini")
services = settings.enabled_services()
expected_services = set()
self.assertEqual(
services, expected_services,
"List of enabled services incorrect, " + "got %s but expected %s" %
(services, expected_services))
settings = self.load_settings_from_files("rsv/ignored.ini")
services = settings.enabled_services()
expected_services = set()
self.assertEqual(
services, expected_services,
"List of enabled services incorrect, " + "got %s but expected %s" %
(services, expected_services))
def testMultipleHosts(self):
"""
Test rsv parsing
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/multiple_hosts.ini")
options = settings.options
variables = {'ce_hosts': 'host1.site.com, host2.site.com',
'gridftp_hosts': 'host3.site.com, host4.site.com',
'gridftp_dir': '/tmp',
'gums_hosts': 'host8.site.com, host9.site.com',
'srm_hosts': 'host5.site.com, host6.site.com, host7.site.com:1234',
'srm_dir': '/srm/dir',
'srm_webservice_path': 'srm/v2/server',
'service_cert': './configs/rsv/rsv1.ini',
'service_key': './configs/rsv/rsv1.ini',
'service_proxy': '/tmp/rsvproxy',
'enable_gratia': True,
'enable_nagios': True}
for var in variables:
self.assertTrue(options.has_key(var),
"Option %s missing" % var)
self.assertEqual(options[var].value,
variables[var],
"Wrong value obtained for %s, got %s but " \
"expected %s" % (var, options[var].value, variables[var]))
def testParsing2(self):
"""
Test rsv parsing
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv2.ini")
options = settings.options
variables = {
'gratia_probes': '',
'ce_hosts': 'my.host.com, my2.host.com',
'gridftp_hosts': 'my.host.com, my2.host.com',
'gridftp_dir': '/tmp',
'gums_hosts': 'my.host.com, my2.host.com',
'srm_hosts': 'my.host.com, my2.host.com',
'srm_dir': '/srm/dir, /srm/dir2',
'srm_webservice_path': 'srm/v2/server,',
'service_cert': '/etc/redhat-release',
'service_key': '/etc/redhat-release',
'service_proxy': '/tmp/rsvproxy',
'enable_gratia': False,
'enable_nagios': False
}
for var in variables:
self.assertTrue(options.has_key(var), "Option %s missing" % var)
self.assertEqual(options[var].value,
variables[var],
"Wrong value obtained for %s, got %s but " \
"expected %s" % (var, options[var].value, variables[var]))
def testServiceList(self):
"""
Test to make sure right services get returned
"""
# If rsv-core is not installed, settings.enabled will always be False
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv1.ini")
services = settings.enabled_services()
expected_services = set(['rsv', 'condor-cron'])
self.assertEqual(services, expected_services,
"List of enabled services incorrect, " +
"got %s but expected %s" % (services, expected_services))
settings = self.load_settings_from_files("rsv/disabled.ini")
services = settings.enabled_services()
expected_services = set()
self.assertEqual(services, expected_services,
"List of enabled services incorrect, " +
"got %s but expected %s" % (services, expected_services))
settings = self.load_settings_from_files("rsv/ignored.ini")
services = settings.enabled_services()
expected_services = set()
self.assertEqual(services, expected_services,
"List of enabled services incorrect, " +
"got %s but expected %s" % (services, expected_services))
def parse_configuration(self, configuration):
"""
Try to get configuration information from ConfigParser or SafeConfigParser
object given by configuration and write recognized settings to attributes
dict
"""
self.log('GipConfiguration.parse_configuration started')
if not utilities.rpm_installed('gip'):
self.log('GIP not installed, disabling GIP')
self.log('GipConfiguration.parse_configuration completed')
self.enabled = False
return
else:
self.enabled = True
# TODO Do we want to force the user to have a GIP section? Uncomment the following if not.
# if not configuration.has_section(self.config_section):
# self.log("%s section not in config file" % self.config_section)
# self.log('GipConfiguration.parse_configuration completed')
# self.enabled = False
# return
self.check_config(configuration)
self._parse_configuration(configuration)
self.log('GipConfiguration.parse_configuration completed')
def setup_htcondor_ce_config(self):
"""
Populate the config file that tells htcondor-ce where the condor
pool is and where the spool directory is.
Returns True if successful, False otherwise
"""
if not utilities.rpm_installed('htcondor-ce'):
self.log("Unable to configure htcondor-ce for Condor: htcondor-ce not installed", level=logging.ERROR)
return False
def get_condor_ce_config_val(variable):
return utilities.get_condor_config_val(variable, executable='condor_ce_config_val', quiet_undefined=True)
# Get values for the settings we want to update. We can get the
# values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME,
# we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing.
# We also get the current / default value from condor_ce_config_val;
# only update the setting in case the value from
# condor_config_val is different from the value from condor_ce_config_val.
condor_ce_config = {}
for condor_ce_config_key, condor_config_keys in [
('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']),
('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']),
('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])]:
condor_config_value = None
for condor_config_value in (utilities.get_condor_config_val(k, quiet_undefined=True) for k in
condor_config_keys):
if condor_config_value:
break
condor_ce_config_value = get_condor_ce_config_val(condor_ce_config_key)
if not (condor_config_value or condor_ce_config_value):
self.log("Unable to determine value for %s from %s and default not set; check your Condor config" %
(condor_ce_config_key, ' or '.join(condor_config_keys)), level=logging.ERROR)
return False
elif not condor_config_value:
continue # can't set anything for this
# Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744)
if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL':
condor_collector_port = (utilities.get_condor_config_val('COLLECTOR_PORT', quiet_undefined=True)
or '9618')
condor_config_value = self._add_port_if_necessary(condor_config_value, condor_collector_port)
if not condor_ce_config_value or condor_ce_config_value != condor_config_value:
condor_ce_config[condor_ce_config_key] = condor_config_value
if condor_ce_config:
buf = utilities.read_file(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
for key, value in condor_ce_config.items():
buf = utilities.add_or_replace_setting(buf, key, value, quote_value=False)
if not utilities.atomic_write(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf):
return False
return True
def test_rpm_install(self):
"""
Test rpm_installed function
"""
rpm_name = 'foo'
self.assertFalse(utilities.rpm_installed(rpm_name),
'foo is not installed, but rpm_installed returned True')
rpm_name = 'filesystem'
self.assertTrue(utilities.rpm_installed(rpm_name),
'filesystem is installed, but rpm_installed returned False')
rpm_names = ['filesystem', 'foo']
self.assertFalse(utilities.rpm_installed(rpm_names),
'foo is not installed, but rpm_installed returned True')
rpm_names = ['filesystem', 'glibc']
self.assertTrue(utilities.rpm_installed(rpm_names),
'filesystem and glibc are installed, but rpm_installed returned False')
def test_rpm_install(self):
"""
Test rpm_installed function
"""
rpm_name = 'foo'
self.assertFalse(utilities.rpm_installed(rpm_name),
'foo is not installed, but rpm_installed returned True')
rpm_name = 'filesystem'
self.assertTrue(utilities.rpm_installed(rpm_name),
'filesystem is installed, but rpm_installed returned False')
rpm_names = ['filesystem', 'foo']
self.assertFalse(utilities.rpm_installed(rpm_names),
'foo is not installed, but rpm_installed returned True')
rpm_names = ['filesystem', 'glibc']
self.assertTrue(utilities.rpm_installed(rpm_names),
'filesystem and glibc are installed, but rpm_installed returned False')
def testServiceList(self):
"""
Test to make sure right services get returned
"""
config_file = get_test_config("misc/misc_xacml.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = misc.MiscConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
services = settings.enabled_services()
if utilities.rpm_installed('fetch-crl'):
expected_services = set(
['fetch-crl-cron', 'fetch-crl-boot', 'gums-client-cron'])
else:
expected_services = set(['gums-client-cron'])
self.assertEqual(
services, expected_services,
"List of enabled services incorrect, " + "got %s but expected %s" %
(services, expected_services))
config_file = get_test_config("misc/misc_gridmap.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = misc.MiscConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception as e:
self.fail("Received exception while parsing configuration: %s" % e)
services = settings.enabled_services()
if utilities.rpm_installed('fetch-crl'):
expected_services = set(
['fetch-crl-cron', 'fetch-crl-boot', 'edg-mkgridmap'])
else:
expected_services = set(['edg-mkgridmap'])
self.assertEqual(
services, expected_services,
"List of enabled services incorrect, " + "got %s but expected %s" %
(services, expected_services))
def __init__(self, *args, **kwargs):
# pylint: disable-msg=W0142
super(InfoServicesConfiguration, self).__init__(*args, **kwargs)
self.log("InfoServicesConfiguration.__init__ started")
# file location for xml file with info services subscriptions
self.config_section = 'Info Services'
self.options = {
'bdii_servers':
configfile.Option(name='bdii_servers', default_value=''),
'ce_collectors':
configfile.Option(name='ce_collectors',
default_value='',
required=configfile.Option.OPTIONAL)
}
self._itb_defaults = {
'bdii_servers':
'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
'ce_collectors':
'collector-itb.opensciencegrid.org:%d' % HTCONDOR_CE_COLLECTOR_PORT
}
self._production_defaults = {
'bdii_servers':
'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
'ce_collectors':
'collector1.opensciencegrid.org:%d,collector2.opensciencegrid.org:%d'
% (HTCONDOR_CE_COLLECTOR_PORT, HTCONDOR_CE_COLLECTOR_PORT)
}
self.bdii_servers = {}
self.copy_host_cert_for_service_cert = False
self.ois_required_rpms_installed = utilities.gateway_installed(
) and utilities.rpm_installed('osg-info-services')
# for htcondor-ce-info-services:
self.ce_collectors = []
self.ce_collector_required_rpms_installed = utilities.rpm_installed(
'htcondor-ce')
self.osg_resource = ""
self.osg_resource_group = ""
self.enabled_batch_systems = []
self.htcondor_gateway_enabled = None
self.resource_catalog = None
self.authorization_method = None
self.subcluster_sections = None
self.log("InfoServicesConfiguration.__init__ completed")
def testValidSettings2(self):
"""
Test the check_attributes function to see if it works on valid settings
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv2.ini")
attributes = settings.get_attributes()
self.assertTrue(settings.check_attributes(attributes),
"Correct configuration incorrectly flagged as incorrect")
def enabled_services(self):
"""Return a list of system services needed for module to work
"""
if not self.enabled or self.ignored:
return set()
services = set()
if utilities.rpm_installed('fetch-crl'):
services = set(['fetch-crl-cron', 'fetch-crl-boot'])
elif utilities.rpm_installed('fetch-crl3'):
services = set(['fetch-crl3-cron', 'fetch-crl3-boot'])
if self.options['authorization_method'].value == 'xacml':
services.add('gums-client-cron')
elif self.options['authorization_method'].value == 'gridmap':
services.add('edg-mkgridmap')
if self.options['enable_cleanup'].value:
services.add('osg-cleanup-cron')
return services
def testInvalidSRMHost(self):
"""
Test the check_attributes with invalid srm host
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/invalid_srm_host.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Invalid srm ignored")
def enabled_services(self):
"""Return a list of system services needed for module to work
"""
if not self.enabled or self.ignored:
return set()
services = set()
if utilities.rpm_installed('fetch-crl'):
services = {'fetch-crl-cron', 'fetch-crl-boot'}
return services
def testValidSettingsIPv6(self):
"""
Test RSV config accepts IPv6 hostnames
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv_ipv6.ini")
attributes = settings.get_attributes()
self.assertTrue(settings.check_attributes(attributes),
"Correct configuration incorrectly flagged as incorrect")
def testMissingProxy(self):
"""
Test the check_attributes with a missing proxy cert file
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/missing_proxy.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Missing rsv proxy file ignored")
def testInvalidSRMHost(self):
"""
Test the check_attributes with invalid srm host
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/invalid_srm_host.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Invalid srm ignored")
def testMissingProxy(self):
"""
Test the check_attributes with a missing proxy cert file
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/missing_proxy.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Missing rsv proxy file ignored")
def testParsingDisabled(self):
"""
Test parsing when disabled
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/disabled.ini")
attributes = settings.get_attributes()
self.assertEqual(len(attributes), 0,
"Disabled configuration should have no attributes")
def testParsingDisabled(self):
"""
Test parsing when disabled
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/disabled.ini")
attributes = settings.get_attributes()
self.assertEqual(len(attributes), 0,
"Disabled configuration should have no attributes")
def testValidSettings2(self):
"""
Test the check_attributes function to see if it works on valid settings
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv2.ini")
attributes = settings.get_attributes()
self.assertTrue(
settings.check_attributes(attributes),
"Correct configuration incorrectly flagged as incorrect")
def testValidSettingsIPv6(self):
"""
Test RSV config accepts IPv6 hostnames
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/rsv_ipv6.ini")
attributes = settings.get_attributes()
self.assertTrue(
settings.check_attributes(attributes),
"Correct configuration incorrectly flagged as incorrect")
def __init__(self, *args, **kwargs):
# pylint: disable-msg=W0142
super(InfoServicesConfiguration, self).__init__(*args, **kwargs)
self.log("InfoServicesConfiguration.__init__ started")
# file location for xml file with info services subscriptions
self.config_section = 'Info Services'
self.options = {'bdii_servers': configfile.Option(name='bdii_servers',
default_value=''),
'ce_collectors': configfile.Option(name='ce_collectors',
default_value='',
required=configfile.Option.OPTIONAL)}
self._itb_defaults = {
'bdii_servers': 'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
'ce_collectors': 'collector-itb.opensciencegrid.org:%d' % HTCONDOR_CE_COLLECTOR_PORT}
self._production_defaults = {
'bdii_servers': 'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
'ce_collectors': 'collector1.opensciencegrid.org:%d,collector2.opensciencegrid.org:%d' % (
HTCONDOR_CE_COLLECTOR_PORT, HTCONDOR_CE_COLLECTOR_PORT)}
self.bdii_servers = {}
self.copy_host_cert_for_service_cert = False
self.ois_required_rpms_installed = utilities.gateway_installed() and utilities.rpm_installed(
'osg-info-services')
# for htcondor-ce-info-services:
self.ce_collectors = []
self.ce_collector_required_rpms_installed = utilities.rpm_installed('htcondor-ce')
self.osg_resource = ""
self.osg_resource_group = ""
self.enabled_batch_systems = []
self.htcondor_gateway_enabled = None
self.resource_catalog = None
self.authorization_method = None
self.subcluster_sections = None
self.log("InfoServicesConfiguration.__init__ completed")
def testInvalidGratiaProbes(self):
"""
Test the check_attributes with invalid gratia probes
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/invalid_gratia1.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Invalid gratia probe ignored")
settings = self.load_settings_from_files("rsv/invalid_gratia2.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Invalid gratia probe list ignored")
def testInvalidGratiaProbes(self):
"""
Test the check_attributes with invalid gratia probes
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
settings = self.load_settings_from_files("rsv/invalid_gratia1.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Invalid gratia probe ignored")
settings = self.load_settings_from_files("rsv/invalid_gratia2.ini")
attributes = settings.get_attributes()
self.assertFalse(settings.check_attributes(attributes),
"Invalid gratia probe list ignored")
def testMissingAttribute(self):
"""
Test the parsing when attributes are missing, should get exceptions
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
config_file = get_test_config("rsv/rsv2.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = rsv.RsvConfiguration(logger=global_logger)
settings.rsv_meta_dir = RSV_META_DIR
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def testMissingAttribute(self):
"""
Test the parsing when attributes are missing, should get exceptions
"""
# need to have rsv installed to get rsv tests working
if not utilities.rpm_installed('rsv-core'):
return
config_file = get_test_config("rsv/rsv2.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = rsv.RsvConfiguration(logger=global_logger)
settings.rsv_meta_dir = RSV_META_DIR
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
def _configure_consumers(self):
""" Enable the appropriate consumers """
# The current logic is:
# - we ALWAYS want the html-consumer if we are told to install consumers
# - we NEVER want the gratia-consumer
# - we want the nagios-consumer if enable_nagios is True
# - we want the zabbix-consumer if enable_zabbix is True and rsv-consumers-zabbix is installed
consumers = ["html-consumer"]
if self.opt_val("enable_gratia"):
self.log(
"Your configuration has enabled the Gratia consumer but the service which the Gratia consumer "
"reports to has been shut down. Please turn off 'enable_gratia' in the RSV section. "
"Gratia consumer configuration will be ignored.",
level=logging.WARNING)
if self.options['enable_nagios'].value:
consumers.append("nagios-consumer")
self._configure_nagios_files()
if self.options['enable_zabbix'].value:
if not utilities.rpm_installed('rsv-consumers-zabbix'):
self.log(
'Your configuration has enabled the Zabbix consumer '
'but rsv-consumers-zabbix is not installed. Zabbix consumer configuration will be ignored.',
level=logging.WARNING)
else:
consumers.append("zabbix-consumer")
self._configure_zabbix_files()
consumer_list = " ".join(consumers)
self.log("Enabling consumers: %s " % consumer_list)
if not utilities.run_script([self.rsv_control, "-v0", "--enable"] +
consumers):
raise exceptions.ConfigureError
utilities.run_script(
[self.rsv_control, "-v0", "--disable",
"gratia-consumer"]) # don't care if this fails
def parse_configuration(self, configuration):
"""
Try to get configuration information from ConfigParser or
SafeConfigParser object given by configuration and write recognized settings
to attributes dict
"""
self.log('RsvConfiguration.parse_configuration started')
self.check_config(configuration)
if not configuration.has_section(self.config_section):
self.enabled = False
self.log("%s section not in config file" % self.config_section)
self.log('RsvConfiguration.parse_configuration completed')
return True
if not utilities.rpm_installed('rsv-core'):
self.enabled = False
self.log('rsv-core rpm not installed, disabling RSV configuration')
return True
if not self.set_status(configuration):
self.log('RsvConfiguration.parse_configuration completed')
return True
self.get_options(configuration,
ignore_options=['enabled', 'gratia_collector'])
# If we're on a CE, get the grid group if possible
if configuration.has_section('Site Information'):
if configuration.has_option('Site Information', 'group'):
self.grid_group = configuration.get('Site Information',
'group')
if configuration.has_option('Site Information', 'resource'):
self.site_name = configuration.get('Site Information',
'resource')
elif configuration.has_option('Site Information', 'site_name'):
self.site_name = configuration.get('Site Information',
'site_name')
# Parse lists
self._ce_hosts = split_list_exclude_blank(
self.options['ce_hosts'].value)
self._htcondor_ce_hosts = split_list_exclude_blank(
self.options['htcondor_ce_hosts'].value)
self._srm_hosts = split_list_exclude_blank(
self.options['srm_hosts'].value)
# If the gridftp hosts are not defined then they default to the CE hosts
if self.options['gridftp_hosts'].value == '':
# check to see if the setting is in the config file
if configuration.has_option(self.config_section, 'gridftp_hosts'):
# present and set to default so we don't want gridftp tests
self._gridftp_hosts = []
else:
# option is commented out, use ce_hosts setting
self._gridftp_hosts = self._ce_hosts
else:
self._gridftp_hosts = split_list(
self.options['gridftp_hosts'].value)
if self.options['gratia_probes'].value != '':
self._gratia_probes_2d = self.split_2d_list(
self.options['gratia_probes'].value)
# Check the options for which gateways are enabled
# How we run remote probes depends on this
if configuration.has_section('Gateway'):
if configuration.has_option('Gateway', 'htcondor_gateway_enabled'):
self.htcondor_gateway_enabled = configuration.getboolean(
'Gateway', 'htcondor_gateway_enabled')
if configuration.has_section('Misc Services'):
if configuration.has_option('Misc Services',
'copy_host_cert_for_service_certs'):
self.copy_host_cert_for_service_cert = configuration.getboolean(
'Misc Services', 'copy_host_cert_for_service_certs')
self.log('RsvConfiguration.parse_configuration completed')
def check_attributes(self, attributes):
"""Check attributes currently stored and make sure that they are consistent"""
self.log("SquidConfiguration.check_attributes started")
attributes_ok = True
if not (utilities.gateway_installed() and utilities.rpm_installed("frontier-squid")):
return attributes_ok
if not self.enabled:
self.log(
"Squid is not enabled, sites must enable this \n"
+ "section, location can be set to UNAVAILABLE if squid is \n"
+ "not present",
level=logging.WARNING,
)
self.log("squid not enabled")
self.log("SquidConfiguration.check_attributes completed")
return attributes_ok
if self.ignored:
self.log("Ignored, returning True")
self.log("SquidConfiguration.check_attributes completed")
return attributes_ok
if self.options["location"].value == "None":
self.log(
"location setting must be set, if site does not provide " + "squid, please use UNAVAILABLE",
section=self.config_section,
option="location",
level=logging.WARNING,
)
return attributes_ok
if self.options["location"].value.upper() == "UNAVAILABLE":
self.log(
"Squid location is set to UNAVAILABLE. Although this is \n"
+ "allowed, most jobs function better and use less bandwidth \n"
+ "when a squid proxy is available",
level=logging.WARN,
)
self.options["location"].value = "None"
return attributes_ok
if len(self.options["location"].value.split(":")) != 2:
self.log(
"Bad host specification, got %s expected hostname:port "
"(e.g. localhost:3128)" % self.options["location"].value,
section=self.config_section,
option="location",
level=logging.ERROR,
)
attributes_ok = False
return attributes_ok
(hostname, port) = self.options["location"].value.split(":")
if not validation.valid_domain(hostname, True):
self.log(
"Invalid hostname for squid location: %s" % self.options["location"].value,
section=self.config_section,
option="location",
level=logging.ERROR,
)
attributes_ok = False
try:
int(port)
except ValueError:
self.log(
"The port must be a number(e.g. host:3128) for squid " "location: %s" % self.options["location"].value,
section=self.config_section,
option="location",
level=logging.ERROR,
exception=True,
)
attributes_ok = False
self.log("SquidConfiguration.check_attributes completed")
return attributes_ok
def setup_htcondor_ce_config(self):
"""
Populate the config file that tells htcondor-ce where the condor
pool is and where the spool directory is.
Returns True if successful, False otherwise
"""
if not utilities.rpm_installed('htcondor-ce'):
self.log(
"Unable to configure htcondor-ce for Condor: htcondor-ce not installed",
level=logging.ERROR)
return False
def get_condor_ce_config_val(variable):
return utilities.get_condor_config_val(
variable,
executable='condor_ce_config_val',
quiet_undefined=True)
# Get values for the settings we want to update. We can get the
# values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME,
# we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing.
# We also get the current / default value from condor_ce_config_val;
# only update the setting in case the value from
# condor_config_val is different from the value from condor_ce_config_val.
condor_ce_config = {}
for condor_ce_config_key, condor_config_keys in [
('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']),
('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']),
('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])
]:
condor_config_value = None
for condor_config_value in (utilities.get_condor_config_val(
k, quiet_undefined=True) for k in condor_config_keys):
if condor_config_value:
break
condor_ce_config_value = get_condor_ce_config_val(
condor_ce_config_key)
if not (condor_config_value or condor_ce_config_value):
self.log(
"Unable to determine value for %s from %s and default not set; check your Condor config"
% (condor_ce_config_key, ' or '.join(condor_config_keys)),
level=logging.ERROR)
return False
elif not condor_config_value:
continue # can't set anything for this
# Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744)
if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL':
condor_collector_port = (utilities.get_condor_config_val(
'COLLECTOR_PORT', quiet_undefined=True) or '9618')
condor_config_value = self._add_port_if_necessary(
condor_config_value, condor_collector_port)
if not condor_ce_config_value or condor_ce_config_value != condor_config_value:
condor_ce_config[condor_ce_config_key] = condor_config_value
if condor_ce_config:
buf = utilities.read_file(
JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE,
default="# This file is managed by osg-configure\n")
for key, value in condor_ce_config.items():
buf = utilities.add_or_replace_setting(buf,
key,
value,
quote_value=False)
if not utilities.atomic_write(
JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf):
return False
return True
def testServiceList(self):
"""
Test to make sure right services get returned
"""
config_file = get_test_config("misc/misc_xacml.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = misc.MiscConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
services = settings.enabled_services()
if utilities.rpm_installed('fetch-crl'):
expected_services = set(['fetch-crl-cron',
'fetch-crl-boot',
'gums-client-cron'])
elif utilities.rpm_installed('fetch-crl3'):
expected_services = set(['fetch-crl3-cron',
'fetch-crl3-boot',
'gums-client-cron'])
else:
expected_services = set(['gums-client-cron'])
self.assertEqual(services, expected_services,
"List of enabled services incorrect, " +
"got %s but expected %s" % (services, expected_services))
config_file = get_test_config("misc/misc_gridmap.ini")
def check_attributes(self, attributes):
"""Check attributes currently stored and make sure that they are consistent"""
self.log('SquidConfiguration.check_attributes started')
attributes_ok = True
if not (utilities.gateway_installed()
and utilities.rpm_installed('frontier-squid')):
return attributes_ok
if not self.enabled:
self.log(
"Squid is not enabled, sites must enable this \n" +
"section, location can be set to UNAVAILABLE if squid is \n" +
"not present",
level=logging.WARNING)
self.log('squid not enabled')
self.log('SquidConfiguration.check_attributes completed')
return attributes_ok
if self.ignored:
self.log('Ignored, returning True')
self.log('SquidConfiguration.check_attributes completed')
return attributes_ok
if (self.options['location'].value == 'None'):
self.log(
"location setting must be set, if site does not provide " +
"squid, please use UNAVAILABLE",
section=self.config_section,
option='location',
level=logging.WARNING)
return attributes_ok
if (self.options['location'].value.upper() == 'UNAVAILABLE'):
self.log(
"Squid location is set to UNAVAILABLE. Although this is \n" +
"allowed, most jobs function better and use less bandwidth \n"
+ "when a squid proxy is available",
level=logging.WARN)
self.options['location'].value = 'None'
return attributes_ok
if len(self.options['location'].value.split(':')) != 2:
self.log("Bad host specification, got %s expected hostname:port " \
"(e.g. localhost:3128)" % self.options['location'].value,
section=self.config_section,
option='location',
level=logging.ERROR)
attributes_ok = False
return attributes_ok
(hostname, port) = self.options['location'].value.split(':')
if not validation.valid_domain(hostname, True):
self.log("Invalid hostname for squid location: %s" % \
self.options['location'].value,
section=self.config_section,
option='location',
level=logging.ERROR)
attributes_ok = False
try:
int(port)
except ValueError:
self.log("The port must be a number(e.g. host:3128) for squid " \
"location: %s" % self.options['location'].value,
section=self.config_section,
option='location',
level=logging.ERROR,
exception=True)
attributes_ok = False
self.log('SquidConfiguration.check_attributes completed')
return attributes_ok
def testServiceList(self):
"""
Test to make sure right services get returned
"""
config_file = get_test_config("misc/misc_xacml.ini")
configuration = ConfigParser.SafeConfigParser()
configuration.read(config_file)
settings = misc.MiscConfiguration(logger=global_logger)
try:
settings.parse_configuration(configuration)
except Exception, e:
self.fail("Received exception while parsing configuration: %s" % e)
services = settings.enabled_services()
if utilities.rpm_installed('fetch-crl'):
expected_services = set(
['fetch-crl-cron', 'fetch-crl-boot', 'gums-client-cron'])
elif utilities.rpm_installed('fetch-crl3'):
expected_services = set(
['fetch-crl3-cron', 'fetch-crl3-boot', 'gums-client-cron'])
else:
expected_services = set(['gums-client-cron'])
self.assertEqual(
services, expected_services,
"List of enabled services incorrect, " + "got %s but expected %s" %
(services, expected_services))
config_file = get_test_config("misc/misc_gridmap.ini")
configuration = ConfigParser.SafeConfigParser()
