def get_condor_location(configuration):
"""
Get the condor location based on the information in a configParser
object (configuration argument) and environment variables if possible
"""
location = configfile.Option(name='condor_location',
default_value=utilities.get_condor_location())
configfile.get_option(configuration, 'Condor', location)
return location.value
def get_condor_location(configuration):
"""
Get the condor location based on the information in a configParser
object (configuration argument) and environment variables if possible
"""
location = configfile.Option(
name='condor_location',
default_value=utilities.get_condor_location())
configfile.get_option(configuration, 'Condor', location)
return location.value
def get_options(self, configuration, **kwargs):
"""
Populate self.options based on contents of ConfigParser object,
warns if unknown options are found
arguments:
configuration - a ConfigParser object
keyword arguments:
ignore_options - a list of option names that should be ignored
when checking for unknown options
"""
self.check_config(configuration)
for option in self.options.values():
self.log("Getting value for %s" % option.name)
try:
configfile.get_option(configuration, self.config_section,
option)
self.log("Got %s" % option.value)
except configparser.Error as err:
self.log("Syntax error in configuration: %s" % err,
option=option.name,
section=self.config_section,
level=logging.ERROR,
exception=False)
raise exceptions.SettingError(str(err))
except Exception:
self.log("Received exception when parsing option",
option=option.name,
section=self.config_section,
level=logging.ERROR,
exception=False)
raise
# check and warn if unknown options found
known_options = list(self.options.keys())
known_options.extend(kwargs.get('ignore_options', []))
temp = utilities.get_set_membership(
configuration.options(self.config_section), known_options,
configuration.defaults().keys())
for option in temp:
self.log("Found unknown option",
option=option,
section=self.config_section,
level=logging.WARNING)
def get_options(self, configuration, **kwargs):
"""
Populate self.options based on contents of ConfigParser object,
warns if unknown options are found
arguments:
configuration - a ConfigParser object
keyword arguments:
ignore_options - a list of option names that should be ignored
when checking for unknown options
"""
self.check_config(configuration)
for option in self.options.values():
self.log("Getting value for %s" % option.name)
try:
configfile.get_option(configuration,
self.config_section,
option)
self.log("Got %s" % option.value)
except Exception:
self.log("Received exception when parsing option",
option=option.name,
section=self.config_section,
level=logging.ERROR,
exception=False)
raise
# check and warn if unknown options found
known_options = self.options.keys()
known_options.extend(kwargs.get('ignore_options', []))
temp = utilities.get_set_membership(configuration.options(self.config_section),
known_options,
configuration.defaults().keys())
for option in temp:
self.log("Found unknown option",
option=option,
section=self.config_section,
level=logging.WARNING)
def parse_configuration(self, configuration):
"""
Try to get configuration information from ConfigParser or SafeConfigParser
object given by configuration and write recognized settings to attributes
dict
"""
self.log('GratiaConfiguration.parse_configuration started')
self.check_config(configuration)
if (not configuration.has_section(self.config_section) and requirements_are_installed()):
self.log('CE probes installed but no Gratia section, auto-configuring gratia')
self._auto_configure(configuration)
self.log('GratiaConfiguration.parse_configuration completed')
return True
elif not configuration.has_section(self.config_section):
self.enabled = False
self.log("%s section not in config file" % self.config_section)
self.log('Gratia.parse_configuration completed')
return
if not self.set_status(configuration):
self.log('GratiaConfiguration.parse_configuration completed')
return True
# set the appropriate defaults if we're on a CE
if requirements_are_installed():
if configuration.has_option('Site Information', 'group'):
self.grid_group = configuration.get('Site Information', 'group')
if self.grid_group == 'OSG':
self.options['probes'].default_value = \
self._production_defaults['probes']
elif self.grid_group == 'OSG-ITB':
self.options['probes'].default_value = \
self._itb_defaults['probes']
# grab configuration information for various jobmanagers
probes = self.get_installed_probes()
for probe in probes:
if probe == 'condor':
self._probe_config['condor'] = {'condor_location':
CondorConfiguration.get_condor_location(configuration),
'condor_config':
CondorConfiguration.get_condor_config(configuration)}
elif probe == 'pbs':
if BaseConfiguration.section_disabled(configuration, 'PBS'):
# if the PBS jobmanager is disabled, the CE is probably using LSF
# in any case, setting up the pbs gratia probe is not useful
continue
log_option = configfile.Option(name='log_directory',
required=configfile.Option.OPTIONAL,
default_value='')
configfile.get_option(configuration, 'PBS', log_option)
self._probe_config['pbs'] = {'log_directory': log_option.value}
accounting_log_option = configfile.Option(name='accounting_log_directory',
required=configfile.Option.OPTIONAL,
default_value='')
configfile.get_option(configuration, 'PBS', accounting_log_option)
self._probe_config['pbs'] = {'accounting_log_directory': accounting_log_option.value}
elif probe == 'lsf':
if BaseConfiguration.section_disabled(configuration, 'LSF'):
# if the LSF jobmanager is disabled, the CE is probably using PBS
# in any case, setting up the pbs gratia probe is not useful
continue
lsf_location = configfile.Option(name='lsf_location',
default_value='/usr/bin')
configfile.get_option(configuration, 'LSF', lsf_location)
self._probe_config['lsf'] = {'lsf_location': lsf_location.value}
log_option = configfile.Option(name='log_directory',
required=configfile.Option.OPTIONAL,
default_value='')
configfile.get_option(configuration, 'LSF', log_option)
self._probe_config['lsf']['log_directory'] = log_option.value
elif probe == 'sge':
if BaseConfiguration.section_disabled(configuration, 'SGE'):
# if section is disabled then the following code won't work
# since the parse_configuration will short circuit, so
# give a warning and then move on
self.log("Skipping SGE gratia probe configuration since SGE is disabled",
level=logging.WARNING)
continue
sge_config = SGEConfiguration(logger=self.logger)
sge_config.parse_configuration(configuration)
self._probe_config['sge'] = {'sge_accounting_file': sge_config.get_accounting_file()}
elif probe == 'slurm':
if BaseConfiguration.section_disabled(configuration, 'SLURM'):
# if section is disabled then the following code won't work
# since the parse_configuration will short circuit, so
# give a warning and then move on
self.log("Skipping Slurm gratia probe configuration since Slurm is disabled",
level=logging.WARNING)
continue
slurm_config = SlurmConfiguration(logger=self.logger)
slurm_config.parse_configuration(configuration)
self._probe_config['slurm'] = {'db_host': slurm_config.get_db_host(),
'db_port': slurm_config.get_db_port(),
'db_user': slurm_config.get_db_user(),
'db_pass': slurm_config.get_db_pass(),
'db_name': slurm_config.get_db_name(),
'cluster': slurm_config.get_slurm_cluster(),
'location': slurm_config.get_location()}
self.get_options(configuration,
ignore_options=['itb-jobmanager-gratia',
'itb-gridftp-gratia',
'osg-jobmanager-gratia',
'osg-gridftp-gratia',
'enabled'])
if utilities.blank(self.options['probes'].value):
self.log('GratiaConfiguration.parse_configuration completed')
return
self._parse_probes(self.options['probes'].value)
self.log('GratiaConfiguration.parse_configuration completed')
def parse_configuration(self, configuration):
"""
Try to get configuration information from ConfigParser or SafeConfigParser
object given by configuration and write recognized settings to attributes
dict
"""
self.log('GratiaConfiguration.parse_configuration started')
self.check_config(configuration)
if (not configuration.has_section(self.config_section)
and requirements_are_installed()):
self.log(
'CE probes installed but no Gratia section, auto-configuring gratia'
)
self._auto_configure(configuration)
self.log('GratiaConfiguration.parse_configuration completed')
return True
elif not configuration.has_section(self.config_section):
self.enabled = False
self.log("%s section not in config file" % self.config_section)
self.log('Gratia.parse_configuration completed')
return
if not self.set_status(configuration):
self.log('GratiaConfiguration.parse_configuration completed')
return True
# set the appropriate defaults if we're on a CE
if requirements_are_installed():
if configuration.has_option('Site Information', 'group'):
self.grid_group = configuration.get('Site Information',
'group')
if self.grid_group == 'OSG':
self.options['probes'].default_value = \
self._production_defaults['probes']
elif self.grid_group == 'OSG-ITB':
self.options['probes'].default_value = \
self._itb_defaults['probes']
# grab configuration information for various jobmanagers
probes = self.get_installed_probes()
for probe in probes:
if probe == 'condor':
self._probe_config['condor'] = {
'condor_location':
CondorConfiguration.get_condor_location(configuration),
'condor_config':
CondorConfiguration.get_condor_config(configuration)
}
elif probe == 'pbs':
if BaseConfiguration.section_disabled(
configuration, 'PBS'):
# if the PBS jobmanager is disabled, the CE is probably using LSF
# in any case, setting up the pbs gratia probe is not useful
continue
log_option = configfile.Option(
name='log_directory',
required=configfile.Option.OPTIONAL,
default_value='')
configfile.get_option(configuration, 'PBS', log_option)
self._probe_config['pbs'] = {
'log_directory': log_option.value
}
accounting_log_option = configfile.Option(
name='accounting_log_directory',
required=configfile.Option.OPTIONAL,
default_value='')
configfile.get_option(configuration, 'PBS',
accounting_log_option)
self._probe_config['pbs'] = {
'accounting_log_directory': accounting_log_option.value
}
elif probe == 'lsf':
if BaseConfiguration.section_disabled(
configuration, 'LSF'):
# if the LSF jobmanager is disabled, the CE is probably using PBS
# in any case, setting up the pbs gratia probe is not useful
continue
lsf_location = configfile.Option(name='lsf_location',
default_value='/usr/bin')
configfile.get_option(configuration, 'LSF', lsf_location)
self._probe_config['lsf'] = {
'lsf_location': lsf_location.value
}
log_option = configfile.Option(
name='log_directory',
required=configfile.Option.OPTIONAL,
default_value='')
configfile.get_option(configuration, 'LSF', log_option)
self._probe_config['lsf'][
'log_directory'] = log_option.value
elif probe == 'sge':
if BaseConfiguration.section_disabled(
configuration, 'SGE'):
# if section is disabled then the following code won't work
# since the parse_configuration will short circuit, so
# give a warning and then move on
self.log(
"Skipping SGE gratia probe configuration since SGE is disabled",
level=logging.WARNING)
continue
sge_config = SGEConfiguration(logger=self.logger)
sge_config.parse_configuration(configuration)
self._probe_config['sge'] = {
'sge_accounting_file':
sge_config.get_accounting_file()
}
elif probe == 'slurm':
if BaseConfiguration.section_disabled(
configuration, 'SLURM'):
# if section is disabled then the following code won't work
# since the parse_configuration will short circuit, so
# give a warning and then move on
self.log(
"Skipping Slurm gratia probe configuration since Slurm is disabled",
level=logging.WARNING)
continue
slurm_config = SlurmConfiguration(logger=self.logger)
slurm_config.parse_configuration(configuration)
self._probe_config['slurm'] = {
'db_host': slurm_config.get_db_host(),
'db_port': slurm_config.get_db_port(),
'db_user': slurm_config.get_db_user(),
'db_pass': slurm_config.get_db_pass(),
'db_name': slurm_config.get_db_name(),
'cluster': slurm_config.get_slurm_cluster(),
'location': slurm_config.get_location()
}
elif probe == 'htcondor-ce':
self._probe_config['htcondor-ce'] = {}
self.get_options(configuration,
ignore_options=[
'itb-jobmanager-gratia', 'itb-gridftp-gratia',
'osg-jobmanager-gratia', 'osg-gridftp-gratia',
'enabled'
])
if utilities.blank(self.options['probes'].value):
self.log('GratiaConfiguration.parse_configuration completed')
return
self._parse_probes(self.options['probes'].value)
self.log('GratiaConfiguration.parse_configuration completed')
def test_get_option(self):
"""
Test functionality of get_option function
"""
# check to see if exception is raised if option is not present
config = configparser.ConfigParser()
section = 'Test'
config.add_section(section)
option = configfile.Option(name='foo')
# do missing options get flagged
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
option.required = configfile.Option.OPTIONAL
# do optional settings get handled correctly if they are missing
self.assertEqual(None, configfile.get_option(config, section, option),
'Raised exception for missing optional setting')
option.default_value = 'test'
option.value = ''
# make sure defaults get used
configfile.get_option(config, section, option)
self.assertEqual(
'test', option.value,
"Default value not obtained, got %s, expected %s" %
(option.value, 'test'))
# get integer option
config.set(section, option.name, '1')
option.opt_type = int
configfile.get_option(config, section, option)
self.assertEqual(
1, option.value, 'Should have gotten an integer equal to 1 back ' +
"got %s" % option.value)
# get float option
option.opt_type = float
config.set(section, option.name, '1.23e5')
configfile.get_option(config, section, option)
self.assertAlmostEqual(
1.23e5, option.value, 7,
'Should have gotten a float equal to 1.23e5 ' +
"back, got %s" % option.value)
# check errors when wrong type specified
option.opt_type = int
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
# get boolean option
option.opt_type = bool
config.set(section, option.name, 'False')
configfile.get_option(config, section, option)
self.assertEqual(
False, option.value,
'Should have gotten False back, got %s' % option.value)
# check errors when wrong type specified
option.opt_type = int
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
# check errors when wrong type specified
option.opt_type = bool
config.set(section, option.name, 'abc')
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
# check to make sure that default gets set when setting is set to
# blank/UNAVAILABLE
option.opt_type = str
config.set(section, option.name, 'UNAVAILABLE')
configfile.get_option(config, section, option)
self.assertEqual(
'test', option.value,
"Should have gotten a value of test back, got %s" % option.value)
def test_get_option(self):
"""
Test functionality of get_option function
"""
# check to see if exception is raised if option is not present
config = ConfigParser.ConfigParser()
section = 'Test'
config.add_section(section)
option = configfile.Option(name='foo')
# do missing options get flagged
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
option.required = configfile.Option.OPTIONAL
# do optional settings get handled correctly if they are missing
self.assertEqual(None,
configfile.get_option(config,
section,
option),
'Raised exception for missing optional setting')
option.default_value = 'test'
option.value = ''
# make sure defaults get used
configfile.get_option(config, section, option)
self.assertEqual('test',
option.value,
"Default value not obtained, got %s, expected %s" %
(option.value, 'test'))
# get integer option
config.set(section, option.name, '1')
option.opt_type = int
configfile.get_option(config, section, option)
self.assertEqual(1,
option.value,
'Should have gotten an integer equal to 1 back ' +
"got %s" % option.value)
# get float option
option.opt_type = float
config.set(section, option.name, '1.23e5')
configfile.get_option(config, section, option)
self.assertAlmostEqual(1.23e5,
option.value,
7,
'Should have gotten a float equal to 1.23e5 ' +
"back, got %s" % option.value)
# check errors when wrong type specified
option.opt_type = int
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
# get boolean option
option.opt_type = bool
config.set(section, option.name, 'False')
configfile.get_option(config, section, option)
self.assertEqual(False,
option.value,
'Should have gotten False back, got %s' %
option.value)
# check errors when wrong type specified
option.opt_type = int
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
# check errors when wrong type specified
option.opt_type = bool
config.set(section, option.name, 'abc')
self.assertRaises(exceptions.SettingError,
configfile.get_option,
config=config,
section=section,
option=option)
# check to make sure that default gets set when setting is set to
# blank/UNAVAILABLE
option.opt_type = str
config.set(section, option.name, 'UNAVAILABLE')
configfile.get_option(config, section, option)
self.assertEqual('test',
option.value,
"Should have gotten a value of test back, got %s" %
option.value)
