def get_condor_location(configuration): """ Get the condor location based on the information in a configParser object (configuration argument) and environment variables if possible """ location = configfile.Option(name='condor_location', default_value=utilities.get_condor_location()) configfile.get_option(configuration, 'Condor', location) return location.value
def get_condor_location(configuration): """ Get the condor location based on the information in a configParser object (configuration argument) and environment variables if possible """ location = configfile.Option( name='condor_location', default_value=utilities.get_condor_location()) configfile.get_option(configuration, 'Condor', location) return location.value
def get_options(self, configuration, **kwargs): """ Populate self.options based on contents of ConfigParser object, warns if unknown options are found arguments: configuration - a ConfigParser object keyword arguments: ignore_options - a list of option names that should be ignored when checking for unknown options """ self.check_config(configuration) for option in self.options.values(): self.log("Getting value for %s" % option.name) try: configfile.get_option(configuration, self.config_section, option) self.log("Got %s" % option.value) except configparser.Error as err: self.log("Syntax error in configuration: %s" % err, option=option.name, section=self.config_section, level=logging.ERROR, exception=False) raise exceptions.SettingError(str(err)) except Exception: self.log("Received exception when parsing option", option=option.name, section=self.config_section, level=logging.ERROR, exception=False) raise # check and warn if unknown options found known_options = list(self.options.keys()) known_options.extend(kwargs.get('ignore_options', [])) temp = utilities.get_set_membership( configuration.options(self.config_section), known_options, configuration.defaults().keys()) for option in temp: self.log("Found unknown option", option=option, section=self.config_section, level=logging.WARNING)
def get_options(self, configuration, **kwargs): """ Populate self.options based on contents of ConfigParser object, warns if unknown options are found arguments: configuration - a ConfigParser object keyword arguments: ignore_options - a list of option names that should be ignored when checking for unknown options """ self.check_config(configuration) for option in self.options.values(): self.log("Getting value for %s" % option.name) try: configfile.get_option(configuration, self.config_section, option) self.log("Got %s" % option.value) except Exception: self.log("Received exception when parsing option", option=option.name, section=self.config_section, level=logging.ERROR, exception=False) raise # check and warn if unknown options found known_options = self.options.keys() known_options.extend(kwargs.get('ignore_options', [])) temp = utilities.get_set_membership(configuration.options(self.config_section), known_options, configuration.defaults().keys()) for option in temp: self.log("Found unknown option", option=option, section=self.config_section, level=logging.WARNING)
def parse_configuration(self, configuration): """ Try to get configuration information from ConfigParser or SafeConfigParser object given by configuration and write recognized settings to attributes dict """ self.log('GratiaConfiguration.parse_configuration started') self.check_config(configuration) if (not configuration.has_section(self.config_section) and requirements_are_installed()): self.log('CE probes installed but no Gratia section, auto-configuring gratia') self._auto_configure(configuration) self.log('GratiaConfiguration.parse_configuration completed') return True elif not configuration.has_section(self.config_section): self.enabled = False self.log("%s section not in config file" % self.config_section) self.log('Gratia.parse_configuration completed') return if not self.set_status(configuration): self.log('GratiaConfiguration.parse_configuration completed') return True # set the appropriate defaults if we're on a CE if requirements_are_installed(): if configuration.has_option('Site Information', 'group'): self.grid_group = configuration.get('Site Information', 'group') if self.grid_group == 'OSG': self.options['probes'].default_value = \ self._production_defaults['probes'] elif self.grid_group == 'OSG-ITB': self.options['probes'].default_value = \ self._itb_defaults['probes'] # grab configuration information for various jobmanagers probes = self.get_installed_probes() for probe in probes: if probe == 'condor': self._probe_config['condor'] = {'condor_location': CondorConfiguration.get_condor_location(configuration), 'condor_config': CondorConfiguration.get_condor_config(configuration)} elif probe == 'pbs': if BaseConfiguration.section_disabled(configuration, 'PBS'): # if the PBS jobmanager is disabled, the CE is probably using LSF # in any case, setting up the pbs gratia probe is not useful continue log_option = configfile.Option(name='log_directory', required=configfile.Option.OPTIONAL, default_value='') configfile.get_option(configuration, 'PBS', log_option) self._probe_config['pbs'] = {'log_directory': log_option.value} accounting_log_option = configfile.Option(name='accounting_log_directory', required=configfile.Option.OPTIONAL, default_value='') configfile.get_option(configuration, 'PBS', accounting_log_option) self._probe_config['pbs'] = {'accounting_log_directory': accounting_log_option.value} elif probe == 'lsf': if BaseConfiguration.section_disabled(configuration, 'LSF'): # if the LSF jobmanager is disabled, the CE is probably using PBS # in any case, setting up the pbs gratia probe is not useful continue lsf_location = configfile.Option(name='lsf_location', default_value='/usr/bin') configfile.get_option(configuration, 'LSF', lsf_location) self._probe_config['lsf'] = {'lsf_location': lsf_location.value} log_option = configfile.Option(name='log_directory', required=configfile.Option.OPTIONAL, default_value='') configfile.get_option(configuration, 'LSF', log_option) self._probe_config['lsf']['log_directory'] = log_option.value elif probe == 'sge': if BaseConfiguration.section_disabled(configuration, 'SGE'): # if section is disabled then the following code won't work # since the parse_configuration will short circuit, so # give a warning and then move on self.log("Skipping SGE gratia probe configuration since SGE is disabled", level=logging.WARNING) continue sge_config = SGEConfiguration(logger=self.logger) sge_config.parse_configuration(configuration) self._probe_config['sge'] = {'sge_accounting_file': sge_config.get_accounting_file()} elif probe == 'slurm': if BaseConfiguration.section_disabled(configuration, 'SLURM'): # if section is disabled then the following code won't work # since the parse_configuration will short circuit, so # give a warning and then move on self.log("Skipping Slurm gratia probe configuration since Slurm is disabled", level=logging.WARNING) continue slurm_config = SlurmConfiguration(logger=self.logger) slurm_config.parse_configuration(configuration) self._probe_config['slurm'] = {'db_host': slurm_config.get_db_host(), 'db_port': slurm_config.get_db_port(), 'db_user': slurm_config.get_db_user(), 'db_pass': slurm_config.get_db_pass(), 'db_name': slurm_config.get_db_name(), 'cluster': slurm_config.get_slurm_cluster(), 'location': slurm_config.get_location()} self.get_options(configuration, ignore_options=['itb-jobmanager-gratia', 'itb-gridftp-gratia', 'osg-jobmanager-gratia', 'osg-gridftp-gratia', 'enabled']) if utilities.blank(self.options['probes'].value): self.log('GratiaConfiguration.parse_configuration completed') return self._parse_probes(self.options['probes'].value) self.log('GratiaConfiguration.parse_configuration completed')
def parse_configuration(self, configuration): """ Try to get configuration information from ConfigParser or SafeConfigParser object given by configuration and write recognized settings to attributes dict """ self.log('GratiaConfiguration.parse_configuration started') self.check_config(configuration) if (not configuration.has_section(self.config_section) and requirements_are_installed()): self.log( 'CE probes installed but no Gratia section, auto-configuring gratia' ) self._auto_configure(configuration) self.log('GratiaConfiguration.parse_configuration completed') return True elif not configuration.has_section(self.config_section): self.enabled = False self.log("%s section not in config file" % self.config_section) self.log('Gratia.parse_configuration completed') return if not self.set_status(configuration): self.log('GratiaConfiguration.parse_configuration completed') return True # set the appropriate defaults if we're on a CE if requirements_are_installed(): if configuration.has_option('Site Information', 'group'): self.grid_group = configuration.get('Site Information', 'group') if self.grid_group == 'OSG': self.options['probes'].default_value = \ self._production_defaults['probes'] elif self.grid_group == 'OSG-ITB': self.options['probes'].default_value = \ self._itb_defaults['probes'] # grab configuration information for various jobmanagers probes = self.get_installed_probes() for probe in probes: if probe == 'condor': self._probe_config['condor'] = { 'condor_location': CondorConfiguration.get_condor_location(configuration), 'condor_config': CondorConfiguration.get_condor_config(configuration) } elif probe == 'pbs': if BaseConfiguration.section_disabled( configuration, 'PBS'): # if the PBS jobmanager is disabled, the CE is probably using LSF # in any case, setting up the pbs gratia probe is not useful continue log_option = configfile.Option( name='log_directory', required=configfile.Option.OPTIONAL, default_value='') configfile.get_option(configuration, 'PBS', log_option) self._probe_config['pbs'] = { 'log_directory': log_option.value } accounting_log_option = configfile.Option( name='accounting_log_directory', required=configfile.Option.OPTIONAL, default_value='') configfile.get_option(configuration, 'PBS', accounting_log_option) self._probe_config['pbs'] = { 'accounting_log_directory': accounting_log_option.value } elif probe == 'lsf': if BaseConfiguration.section_disabled( configuration, 'LSF'): # if the LSF jobmanager is disabled, the CE is probably using PBS # in any case, setting up the pbs gratia probe is not useful continue lsf_location = configfile.Option(name='lsf_location', default_value='/usr/bin') configfile.get_option(configuration, 'LSF', lsf_location) self._probe_config['lsf'] = { 'lsf_location': lsf_location.value } log_option = configfile.Option( name='log_directory', required=configfile.Option.OPTIONAL, default_value='') configfile.get_option(configuration, 'LSF', log_option) self._probe_config['lsf'][ 'log_directory'] = log_option.value elif probe == 'sge': if BaseConfiguration.section_disabled( configuration, 'SGE'): # if section is disabled then the following code won't work # since the parse_configuration will short circuit, so # give a warning and then move on self.log( "Skipping SGE gratia probe configuration since SGE is disabled", level=logging.WARNING) continue sge_config = SGEConfiguration(logger=self.logger) sge_config.parse_configuration(configuration) self._probe_config['sge'] = { 'sge_accounting_file': sge_config.get_accounting_file() } elif probe == 'slurm': if BaseConfiguration.section_disabled( configuration, 'SLURM'): # if section is disabled then the following code won't work # since the parse_configuration will short circuit, so # give a warning and then move on self.log( "Skipping Slurm gratia probe configuration since Slurm is disabled", level=logging.WARNING) continue slurm_config = SlurmConfiguration(logger=self.logger) slurm_config.parse_configuration(configuration) self._probe_config['slurm'] = { 'db_host': slurm_config.get_db_host(), 'db_port': slurm_config.get_db_port(), 'db_user': slurm_config.get_db_user(), 'db_pass': slurm_config.get_db_pass(), 'db_name': slurm_config.get_db_name(), 'cluster': slurm_config.get_slurm_cluster(), 'location': slurm_config.get_location() } elif probe == 'htcondor-ce': self._probe_config['htcondor-ce'] = {} self.get_options(configuration, ignore_options=[ 'itb-jobmanager-gratia', 'itb-gridftp-gratia', 'osg-jobmanager-gratia', 'osg-gridftp-gratia', 'enabled' ]) if utilities.blank(self.options['probes'].value): self.log('GratiaConfiguration.parse_configuration completed') return self._parse_probes(self.options['probes'].value) self.log('GratiaConfiguration.parse_configuration completed')
def test_get_option(self): """ Test functionality of get_option function """ # check to see if exception is raised if option is not present config = configparser.ConfigParser() section = 'Test' config.add_section(section) option = configfile.Option(name='foo') # do missing options get flagged self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) option.required = configfile.Option.OPTIONAL # do optional settings get handled correctly if they are missing self.assertEqual(None, configfile.get_option(config, section, option), 'Raised exception for missing optional setting') option.default_value = 'test' option.value = '' # make sure defaults get used configfile.get_option(config, section, option) self.assertEqual( 'test', option.value, "Default value not obtained, got %s, expected %s" % (option.value, 'test')) # get integer option config.set(section, option.name, '1') option.opt_type = int configfile.get_option(config, section, option) self.assertEqual( 1, option.value, 'Should have gotten an integer equal to 1 back ' + "got %s" % option.value) # get float option option.opt_type = float config.set(section, option.name, '1.23e5') configfile.get_option(config, section, option) self.assertAlmostEqual( 1.23e5, option.value, 7, 'Should have gotten a float equal to 1.23e5 ' + "back, got %s" % option.value) # check errors when wrong type specified option.opt_type = int self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) # get boolean option option.opt_type = bool config.set(section, option.name, 'False') configfile.get_option(config, section, option) self.assertEqual( False, option.value, 'Should have gotten False back, got %s' % option.value) # check errors when wrong type specified option.opt_type = int self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) # check errors when wrong type specified option.opt_type = bool config.set(section, option.name, 'abc') self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) # check to make sure that default gets set when setting is set to # blank/UNAVAILABLE option.opt_type = str config.set(section, option.name, 'UNAVAILABLE') configfile.get_option(config, section, option) self.assertEqual( 'test', option.value, "Should have gotten a value of test back, got %s" % option.value)
def test_get_option(self): """ Test functionality of get_option function """ # check to see if exception is raised if option is not present config = ConfigParser.ConfigParser() section = 'Test' config.add_section(section) option = configfile.Option(name='foo') # do missing options get flagged self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) option.required = configfile.Option.OPTIONAL # do optional settings get handled correctly if they are missing self.assertEqual(None, configfile.get_option(config, section, option), 'Raised exception for missing optional setting') option.default_value = 'test' option.value = '' # make sure defaults get used configfile.get_option(config, section, option) self.assertEqual('test', option.value, "Default value not obtained, got %s, expected %s" % (option.value, 'test')) # get integer option config.set(section, option.name, '1') option.opt_type = int configfile.get_option(config, section, option) self.assertEqual(1, option.value, 'Should have gotten an integer equal to 1 back ' + "got %s" % option.value) # get float option option.opt_type = float config.set(section, option.name, '1.23e5') configfile.get_option(config, section, option) self.assertAlmostEqual(1.23e5, option.value, 7, 'Should have gotten a float equal to 1.23e5 ' + "back, got %s" % option.value) # check errors when wrong type specified option.opt_type = int self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) # get boolean option option.opt_type = bool config.set(section, option.name, 'False') configfile.get_option(config, section, option) self.assertEqual(False, option.value, 'Should have gotten False back, got %s' % option.value) # check errors when wrong type specified option.opt_type = int self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) # check errors when wrong type specified option.opt_type = bool config.set(section, option.name, 'abc') self.assertRaises(exceptions.SettingError, configfile.get_option, config=config, section=section, option=option) # check to make sure that default gets set when setting is set to # blank/UNAVAILABLE option.opt_type = str config.set(section, option.name, 'UNAVAILABLE') configfile.get_option(config, section, option) self.assertEqual('test', option.value, "Should have gotten a value of test back, got %s" % option.value)