Ejemplo n.º 1
0
    def parse_configuration(self, configuration):
        """
        Try to get configuration information from ConfigParser or SafeConfigParser
        object given by configuration and write recognized settings to attributes
        dict
        """
        self.log('GipConfiguration.parse_configuration started')

        if not utilities.rpm_installed('gip'):
            self.log('GIP not installed, disabling GIP')
            self.log('GipConfiguration.parse_configuration completed')
            self.enabled = False
            return
        else:
            self.enabled = True

        # TODO Do we want to force the user to have a GIP section? Uncomment the following if not.
        # if not configuration.has_section(self.config_section):
        #   self.log("%s section not in config file" % self.config_section)
        #   self.log('GipConfiguration.parse_configuration completed')
        #   self.enabled = False
        #   return

        self.check_config(configuration)

        self._parse_configuration(configuration)

        self.log('GipConfiguration.parse_configuration completed')
Ejemplo n.º 2
0
    def testParsing2(self):
        """
        Test rsv parsing
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return

        settings = self.load_settings_from_files("rsv/rsv2.ini")
        options = settings.options
        variables = {'gratia_probes': '',
                     'ce_hosts': 'my.host.com, my2.host.com',
                     'gridftp_hosts': 'my.host.com, my2.host.com',
                     'gridftp_dir': '/tmp',
                     'gums_hosts': 'my.host.com, my2.host.com',
                     'srm_hosts': 'my.host.com, my2.host.com',
                     'srm_dir': '/srm/dir, /srm/dir2',
                     'srm_webservice_path': 'srm/v2/server,',
                     'service_cert': '/etc/redhat-release',
                     'service_key': '/etc/redhat-release',
                     'service_proxy': '/tmp/rsvproxy',
                     'enable_gratia': False,
                     'enable_nagios': False}
        for var in variables:
            self.assertTrue(options.has_key(var),
                            "Option %s missing" % var)
            self.assertEqual(options[var].value,
                             variables[var],
                             "Wrong value obtained for %s, got %s but " \
                             "expected %s" % (var, options[var].value, variables[var]))
Ejemplo n.º 3
0
    def testMultipleHosts(self):
        """
        Test rsv parsing
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/multiple_hosts.ini")
        options = settings.options
        variables = {
            'ce_hosts': 'host1.site.com, host2.site.com',
            'gridftp_hosts': 'host3.site.com, host4.site.com',
            'gridftp_dir': '/tmp',
            'gums_hosts': 'host8.site.com, host9.site.com',
            'srm_hosts': 'host5.site.com, host6.site.com, host7.site.com:1234',
            'srm_dir': '/srm/dir',
            'srm_webservice_path': 'srm/v2/server',
            'service_cert': './configs/rsv/rsv1.ini',
            'service_key': './configs/rsv/rsv1.ini',
            'service_proxy': '/tmp/rsvproxy',
            'enable_gratia': True,
            'enable_nagios': True
        }
        for var in variables:
            self.assertTrue(options.has_key(var), "Option %s missing" % var)
            self.assertEqual(options[var].value,
                             variables[var],
                             "Wrong value obtained for %s, got %s but " \
                             "expected %s" % (var, options[var].value, variables[var]))
Ejemplo n.º 4
0
    def __init__(self, *args, **kwargs):
        # pylint: disable-msg=W0142
        super(InfoServicesConfiguration, self).__init__(*args, **kwargs)
        self.logger = logging.getLogger(__name__)
        self.log("InfoServicesConfiguration.__init__ started")
        self.config_section = 'Info Services'
        self.options = {
            'ce_collectors':
            configfile.Option(name='ce_collectors',
                              default_value='',
                              required=configfile.Option.OPTIONAL)
        }
        self._itb_default_ce_collectors = \
            'collector-itb.opensciencegrid.org:%d' % HTCONDOR_CE_COLLECTOR_PORT
        self._production_default_ce_collectors = \
            'collector1.opensciencegrid.org:%d,collector2.opensciencegrid.org:%d' % (
                HTCONDOR_CE_COLLECTOR_PORT, HTCONDOR_CE_COLLECTOR_PORT)

        self.ce_collectors = []
        self.ce_collector_required_rpms_installed = utilities.rpm_installed(
            'htcondor-ce')
        self.osg_resource = ""
        self.osg_resource_group = ""
        self.enabled_batch_systems = []
        self.htcondor_gateway_enabled = None
        self.resource_catalog = None
        self.authorization_method = None
        self.subcluster_sections = None
        self.gums_host = None
        self.misc_module = MiscConfiguration(*args, **kwargs)

        self.log("InfoServicesConfiguration.__init__ completed")
Ejemplo n.º 5
0
    def testServiceList(self):
        """
        Test to make sure right services get returned
        """
        # If rsv-core is not installed, settings.enabled will always be False
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/rsv1.ini")
        services = settings.enabled_services()
        expected_services = set(['rsv', 'condor-cron'])
        self.assertEqual(
            services, expected_services,
            "List of enabled services incorrect, " + "got %s but expected %s" %
            (services, expected_services))

        settings = self.load_settings_from_files("rsv/disabled.ini")
        services = settings.enabled_services()
        expected_services = set()
        self.assertEqual(
            services, expected_services,
            "List of enabled services incorrect, " + "got %s but expected %s" %
            (services, expected_services))

        settings = self.load_settings_from_files("rsv/ignored.ini")
        services = settings.enabled_services()
        expected_services = set()
        self.assertEqual(
            services, expected_services,
            "List of enabled services incorrect, " + "got %s but expected %s" %
            (services, expected_services))
Ejemplo n.º 6
0
    def testMultipleHosts(self):
        """
        Test rsv parsing
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/multiple_hosts.ini")
        options = settings.options
        variables = {'ce_hosts': 'host1.site.com, host2.site.com',
                     'gridftp_hosts': 'host3.site.com, host4.site.com',
                     'gridftp_dir': '/tmp',
                     'gums_hosts': 'host8.site.com, host9.site.com',
                     'srm_hosts': 'host5.site.com, host6.site.com, host7.site.com:1234',
                     'srm_dir': '/srm/dir',
                     'srm_webservice_path': 'srm/v2/server',
                     'service_cert': './configs/rsv/rsv1.ini',
                     'service_key': './configs/rsv/rsv1.ini',
                     'service_proxy': '/tmp/rsvproxy',
                     'enable_gratia': True,
                     'enable_nagios': True}
        for var in variables:
            self.assertTrue(options.has_key(var),
                            "Option %s missing" % var)
            self.assertEqual(options[var].value,
                             variables[var],
                             "Wrong value obtained for %s, got %s but " \
                             "expected %s" % (var, options[var].value, variables[var]))
Ejemplo n.º 7
0
    def testParsing2(self):
        """
        Test rsv parsing
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return

        settings = self.load_settings_from_files("rsv/rsv2.ini")
        options = settings.options
        variables = {
            'gratia_probes': '',
            'ce_hosts': 'my.host.com, my2.host.com',
            'gridftp_hosts': 'my.host.com, my2.host.com',
            'gridftp_dir': '/tmp',
            'gums_hosts': 'my.host.com, my2.host.com',
            'srm_hosts': 'my.host.com, my2.host.com',
            'srm_dir': '/srm/dir, /srm/dir2',
            'srm_webservice_path': 'srm/v2/server,',
            'service_cert': '/etc/redhat-release',
            'service_key': '/etc/redhat-release',
            'service_proxy': '/tmp/rsvproxy',
            'enable_gratia': False,
            'enable_nagios': False
        }
        for var in variables:
            self.assertTrue(options.has_key(var), "Option %s missing" % var)
            self.assertEqual(options[var].value,
                             variables[var],
                             "Wrong value obtained for %s, got %s but " \
                             "expected %s" % (var, options[var].value, variables[var]))
Ejemplo n.º 8
0
    def testServiceList(self):
        """
        Test to make sure right services get returned
        """
        # If rsv-core is not installed, settings.enabled will always be False
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/rsv1.ini")
        services = settings.enabled_services()
        expected_services = set(['rsv', 'condor-cron'])
        self.assertEqual(services, expected_services,
                         "List of enabled services incorrect, " +
                         "got %s but expected %s" % (services, expected_services))

        settings = self.load_settings_from_files("rsv/disabled.ini")
        services = settings.enabled_services()
        expected_services = set()
        self.assertEqual(services, expected_services,
                         "List of enabled services incorrect, " +
                         "got %s but expected %s" % (services, expected_services))

        settings = self.load_settings_from_files("rsv/ignored.ini")
        services = settings.enabled_services()
        expected_services = set()
        self.assertEqual(services, expected_services,
                         "List of enabled services incorrect, " +
                         "got %s but expected %s" % (services, expected_services))
Ejemplo n.º 9
0
    def parse_configuration(self, configuration):
        """
        Try to get configuration information from ConfigParser or SafeConfigParser
        object given by configuration and write recognized settings to attributes
        dict
        """
        self.log('GipConfiguration.parse_configuration started')

        if not utilities.rpm_installed('gip'):
            self.log('GIP not installed, disabling GIP')
            self.log('GipConfiguration.parse_configuration completed')
            self.enabled = False
            return
        else:
            self.enabled = True

        # TODO Do we want to force the user to have a GIP section? Uncomment the following if not.
        # if not configuration.has_section(self.config_section):
        #   self.log("%s section not in config file" % self.config_section)
        #   self.log('GipConfiguration.parse_configuration completed')
        #   self.enabled = False
        #   return

        self.check_config(configuration)

        self._parse_configuration(configuration)

        self.log('GipConfiguration.parse_configuration completed')
Ejemplo n.º 10
0
    def setup_htcondor_ce_config(self):
        """
        Populate the config file that tells htcondor-ce where the condor
        pool is and where the spool directory is.

        Returns True if successful, False otherwise
        """
        if not utilities.rpm_installed('htcondor-ce'):
            self.log("Unable to configure htcondor-ce for Condor: htcondor-ce not installed", level=logging.ERROR)
            return False

        def get_condor_ce_config_val(variable):
            return utilities.get_condor_config_val(variable, executable='condor_ce_config_val', quiet_undefined=True)

        # Get values for the settings we want to update. We can get the
        # values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME,
        # we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing.
        # We also get the current / default value from condor_ce_config_val;
        # only update the setting in case the value from
        # condor_config_val is different from the value from condor_ce_config_val.
        condor_ce_config = {}
        for condor_ce_config_key, condor_config_keys in [
            ('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']),
            ('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']),
            ('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])]:

            condor_config_value = None
            for condor_config_value in (utilities.get_condor_config_val(k, quiet_undefined=True) for k in
                                        condor_config_keys):
                if condor_config_value:
                    break

            condor_ce_config_value = get_condor_ce_config_val(condor_ce_config_key)
            if not (condor_config_value or condor_ce_config_value):
                self.log("Unable to determine value for %s from %s and default not set; check your Condor config" %
                         (condor_ce_config_key, ' or '.join(condor_config_keys)), level=logging.ERROR)
                return False
            elif not condor_config_value:
                continue  # can't set anything for this

            # Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744)
            if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL':
                condor_collector_port = (utilities.get_condor_config_val('COLLECTOR_PORT', quiet_undefined=True)
                                         or '9618')
                condor_config_value = self._add_port_if_necessary(condor_config_value, condor_collector_port)

            if not condor_ce_config_value or condor_ce_config_value != condor_config_value:
                condor_ce_config[condor_ce_config_key] = condor_config_value

        if condor_ce_config:
            buf = utilities.read_file(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE,
                                      default="# This file is managed by osg-configure\n")
            for key, value in condor_ce_config.items():
                buf = utilities.add_or_replace_setting(buf, key, value, quote_value=False)

            if not utilities.atomic_write(JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf):
                return False

        return True
Ejemplo n.º 11
0
    def test_rpm_install(self):
        """
        Test rpm_installed function
        """
        rpm_name = 'foo'
        self.assertFalse(utilities.rpm_installed(rpm_name),
                         'foo is not installed, but rpm_installed returned True')
        rpm_name = 'filesystem'
        self.assertTrue(utilities.rpm_installed(rpm_name),
                        'filesystem is installed, but rpm_installed returned False')

        rpm_names = ['filesystem', 'foo']
        self.assertFalse(utilities.rpm_installed(rpm_names),
                         'foo is not installed, but rpm_installed returned True')
        rpm_names = ['filesystem', 'glibc']
        self.assertTrue(utilities.rpm_installed(rpm_names),
                        'filesystem and glibc are installed, but rpm_installed returned False')
Ejemplo n.º 12
0
    def test_rpm_install(self):
        """
        Test rpm_installed function
        """
        rpm_name = 'foo'
        self.assertFalse(utilities.rpm_installed(rpm_name),
                         'foo is not installed, but rpm_installed returned True')
        rpm_name = 'filesystem'
        self.assertTrue(utilities.rpm_installed(rpm_name),
                        'filesystem is installed, but rpm_installed returned False')

        rpm_names = ['filesystem', 'foo']
        self.assertFalse(utilities.rpm_installed(rpm_names),
                         'foo is not installed, but rpm_installed returned True')
        rpm_names = ['filesystem', 'glibc']
        self.assertTrue(utilities.rpm_installed(rpm_names),
                        'filesystem and glibc are installed, but rpm_installed returned False')
Ejemplo n.º 13
0
    def testServiceList(self):
        """
        Test to make sure right services get returned
        """

        config_file = get_test_config("misc/misc_xacml.ini")
        configuration = ConfigParser.SafeConfigParser()
        configuration.read(config_file)

        settings = misc.MiscConfiguration(logger=global_logger)
        try:
            settings.parse_configuration(configuration)
        except Exception as e:
            self.fail("Received exception while parsing configuration: %s" % e)
        services = settings.enabled_services()
        if utilities.rpm_installed('fetch-crl'):
            expected_services = set(
                ['fetch-crl-cron', 'fetch-crl-boot', 'gums-client-cron'])
        else:
            expected_services = set(['gums-client-cron'])

        self.assertEqual(
            services, expected_services,
            "List of enabled services incorrect, " + "got %s but expected %s" %
            (services, expected_services))

        config_file = get_test_config("misc/misc_gridmap.ini")
        configuration = ConfigParser.SafeConfigParser()
        configuration.read(config_file)

        settings = misc.MiscConfiguration(logger=global_logger)
        try:
            settings.parse_configuration(configuration)
        except Exception as e:
            self.fail("Received exception while parsing configuration: %s" % e)
        services = settings.enabled_services()
        if utilities.rpm_installed('fetch-crl'):
            expected_services = set(
                ['fetch-crl-cron', 'fetch-crl-boot', 'edg-mkgridmap'])
        else:
            expected_services = set(['edg-mkgridmap'])

        self.assertEqual(
            services, expected_services,
            "List of enabled services incorrect, " + "got %s but expected %s" %
            (services, expected_services))
Ejemplo n.º 14
0
    def __init__(self, *args, **kwargs):
        # pylint: disable-msg=W0142
        super(InfoServicesConfiguration, self).__init__(*args, **kwargs)
        self.log("InfoServicesConfiguration.__init__ started")
        # file location for xml file with info services subscriptions
        self.config_section = 'Info Services'
        self.options = {
            'bdii_servers':
            configfile.Option(name='bdii_servers', default_value=''),
            'ce_collectors':
            configfile.Option(name='ce_collectors',
                              default_value='',
                              required=configfile.Option.OPTIONAL)
        }
        self._itb_defaults = {
            'bdii_servers':
            'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
            'ce_collectors':
            'collector-itb.opensciencegrid.org:%d' % HTCONDOR_CE_COLLECTOR_PORT
        }
        self._production_defaults = {
            'bdii_servers':
            'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
            'ce_collectors':
            'collector1.opensciencegrid.org:%d,collector2.opensciencegrid.org:%d'
            % (HTCONDOR_CE_COLLECTOR_PORT, HTCONDOR_CE_COLLECTOR_PORT)
        }
        self.bdii_servers = {}
        self.copy_host_cert_for_service_cert = False

        self.ois_required_rpms_installed = utilities.gateway_installed(
        ) and utilities.rpm_installed('osg-info-services')

        # for htcondor-ce-info-services:
        self.ce_collectors = []
        self.ce_collector_required_rpms_installed = utilities.rpm_installed(
            'htcondor-ce')
        self.osg_resource = ""
        self.osg_resource_group = ""
        self.enabled_batch_systems = []
        self.htcondor_gateway_enabled = None
        self.resource_catalog = None
        self.authorization_method = None
        self.subcluster_sections = None

        self.log("InfoServicesConfiguration.__init__ completed")
Ejemplo n.º 15
0
 def testValidSettings2(self):
     """
     Test the check_attributes function to see if it works on valid settings
     """
     # need to have rsv installed to get rsv tests working
     if not utilities.rpm_installed('rsv-core'):
         return
     settings = self.load_settings_from_files("rsv/rsv2.ini")
     attributes = settings.get_attributes()
     self.assertTrue(settings.check_attributes(attributes),
                     "Correct configuration incorrectly flagged as incorrect")
Ejemplo n.º 16
0
    def enabled_services(self):
        """Return a list of  system services needed for module to work
        """

        if not self.enabled or self.ignored:
            return set()

        services = set()
        if utilities.rpm_installed('fetch-crl'):
            services = set(['fetch-crl-cron', 'fetch-crl-boot'])
        elif utilities.rpm_installed('fetch-crl3'):
            services = set(['fetch-crl3-cron', 'fetch-crl3-boot'])

        if self.options['authorization_method'].value == 'xacml':
            services.add('gums-client-cron')
        elif self.options['authorization_method'].value == 'gridmap':
            services.add('edg-mkgridmap')
        if self.options['enable_cleanup'].value:
            services.add('osg-cleanup-cron')
        return services
Ejemplo n.º 17
0
    def testInvalidSRMHost(self):
        """
        Test the check_attributes with invalid srm host
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/invalid_srm_host.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Invalid srm ignored")
Ejemplo n.º 18
0
    def enabled_services(self):
        """Return a list of  system services needed for module to work
        """

        if not self.enabled or self.ignored:
            return set()

        services = set()
        if utilities.rpm_installed('fetch-crl'):
            services = {'fetch-crl-cron', 'fetch-crl-boot'}

        return services
Ejemplo n.º 19
0
    def testValidSettingsIPv6(self):
        """
        Test RSV config accepts IPv6 hostnames
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/rsv_ipv6.ini")
        attributes = settings.get_attributes()
        self.assertTrue(settings.check_attributes(attributes),
                        "Correct configuration incorrectly flagged as incorrect")
Ejemplo n.º 20
0
    def testMissingProxy(self):
        """
        Test the check_attributes with a missing proxy cert file
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/missing_proxy.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Missing rsv proxy file ignored")
Ejemplo n.º 21
0
    def testInvalidSRMHost(self):
        """
        Test the check_attributes with invalid srm host
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/invalid_srm_host.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Invalid srm ignored")
Ejemplo n.º 22
0
    def testMissingProxy(self):
        """
        Test the check_attributes with a missing proxy cert file
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/missing_proxy.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Missing rsv proxy file ignored")
Ejemplo n.º 23
0
    def testParsingDisabled(self):
        """
        Test parsing when disabled
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/disabled.ini")
        attributes = settings.get_attributes()
        self.assertEqual(len(attributes), 0,
                         "Disabled configuration should have no attributes")
Ejemplo n.º 24
0
    def testParsingDisabled(self):
        """
        Test parsing when disabled
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/disabled.ini")
        attributes = settings.get_attributes()
        self.assertEqual(len(attributes), 0,
                         "Disabled configuration should have no attributes")
Ejemplo n.º 25
0
 def testValidSettings2(self):
     """
     Test the check_attributes function to see if it works on valid settings
     """
     # need to have rsv installed to get rsv tests working
     if not utilities.rpm_installed('rsv-core'):
         return
     settings = self.load_settings_from_files("rsv/rsv2.ini")
     attributes = settings.get_attributes()
     self.assertTrue(
         settings.check_attributes(attributes),
         "Correct configuration incorrectly flagged as incorrect")
Ejemplo n.º 26
0
    def testValidSettingsIPv6(self):
        """
        Test RSV config accepts IPv6 hostnames
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/rsv_ipv6.ini")
        attributes = settings.get_attributes()
        self.assertTrue(
            settings.check_attributes(attributes),
            "Correct configuration incorrectly flagged as incorrect")
Ejemplo n.º 27
0
    def __init__(self, *args, **kwargs):
        # pylint: disable-msg=W0142
        super(InfoServicesConfiguration, self).__init__(*args, **kwargs)
        self.log("InfoServicesConfiguration.__init__ started")
        # file location for xml file with info services subscriptions
        self.config_section = 'Info Services'
        self.options = {'bdii_servers': configfile.Option(name='bdii_servers',
                                                          default_value=''),
                        'ce_collectors': configfile.Option(name='ce_collectors',
                                                           default_value='',
                                                           required=configfile.Option.OPTIONAL)}
        self._itb_defaults = {
            'bdii_servers': 'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
            'ce_collectors': 'collector-itb.opensciencegrid.org:%d' % HTCONDOR_CE_COLLECTOR_PORT}
        self._production_defaults = {
            'bdii_servers': 'http://is1.grid.iu.edu:14001[RAW],http://is2.grid.iu.edu:14001[RAW]',
            'ce_collectors': 'collector1.opensciencegrid.org:%d,collector2.opensciencegrid.org:%d' % (
                HTCONDOR_CE_COLLECTOR_PORT, HTCONDOR_CE_COLLECTOR_PORT)}
        self.bdii_servers = {}
        self.copy_host_cert_for_service_cert = False

        self.ois_required_rpms_installed = utilities.gateway_installed() and utilities.rpm_installed(
            'osg-info-services')

        # for htcondor-ce-info-services:
        self.ce_collectors = []
        self.ce_collector_required_rpms_installed = utilities.rpm_installed('htcondor-ce')
        self.osg_resource = ""
        self.osg_resource_group = ""
        self.enabled_batch_systems = []
        self.htcondor_gateway_enabled = None
        self.resource_catalog = None
        self.authorization_method = None
        self.subcluster_sections = None

        self.log("InfoServicesConfiguration.__init__ completed")
Ejemplo n.º 28
0
    def testInvalidGratiaProbes(self):
        """
        Test the check_attributes with invalid gratia probes
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/invalid_gratia1.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Invalid gratia probe ignored")

        settings = self.load_settings_from_files("rsv/invalid_gratia2.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Invalid gratia probe list ignored")
Ejemplo n.º 29
0
    def testInvalidGratiaProbes(self):
        """
        Test the check_attributes with invalid gratia probes
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        settings = self.load_settings_from_files("rsv/invalid_gratia1.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Invalid gratia probe ignored")

        settings = self.load_settings_from_files("rsv/invalid_gratia2.ini")
        attributes = settings.get_attributes()
        self.assertFalse(settings.check_attributes(attributes),
                         "Invalid gratia probe list ignored")
Ejemplo n.º 30
0
    def testMissingAttribute(self):
        """
        Test the parsing when attributes are missing, should get exceptions
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        config_file = get_test_config("rsv/rsv2.ini")
        configuration = ConfigParser.SafeConfigParser()
        configuration.read(config_file)

        settings = rsv.RsvConfiguration(logger=global_logger)
        settings.rsv_meta_dir = RSV_META_DIR
        try:
            settings.parse_configuration(configuration)
        except Exception, e:
            self.fail("Received exception while parsing configuration: %s" % e)
Ejemplo n.º 31
0
    def testMissingAttribute(self):
        """
        Test the parsing when attributes are missing, should get exceptions
        """

        # need to have rsv installed to get rsv tests working
        if not utilities.rpm_installed('rsv-core'):
            return
        config_file = get_test_config("rsv/rsv2.ini")
        configuration = ConfigParser.SafeConfigParser()
        configuration.read(config_file)

        settings = rsv.RsvConfiguration(logger=global_logger)
        settings.rsv_meta_dir = RSV_META_DIR
        try:
            settings.parse_configuration(configuration)
        except Exception, e:
            self.fail("Received exception while parsing configuration: %s" % e)
Ejemplo n.º 32
0
    def _configure_consumers(self):
        """ Enable the appropriate consumers """

        # The current logic is:
        #  - we ALWAYS want the html-consumer if we are told to install consumers
        #  - we NEVER want the gratia-consumer
        #  - we want the nagios-consumer if enable_nagios is True
        #  - we want the zabbix-consumer if enable_zabbix is True and rsv-consumers-zabbix is installed

        consumers = ["html-consumer"]

        if self.opt_val("enable_gratia"):
            self.log(
                "Your configuration has enabled the Gratia consumer but the service which the Gratia consumer "
                "reports to has been shut down. Please turn off 'enable_gratia' in the RSV section. "
                "Gratia consumer configuration will be ignored.",
                level=logging.WARNING)

        if self.options['enable_nagios'].value:
            consumers.append("nagios-consumer")
            self._configure_nagios_files()

        if self.options['enable_zabbix'].value:
            if not utilities.rpm_installed('rsv-consumers-zabbix'):
                self.log(
                    'Your configuration has enabled the Zabbix consumer '
                    'but rsv-consumers-zabbix is not installed. Zabbix consumer configuration will be ignored.',
                    level=logging.WARNING)
            else:
                consumers.append("zabbix-consumer")
                self._configure_zabbix_files()

        consumer_list = " ".join(consumers)
        self.log("Enabling consumers: %s " % consumer_list)

        if not utilities.run_script([self.rsv_control, "-v0", "--enable"] +
                                    consumers):
            raise exceptions.ConfigureError
        utilities.run_script(
            [self.rsv_control, "-v0", "--disable",
             "gratia-consumer"])  # don't care if this fails
Ejemplo n.º 33
0
    def parse_configuration(self, configuration):
        """
        Try to get configuration information from ConfigParser or
        SafeConfigParser object given by configuration and write recognized settings
        to attributes dict
        """
        self.log('RsvConfiguration.parse_configuration started')

        self.check_config(configuration)

        if not configuration.has_section(self.config_section):
            self.enabled = False
            self.log("%s section not in config file" % self.config_section)
            self.log('RsvConfiguration.parse_configuration completed')
            return True

        if not utilities.rpm_installed('rsv-core'):
            self.enabled = False
            self.log('rsv-core rpm not installed, disabling RSV configuration')
            return True

        if not self.set_status(configuration):
            self.log('RsvConfiguration.parse_configuration completed')
            return True

        self.get_options(configuration,
                         ignore_options=['enabled', 'gratia_collector'])

        # If we're on a CE, get the grid group if possible
        if configuration.has_section('Site Information'):
            if configuration.has_option('Site Information', 'group'):
                self.grid_group = configuration.get('Site Information',
                                                    'group')

            if configuration.has_option('Site Information', 'resource'):
                self.site_name = configuration.get('Site Information',
                                                   'resource')
            elif configuration.has_option('Site Information', 'site_name'):
                self.site_name = configuration.get('Site Information',
                                                   'site_name')

        # Parse lists
        self._ce_hosts = split_list_exclude_blank(
            self.options['ce_hosts'].value)
        self._htcondor_ce_hosts = split_list_exclude_blank(
            self.options['htcondor_ce_hosts'].value)
        self._srm_hosts = split_list_exclude_blank(
            self.options['srm_hosts'].value)

        # If the gridftp hosts are not defined then they default to the CE hosts
        if self.options['gridftp_hosts'].value == '':
            # check to see if the setting is in the config file
            if configuration.has_option(self.config_section, 'gridftp_hosts'):
                # present and set to default so we don't want gridftp tests
                self._gridftp_hosts = []
            else:
                # option is commented out, use ce_hosts setting
                self._gridftp_hosts = self._ce_hosts
        else:
            self._gridftp_hosts = split_list(
                self.options['gridftp_hosts'].value)

        if self.options['gratia_probes'].value != '':
            self._gratia_probes_2d = self.split_2d_list(
                self.options['gratia_probes'].value)

        # Check the options for which gateways are enabled
        # How we run remote probes depends on this
        if configuration.has_section('Gateway'):
            if configuration.has_option('Gateway', 'htcondor_gateway_enabled'):
                self.htcondor_gateway_enabled = configuration.getboolean(
                    'Gateway', 'htcondor_gateway_enabled')

        if configuration.has_section('Misc Services'):
            if configuration.has_option('Misc Services',
                                        'copy_host_cert_for_service_certs'):
                self.copy_host_cert_for_service_cert = configuration.getboolean(
                    'Misc Services', 'copy_host_cert_for_service_certs')

        self.log('RsvConfiguration.parse_configuration completed')
Ejemplo n.º 34
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log("SquidConfiguration.check_attributes started")
        attributes_ok = True
        if not (utilities.gateway_installed() and utilities.rpm_installed("frontier-squid")):
            return attributes_ok

        if not self.enabled:
            self.log(
                "Squid is not enabled, sites must enable this \n"
                + "section, location can be set to UNAVAILABLE if squid is \n"
                + "not present",
                level=logging.WARNING,
            )
            self.log("squid not enabled")
            self.log("SquidConfiguration.check_attributes completed")
            return attributes_ok

        if self.ignored:
            self.log("Ignored, returning True")
            self.log("SquidConfiguration.check_attributes completed")
            return attributes_ok

        if self.options["location"].value == "None":
            self.log(
                "location setting must be set, if site does not provide " + "squid, please use UNAVAILABLE",
                section=self.config_section,
                option="location",
                level=logging.WARNING,
            )
            return attributes_ok

        if self.options["location"].value.upper() == "UNAVAILABLE":
            self.log(
                "Squid location is set to UNAVAILABLE.  Although this is \n"
                + "allowed, most jobs function better and use less bandwidth \n"
                + "when a squid proxy is available",
                level=logging.WARN,
            )
            self.options["location"].value = "None"
            return attributes_ok

        if len(self.options["location"].value.split(":")) != 2:
            self.log(
                "Bad host specification, got %s expected hostname:port "
                "(e.g. localhost:3128)" % self.options["location"].value,
                section=self.config_section,
                option="location",
                level=logging.ERROR,
            )
            attributes_ok = False
            return attributes_ok
        (hostname, port) = self.options["location"].value.split(":")
        if not validation.valid_domain(hostname, True):
            self.log(
                "Invalid hostname for squid location: %s" % self.options["location"].value,
                section=self.config_section,
                option="location",
                level=logging.ERROR,
            )
            attributes_ok = False
        try:
            int(port)
        except ValueError:
            self.log(
                "The port must be a number(e.g. host:3128) for squid " "location: %s" % self.options["location"].value,
                section=self.config_section,
                option="location",
                level=logging.ERROR,
                exception=True,
            )
            attributes_ok = False

        self.log("SquidConfiguration.check_attributes completed")
        return attributes_ok
Ejemplo n.º 35
0
    def setup_htcondor_ce_config(self):
        """
        Populate the config file that tells htcondor-ce where the condor
        pool is and where the spool directory is.

        Returns True if successful, False otherwise
        """
        if not utilities.rpm_installed('htcondor-ce'):
            self.log(
                "Unable to configure htcondor-ce for Condor: htcondor-ce not installed",
                level=logging.ERROR)
            return False

        def get_condor_ce_config_val(variable):
            return utilities.get_condor_config_val(
                variable,
                executable='condor_ce_config_val',
                quiet_undefined=True)

        # Get values for the settings we want to update. We can get the
        # values from condor_config_val; in the case of JOB_ROUTER_SCHEDD2_NAME,
        # we have FULL_HOSTNAME as a fallback in case SCHEDD_NAME is missing.
        # We also get the current / default value from condor_ce_config_val;
        # only update the setting in case the value from
        # condor_config_val is different from the value from condor_ce_config_val.
        condor_ce_config = {}
        for condor_ce_config_key, condor_config_keys in [
            ('JOB_ROUTER_SCHEDD2_NAME', ['SCHEDD_NAME', 'FULL_HOSTNAME']),
            ('JOB_ROUTER_SCHEDD2_POOL', ['COLLECTOR_HOST']),
            ('JOB_ROUTER_SCHEDD2_SPOOL', ['SPOOL'])
        ]:

            condor_config_value = None
            for condor_config_value in (utilities.get_condor_config_val(
                    k, quiet_undefined=True) for k in condor_config_keys):
                if condor_config_value:
                    break

            condor_ce_config_value = get_condor_ce_config_val(
                condor_ce_config_key)
            if not (condor_config_value or condor_ce_config_value):
                self.log(
                    "Unable to determine value for %s from %s and default not set; check your Condor config"
                    % (condor_ce_config_key, ' or '.join(condor_config_keys)),
                    level=logging.ERROR)
                return False
            elif not condor_config_value:
                continue  # can't set anything for this

            # Special case for JOB_ROUTER_SCHEDD2_POOL: append port if necessary (SOFTWARE-1744)
            if condor_ce_config_key == 'JOB_ROUTER_SCHEDD2_POOL':
                condor_collector_port = (utilities.get_condor_config_val(
                    'COLLECTOR_PORT', quiet_undefined=True) or '9618')
                condor_config_value = self._add_port_if_necessary(
                    condor_config_value, condor_collector_port)

            if not condor_ce_config_value or condor_ce_config_value != condor_config_value:
                condor_ce_config[condor_ce_config_key] = condor_config_value

        if condor_ce_config:
            buf = utilities.read_file(
                JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE,
                default="# This file is managed by osg-configure\n")
            for key, value in condor_ce_config.items():
                buf = utilities.add_or_replace_setting(buf,
                                                       key,
                                                       value,
                                                       quote_value=False)

            if not utilities.atomic_write(
                    JobManagerConfiguration.HTCONDOR_CE_CONFIG_FILE, buf):
                return False

        return True
Ejemplo n.º 36
0
    def testServiceList(self):
        """
        Test to make sure right services get returned
        """

        config_file = get_test_config("misc/misc_xacml.ini")
        configuration = ConfigParser.SafeConfigParser()
        configuration.read(config_file)

        settings = misc.MiscConfiguration(logger=global_logger)
        try:
            settings.parse_configuration(configuration)
        except Exception, e:
            self.fail("Received exception while parsing configuration: %s" % e)
        services = settings.enabled_services()
        if utilities.rpm_installed('fetch-crl'):
            expected_services = set(['fetch-crl-cron',
                                     'fetch-crl-boot',
                                     'gums-client-cron'])
        elif utilities.rpm_installed('fetch-crl3'):
            expected_services = set(['fetch-crl3-cron',
                                     'fetch-crl3-boot',
                                     'gums-client-cron'])
        else:
            expected_services = set(['gums-client-cron'])

        self.assertEqual(services, expected_services,
                         "List of enabled services incorrect, " +
                         "got %s but expected %s" % (services, expected_services))

        config_file = get_test_config("misc/misc_gridmap.ini")
Ejemplo n.º 37
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('SquidConfiguration.check_attributes started')
        attributes_ok = True
        if not (utilities.gateway_installed()
                and utilities.rpm_installed('frontier-squid')):
            return attributes_ok

        if not self.enabled:
            self.log(
                "Squid is not enabled, sites must enable this \n" +
                "section, location can be set to UNAVAILABLE if squid is \n" +
                "not present",
                level=logging.WARNING)
            self.log('squid not enabled')
            self.log('SquidConfiguration.check_attributes completed')
            return attributes_ok

        if self.ignored:
            self.log('Ignored, returning True')
            self.log('SquidConfiguration.check_attributes completed')
            return attributes_ok

        if (self.options['location'].value == 'None'):
            self.log(
                "location setting must be set, if site does not provide " +
                "squid, please use UNAVAILABLE",
                section=self.config_section,
                option='location',
                level=logging.WARNING)
            return attributes_ok

        if (self.options['location'].value.upper() == 'UNAVAILABLE'):
            self.log(
                "Squid location is set to UNAVAILABLE.  Although this is \n" +
                "allowed, most jobs function better and use less bandwidth \n"
                + "when a squid proxy is available",
                level=logging.WARN)
            self.options['location'].value = 'None'
            return attributes_ok

        if len(self.options['location'].value.split(':')) != 2:
            self.log("Bad host specification, got %s expected hostname:port " \
                     "(e.g. localhost:3128)" % self.options['location'].value,
                     section=self.config_section,
                     option='location',
                     level=logging.ERROR)
            attributes_ok = False
            return attributes_ok
        (hostname, port) = self.options['location'].value.split(':')
        if not validation.valid_domain(hostname, True):
            self.log("Invalid hostname for squid location: %s" % \
                     self.options['location'].value,
                     section=self.config_section,
                     option='location',
                     level=logging.ERROR)
            attributes_ok = False
        try:
            int(port)
        except ValueError:
            self.log("The port must be a number(e.g. host:3128) for squid " \
                     "location: %s" % self.options['location'].value,
                     section=self.config_section,
                     option='location',
                     level=logging.ERROR,
                     exception=True)
            attributes_ok = False

        self.log('SquidConfiguration.check_attributes completed')
        return attributes_ok
Ejemplo n.º 38
0
    def testServiceList(self):
        """
        Test to make sure right services get returned
        """

        config_file = get_test_config("misc/misc_xacml.ini")
        configuration = ConfigParser.SafeConfigParser()
        configuration.read(config_file)

        settings = misc.MiscConfiguration(logger=global_logger)
        try:
            settings.parse_configuration(configuration)
        except Exception, e:
            self.fail("Received exception while parsing configuration: %s" % e)
        services = settings.enabled_services()
        if utilities.rpm_installed('fetch-crl'):
            expected_services = set(
                ['fetch-crl-cron', 'fetch-crl-boot', 'gums-client-cron'])
        elif utilities.rpm_installed('fetch-crl3'):
            expected_services = set(
                ['fetch-crl3-cron', 'fetch-crl3-boot', 'gums-client-cron'])
        else:
            expected_services = set(['gums-client-cron'])

        self.assertEqual(
            services, expected_services,
            "List of enabled services incorrect, " + "got %s but expected %s" %
            (services, expected_services))

        config_file = get_test_config("misc/misc_gridmap.ini")
        configuration = ConfigParser.SafeConfigParser()