Beispiel #1
0
    def configure(self, attributes):
        """
        Setup basic osg/vdt services
        """

        self.log("NetworkConfiguration.configure started")
        status = True

        header = "# This file is automatically generated by osg-configure\n"
        header += "# based on the settings in the [Network] section, please\n"
        header += "# make changes there instead of manually editing this file\n"
        source_settings_sh = ''
        source_settings_csh = ''
        port_settings_sh = ''
        port_settings_csh = ''
        if not utilities.blank(self.options['source_range'].value):
            source_settings_sh = "export GLOBUS_TCP_SOURCE_RANGE_STATE_FILE=%s\n" % \
                                 self.options['source_state_file'].value
            source_settings_sh += "export GLOBUS_TCP_SOURCE_RANGE=%s\n" % \
                                  self.options['source_range'].value
            source_settings_csh = "setenv GLOBUS_TCP_SOURCE_RANGE_STATE_FILE %s\n" % \
                                  self.options['source_state_file'].value
            source_settings_csh += "setenv GLOBUS_TCP_SOURCE_RANGE %s\n" % \
                                   self.options['source_range'].value
        if not utilities.blank(self.options['port_range'].value):
            port_settings_sh = "export GLOBUS_TCP_PORT_RANGE_STATE_FILE=%s\n" % \
                               self.options['port_state_file'].value
            port_settings_sh += "export GLOBUS_TCP_PORT_RANGE=%s\n" % \
                                self.options['port_range'].value
            port_settings_csh = "setenv GLOBUS_TCP_PORT_RANGE_STATE_FILE %s\n" % \
                                self.options['port_state_file'].value
            port_settings_csh += "setenv GLOBUS_TCP_PORT_RANGE %s\n" % \
                                 self.options['port_range'].value
        contents = "#!/bin/sh\n" + header + source_settings_sh + port_settings_sh
        filename = os.path.join('/', 'var', 'lib', 'osg', 'globus-firewall')
        if not utilities.atomic_write(filename, contents):
            self.log("Error writing to %s" % filename,
                     level=logging.ERROR)
            status = False
        filename = os.path.join('/', 'etc', 'profile.d', 'osg.sh')
        if not utilities.atomic_write(filename, contents):
            self.log("Error writing to %s" % filename,
                     level=logging.ERROR)
            status = False
        contents = "#!/bin/csh\n" + header + source_settings_csh + port_settings_csh
        filename = os.path.join('/', 'etc', 'profile.d', 'osg.csh')
        if not utilities.atomic_write(filename, contents):
            self.log("Error writing to %s" % filename,
                     level=logging.ERROR)
            status = False

        self.log("NetworkConfiguration.configure completed")
        return status
Beispiel #2
0
    def _check_srm_settings(self):
        """
        Check srm settings to make sure settings are consistent and properly
        set
        """
        if (self._srm_hosts == [] or self._srm_hosts is None
                or utilities.blank(self.options['srm_hosts'].value)):
            return True

        all_ok = True
        if self.options['srm_dir'].value.upper() == 'DEFAULT':
            self.log(
                "srm_dir has to be set and can't be set to DEFAULT for each " +
                "srm host defined (set to %s)" % dir,
                option='srm_dir',
                section='rsv',
                level=logging.ERROR)
            all_ok = False

        srm_dirs = split_list(self.options['srm_dir'].value)
        if len(self._srm_hosts) != len(srm_dirs):
            self.log("When enabling SRM metrics you must specify the same number " +
                     "of entries in the srm_dir variable as you have in the " +
                     "srm_hosts section.  There are %i host entries and %i " \
                     "srm_dir entries." % (len(self._srm_hosts), len(srm_dirs)),
                     level=logging.ERROR)
            all_ok = False

        for directory in srm_dirs:
            if directory.upper() == 'DEFAULT':
                self.log(
                    "srm_dir has to be set and can't be set to DEFAULT for each "
                    + "srm host defined (set to %s)" % directory,
                    option='srm_dir',
                    section='rsv',
                    level=logging.ERROR)
                all_ok = False

        if not utilities.blank(self.options['srm_webservice_path'].value):
            srm_ws_paths = split_list(
                self.options['srm_webservice_path'].value)
            if len(self._srm_hosts) != len(srm_ws_paths):
                self.log("If you set srm_webservice_path when enabling SRM metrics " +
                         "you must specify the same number of entries in the " +
                         "srm_webservice_path variable as you have in the srm_hosts " +
                         "section.  There are %i host entries and %i " \
                         "srm_webservice_path entries." % (len(self._srm_hosts),
                                                           len(srm_ws_paths)),
                         level=logging.ERROR)
                all_ok = False

        return all_ok
    def test_blank(self):
        """
        Test functionality of blank function
        """

        self.assertFalse(utilities.blank(1),
                         'blank indicated 1 was a blank value')
        self.assertFalse(utilities.blank('a'),
                         'blank indicated a was a blank value')
        self.assertTrue(utilities.blank('unavailable'),
                        'blank did not indicate unavailable was a blank value')
        self.assertTrue(utilities.blank(None),
                        'blank did not indicate None was a blank value')
        self.assertTrue(utilities.blank('unavAilablE'),
                        'blank did not indicate unavAilablE was a blank value')
    def test_blank(self):
        """
        Test functionality of blank function
        """

        self.assertFalse(utilities.blank(1),
                         'blank indicated 1 was a blank value')
        self.assertFalse(utilities.blank('a'),
                         'blank indicated a was a blank value')
        self.assertTrue(utilities.blank('unavailable'),
                        'blank did not indicate unavailable was a blank value')
        self.assertTrue(utilities.blank(None),
                        'blank did not indicate None was a blank value')
        self.assertTrue(utilities.blank('unavAilablE'),
                        'blank did not indicate unavAilablE was a blank value')
    def get_attributes(self, converter=str):
        """
        Get attributes for the osg attributes file using the dict in self.options

        Returns a dictionary of ATTRIBUTE => value mappings

        Need to override parent class method since two options may map to OSG_SITE_NAME
        """

        self.log("%s.get_attributes started" % self.__class__)

        attributes = BaseConfiguration.get_attributes(self, converter)
        if attributes == {}:
            self.log("%s.get_attributes completed" % self.__class__)
            return attributes

        if (
            "OSG_SITE_NAME" in attributes
            and self.options["resource"].value is not None
            and not utilities.blank(self.options["resource"].value)
        ):
            attributes["OSG_SITE_NAME"] = self.options["resource"].value

        self.log("%s.get_attributes completed" % self.__class__)
        return attributes
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log("StorageConfiguration.check_attributes started")
        attributes_ok = True

        if not self.enabled:
            self.log("Not enabled, returning True")
            self.log("StorageConfiguration.check_attributes completed")
            return attributes_ok

        # make sure locations exist
        if not self._check_app_dir(self.options["app_dir"].value):
            self.log(
                "The app_dir and app_dir/etc directories should exist and "
                + "have permissions of 1777 or 777 on OSG installations.",
                section=self.config_section,
                option="app_dir",
                level=logging.ERROR,
            )
            attributes_ok = False

        # WN_TMP may be blank if the job manager dynamically generates it but
        # warni just in case
        if utilities.blank(self.options["worker_node_temp"].value):
            self.log(
                "worker_node_temp is blank, this is okay if you've set your "
                + "job manager to set this dynamically, otherwise jobs may "
                + "fail to run",
                section=self.config_section,
                option="worker_node_temp",
                level=logging.WARNING,
            )
        self.log("StorageConfiguration.check_attributes completed")
        return attributes_ok
Beispiel #7
0
    def parse_configuration(self, configuration):
        """Try to get configuration information from ConfigParser or SafeConfigParser object given
        by configuration and write recognized settings to attributes dict
        """
        self.log("SquidConfiguration.parse_configuration started")

        self.check_config(configuration)

        if not configuration.has_section(self.config_section):
            self.enabled = False
            self.log("%s section not in config file" % self.config_section)
            self.log("SquidConfiguration.parse_configuration completed")
            return

        if not self.set_status(configuration):
            self.log("SquidConfiguration.parse_configuration completed")
            if not self.ignored and not self.enabled:
                return True

        self.get_options(configuration, ignore_options=["enabled"])

        if not (utilities.blank(self.options["location"].value) or self.options["location"].value == "None"):
            if ":" not in self.options["location"].value:
                self.options["location"].value += ":3128"

        if configuration.get(self.config_section, "location").upper() == "UNAVAILABLE":
            self.options["location"].value = "UNAVAILABLE"
        self.log("SquidConfiguration.parse_configuration completed")
Beispiel #8
0
    def parse_configuration(self, configuration):
        """Try to get configuration information from ConfigParser or SafeConfigParser object given
        by configuration and write recognized settings to attributes dict
        """
        self.log('SquidConfiguration.parse_configuration started')

        self.check_config(configuration)

        if not configuration.has_section(self.config_section):
            self.enabled = False
            self.log("%s section not in config file" % self.config_section)
            self.log('SquidConfiguration.parse_configuration completed')
            return

        if not self.set_status(configuration):
            self.log('SquidConfiguration.parse_configuration completed')
            if not self.ignored and not self.enabled:
                return True

        self.get_options(configuration, ignore_options=['enabled'])

        if not (utilities.blank(self.options['location'].value)
                or self.options['location'].value == 'None'):
            if ":" not in self.options['location'].value:
                self.options['location'].value += ":3128"

        if configuration.get(self.config_section,
                             'location').upper() == 'UNAVAILABLE':
            self.options['location'].value = 'UNAVAILABLE'
        self.log('SquidConfiguration.parse_configuration completed')
Beispiel #9
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('StorageConfiguration.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('StorageConfiguration.check_attributes completed')
            return attributes_ok

        # warn if locations don't exist
        app_dir = self.options['app_dir'].value
        if not self._check_app_dir(app_dir):
            self.log(
                "app_dir is used for $OSG_APP and $OSG_APP/etc on worker nodes, where they should exist and"
                " have permissions of 1777 or 777.",
                level=logging.WARNING)

        # WN_TMP may be blank if the job manager dynamically generates it but
        # warni just in case
        if utilities.blank(self.options['worker_node_temp'].value):
            self.log(
                "worker_node_temp is blank, this is okay if you've set your " +
                "job manager to set this dynamically, otherwise jobs may " +
                "fail to run",
                section=self.config_section,
                option='worker_node_temp',
                level=logging.WARNING)
        self.log('StorageConfiguration.check_attributes completed')
        return attributes_ok
Beispiel #10
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('StorageConfiguration.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('StorageConfiguration.check_attributes completed')
            return attributes_ok

        # make sure locations exist
        if not self._check_app_dir(self.options['app_dir'].value):
            self.log("The app_dir and app_dir/etc directories should exist and " +
                     "have permissions of 1777 or 777 on OSG installations.",
                     section=self.config_section,
                     option='app_dir',
                     level=logging.ERROR)
            attributes_ok = False

        # WN_TMP may be blank if the job manager dynamically generates it but
        # warni just in case
        if utilities.blank(self.options['worker_node_temp'].value):
            self.log("worker_node_temp is blank, this is okay if you've set your " +
                     "job manager to set this dynamically, otherwise jobs may " +
                     "fail to run",
                     section=self.config_section,
                     option='worker_node_temp',
                     level=logging.WARNING)
        self.log('StorageConfiguration.check_attributes completed')
        return attributes_ok
Beispiel #11
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('MiscConfiguration.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('MiscConfiguration.check_attributes completed')
            return True

        if (self.options['authorization_method'].value not in \
                    ['gridmap', 'local-gridmap', 'xacml']):
            self.log("Setting is not xacml, or gridmap",
                     option='authorization_method',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        if self.options['authorization_method'].value == 'xacml':
            if utilities.blank(self.options['gums_host'].value):
                self.log("Gums host not given",
                         section=self.config_section,
                         option='gums_host',
                         level=logging.ERROR)
                attributes_ok = False

            if not validation.valid_domain(self.options['gums_host'].value, resolve=True):
                self.log("Gums host not a valid domain name or does not resolve",
                         section=self.config_section,
                         option='gums_host',
                         level=logging.ERROR)
                attributes_ok = False

        self.log('MiscConfiguration.check_attributes completed')
        return attributes_ok
Beispiel #12
0
    def parse_configuration(self, configuration):
        """
        Try to get configuration information from ConfigParser or SafeConfigParser
        object given by configuration and write recognized settings to options dict
        """
        self.log('MiscConfiguration.parse_configuration started')

        self.check_config(configuration)

        if not configuration.has_section(self.config_section):
            self.enabled = False
            self.log("%s section not in config files" % self.config_section)
            self.log('MiscConfiguration.parse_configuration completed')
            return

        self.enabled = True
        self.get_options(configuration, ignore_options=IGNORED_OPTIONS)

        self.htcondor_gateway_enabled = utilities.config_safe_getboolean(
            configuration, 'Gateway', 'htcondor_gateway_enabled', True)
        self.authorization_method = self.options['authorization_method'].value
        self.using_glexec = not utilities.blank(
            self.options['glexec_location'].value)
        self.all_fqans = self.options['all_fqans'].value

        self.log('MiscConfiguration.parse_configuration completed')
Beispiel #13
0
    def _parse_configuration(self, configuration):
        """
        The meat of parse_configuration, runs after we've checked that GIP is
        enabled and we have the right RPMs installed.
        """
        # The following is to set the user that gip files need to belong to
        # This can be overridden by setting the 'user' option in the [GIP] section
        self.gip_user = '******'
        if configuration.has_option(self.config_section, 'batch'):
            batch_opt = configuration.get(self.config_section, 'batch').lower()
            if (not utilities.blank(batch_opt)
                    and batch_opt not in self._valid_batch_opt):
                msg = "The batch setting in %s must be a valid option " \
                      "(e.g. %s), %s was given" % (self.config_section,
                                                   ",".join(self._valid_batch_opt),
                                                   batch_opt)
                self.log(msg, level=logging.ERROR)
                raise exceptions.SettingError(msg)

        if utilities.ce_installed():
            self._parse_configuration_ce(configuration)

        # Check for the presence of the classic SE
        has_classic_se = True
        try:
            has_classic_se = configuration.getboolean("GIP",
                                                      "advertise_gsiftp")
        # pylint: disable-msg=W0702
        # pylint: disable-msg=W0703
        # pylint: disable-msg=W0704
        except Exception:
            pass

        has_se = False
        for section in configuration.sections():
            if not section.lower().startswith('se'):
                continue
            has_se = True
            self.check_se(configuration, section)
        if not has_se and not has_classic_se:
            try:
                self._check_entry(configuration, "GIP", "se_name", REQUIRED,
                                  STRING)
            except:
                msg = "There is no `SE` section, the old-style SE" + \
                      "setup in GIP is not configured, and there is no classic SE. " + \
                      " At least one must be configured.  Please see the configuration" \
                      " documentation."
                raise exceptions.SettingError(msg)
        if configuration.has_option(self.config_section, 'user'):
            username = configuration.get(self.config_section, 'user')
            if not validation.valid_user(username):
                err_msg = "%s is not a valid account on this system" % username
                self.log(err_msg,
                         section=self.config_section,
                         option='user',
                         level=logging.ERROR)
                raise exceptions.SettingError(err_msg)
            self.gip_user = username
Beispiel #14
0
    def _create_cert_key_if_needed(self):
        if not self.copy_host_cert_for_service_cert:
            # User explicitly told us not to make a copy
            return
        service_cert = self.options['service_cert'].value
        service_key = self.options['service_key'].value
        if utilities.blank(service_cert) or utilities.blank(service_key):
            # cert/key location not specified so don't make anything
            return

        if not self.create_missing_service_cert_key(service_cert, service_key,
                                                    'rsv'):
            # creation unsuccessful
            self.log("Could not create service cert (%s) and key (%s)" %
                     (service_cert, service_key),
                     level=logging.ERROR)
            raise exceptions.ConfigureError
Beispiel #15
0
    def _parse_configuration(self, configuration):
        """
        The meat of parse_configuration, runs after we've checked that GIP is
        enabled and we have the right RPMs installed.
        """
        # The following is to set the user that gip files need to belong to
        # This can be overridden by setting the 'user' option in the [GIP] section
        self.gip_user = '******'
        if configuration.has_option(self.config_section, 'batch'):
            batch_opt = configuration.get(self.config_section, 'batch').lower()
            if (not utilities.blank(batch_opt) and
                        batch_opt not in self._valid_batch_opt):
                msg = "The batch setting in %s must be a valid option " \
                      "(e.g. %s), %s was given" % (self.config_section,
                                                   ",".join(self._valid_batch_opt),
                                                   batch_opt)
                self.log(msg, level=logging.ERROR)
                raise exceptions.SettingError(msg)

        if utilities.ce_installed():
            self._parse_configuration_ce(configuration)

        # Check for the presence of the classic SE
        has_classic_se = True
        try:
            has_classic_se = configuration.getboolean("GIP", "advertise_gsiftp")
        # pylint: disable-msg=W0702
        # pylint: disable-msg=W0703
        # pylint: disable-msg=W0704
        except Exception:
            pass

        has_se = False
        for section in configuration.sections():
            if not section.lower().startswith('se'):
                continue
            has_se = True
            self.check_se(configuration, section)
        if not has_se and not has_classic_se:
            try:
                self._check_entry(configuration, "GIP", "se_name", REQUIRED, STRING)
            except:
                msg = "There is no `SE` section, the old-style SE" + \
                      "setup in GIP is not configured, and there is no classic SE. " + \
                      " At least one must be configured.  Please see the configuration" \
                      " documentation."
                raise exceptions.SettingError(msg)
        if configuration.has_option(self.config_section, 'user'):
            username = configuration.get(self.config_section, 'user')
            if not validation.valid_user(username):
                err_msg = "%s is not a valid account on this system" % username
                self.log(err_msg,
                         section=self.config_section,
                         option='user',
                         level=logging.ERROR)
                raise exceptions.SettingError(err_msg)
            self.gip_user = username
Beispiel #16
0
def split_list(item_list):
    """ Split a comma separated list of items """

    # Special case - when the list just contains UNAVAILABLE we want to ignore it
    if utilities.blank(item_list):
        return []

    items = []
    for entry in item_list.split(','):
        items.append(entry.strip())

    return items
Beispiel #17
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('MiscConfiguration.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('MiscConfiguration.check_attributes completed')
            return True

        if self.authorization_method not in VALID_AUTH_METHODS:
            self.log("Setting is not one of: " + ", ".join(VALID_AUTH_METHODS),
                     option='authorization_method',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        if self.authorization_method == 'xacml':
            gums_host = self.options['gums_host'].value
            if utilities.blank(gums_host):
                self.log("Gums host not given",
                         section=self.config_section,
                         option='gums_host',
                         level=logging.ERROR)
                attributes_ok = False
            elif not validation.valid_domain(gums_host, resolve=True):
                self.log(
                    "Gums host not a valid domain name or does not resolve",
                    section=self.config_section,
                    option='gums_host',
                    level=logging.ERROR)
                attributes_ok = False
            self.log(
                "Gums is deprecated in OSG 3.4. The replacement is the LCMAPS VOMS plugin; please see"
                " installation instructions at"
                " https://opensciencegrid.github.io/docs/security/lcmaps-voms-authentication/",
                section=self.config_section,
                option='authorization_method',
                level=logging.WARNING)

        if self.using_glexec:
            msg = "glExec is not supported in OSG 3.4; unset glexec_location"
            self.log(msg,
                     options='glexec_location',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        self.log('MiscConfiguration.check_attributes completed')
        return attributes_ok
Beispiel #18
0
    def _check_gridftp_settings(self):
        """ Check gridftp settings and make sure they are valid """
        status_check = self._validate_host_list(self._gridftp_hosts,
                                                "gridftp_hosts")

        if utilities.blank(self.options['gridftp_dir'].value):
            self.log("Invalid gridftp_dir given: %s" %
                     self.options['gridftp_dir'].value,
                     section=self.config_section,
                     option='gridftp_dir',
                     level=logging.ERROR)
            status_check = False

        return status_check
Beispiel #19
0
def get_option(config, section, option=None):
    """
    Get an option from a config file with optional defaults and mandatory
    options.

    Arguments
    config  -- a ConfigParser object to query
    section --  the ini section the option is located in
    option  --  an Option object to information on the option to retrieve

    """

    if option is None:
        raise exceptions.SettingError('No option passed to get_option')

    if config.has_option(section, option.name):
        try:

            if not utilities.blank(config.get(section, option.name)):
                if option.opt_type == bool:
                    option.value = config.getboolean(section, option.name)
                elif option.opt_type == int:
                    option.value = config.getint(section, option.name)
                elif option.opt_type == float:
                    option.value = config.getfloat(section, option.name)
                else:
                    option.value = config.get(section, option.name)
            else:
                # if option is blank and there's a default for the option
                # return the default if possible, otherwise raise an exception
                # if the option is mandatory

                if (option.required == Option.MANDATORY and
                            option.default_value is None):
                    raise exceptions.SettingError("Can't get value for %s in %s " \
                                                  "section and no default given" % \
                                                  (option.name, section))
                option.value = option.default_value


        except ValueError:
            error_mesg = "%s  in %s section is of the wrong type" % (option.name, section)
            raise exceptions.SettingError(error_mesg)
    elif option.required == Option.MANDATORY:
        err_mesg = "Can't get value for %s in %s section" % (option.name, section)
        raise exceptions.SettingError(err_mesg)
    else:
        option.value = option.default_value
Beispiel #20
0
def get_option(config, section, option=None):
    """
    Get an option from a config file with optional defaults and mandatory
    options.

    Arguments
    config  -- a ConfigParser object to query
    section --  the ini section the option is located in
    option  --  an Option object to information on the option to retrieve

    """

    if option is None:
        raise exceptions.SettingError('No option passed to get_option')

    if config.has_option(section, option.name):
        try:

            if not utilities.blank(config.get(section, option.name)):
                if option.opt_type == bool:
                    option.value = config.getboolean(section, option.name)
                elif option.opt_type == int:
                    option.value = config.getint(section, option.name)
                elif option.opt_type == float:
                    option.value = config.getfloat(section, option.name)
                else:
                    option.value = config.get(section, option.name)
            else:
                # if option is blank and there's a default for the option
                # return the default if possible, otherwise raise an exception
                # if the option is mandatory

                if (option.required == Option.MANDATORY and
                            option.default_value is None):
                    raise exceptions.SettingError("Can't get value for %s in %s " \
                                                  "section and no default given" % \
                                                  (option.name, section))
                option.value = option.default_value


        except ValueError:
            error_mesg = "%s  in %s section is of the wrong type" % (option.name, section)
            raise exceptions.SettingError(error_mesg)
    elif option.required == Option.MANDATORY:
        err_mesg = "Can't get value for %s in %s section" % (option.name, section)
        raise exceptions.SettingError(err_mesg)
    else:
        option.value = option.default_value
Beispiel #21
0
    def _configure_srm_metrics(self):
        """ Enable SRM metric """

        if not self._srm_hosts:
            self.log("No srm_hosts defined.  Not configuring SRM metrics")
            return

        # Do some checking on the values.  perhaps this should be in the validate section?
        srm_dirs = split_list(self.options['srm_dir'].value)
        if len(self._srm_hosts) != len(srm_dirs):
            self.log("When enabling SRM metrics you must specify the same number " +
                     "of entries in the srm_dir variable as you have in the " +
                     "srm_hosts section.  There are %i host entries and %i " \
                     "srm_dir entries." % (len(self._srm_hosts), len(srm_dirs)),
                     level=logging.ERROR)
            raise exceptions.ConfigureError("Failed to configure RSV")

        srm_ws_paths = []
        if not utilities.blank(self.options['srm_webservice_path'].value):
            srm_ws_paths = split_list(
                self.options['srm_webservice_path'].value)

            if len(self._srm_hosts) != len(srm_ws_paths):
                self.log("If you set srm_webservice_path when enabling SRM metrics " +
                         "you must specify the same number of entries in the " +
                         "srm_webservice_path variable as you have in the srm_hosts " +
                         "section.  There are %i host entries and %i " \
                         "srm_webservice_path entries." % (len(self._srm_hosts),
                                                           len(srm_ws_paths)),
                         level=logging.ERROR)
                raise exceptions.ConfigureError("Failed to configure RSV")

        # Now time to do the actual configuration
        srm_metrics = self._get_metrics_by_type("OSG-SRM")
        count = 0
        for srm_host in self._srm_hosts:
            self.log("Enabling SRM metrics for host '%s'" % srm_host)

            args = ["--arg", "srm-destination-dir=%s" % srm_dirs[count]]
            if srm_ws_paths:
                args += [
                    "--arg",
                    "srm-webservice-path=%s" % srm_ws_paths[count]
                ]

            self._enable_metrics(srm_host, srm_metrics, args)

            count += 1
Beispiel #22
0
    def setup_gram_config(self):
        """
        Populate the gram config file with correct values

        Returns True if successful, False otherwise
        """

        buf = open(CondorConfiguration.GRAM_CONFIG_FILE).read()
        for binfile in ['condor_submit', 'condor_rm']:
            bin_location = os.path.join(self.condor_bin_location, binfile)
            if validation.valid_file(bin_location):
                buf = utilities.add_or_replace_setting(buf, binfile, bin_location)
        if not utilities.blank(self.options['condor_config'].value):
            buf = utilities.add_or_replace_setting(buf, 'condor_config', self.options['condor_config'].value)

        if not utilities.atomic_write(CondorConfiguration.GRAM_CONFIG_FILE, buf):
            return False

        return True
Beispiel #23
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('NetworkConfiguration.check_attributes started')
        attributes_ok = True

        for name in ['source_state_file', 'port_state_file']:
            if utilities.blank(self.options[name].value):
                continue

            if not validation.valid_location(self.options[name].value):
                self.log("File is not present: %s" % self.options[name].value,
                         option=name,
                         section=self.config_section,
                         level=logging.WARNING)

        for name in ['source_range', 'port_range']:
            if utilities.blank(self.options[name].value):
                continue

            matches = re.match(r'(\d+),(\d+)', self.options[name].value)
            if matches is None:
                self.log("Invalid range specification, expected low_port,high_port, " +
                         "got %s" % self.options[name].value,
                         option=name,
                         section=self.config_section,
                         level=logging.ERROR)
                attributes_ok = False

        if (not utilities.blank(self.options['source_state_file'].value) and
                utilities.blank(self.options['source_range'].value)):
            self.log("If you specify a source_state_file, " +
                     "source_range must be given",
                     option='source_state_file',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        if (not utilities.blank(self.options['port_state_file'].value) and
                utilities.blank(self.options['port_range'].value)):
            self.log("If you specify a port_state_file, " +
                     "port_range must be given",
                     option='port_state_file',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        self.log('NetworkConfiguration.check_attributes completed')
        return attributes_ok
Beispiel #24
0
    def setup_gram_config(self):
        """
        Populate the gram config file with correct values

        Returns True if successful, False otherwise
        """

        buf = open(CondorConfiguration.GRAM_CONFIG_FILE).read()
        for binfile in ['condor_submit', 'condor_rm']:
            bin_location = os.path.join(self.condor_bin_location, binfile)
            if validation.valid_file(bin_location):
                buf = utilities.add_or_replace_setting(buf, binfile,
                                                       bin_location)
        if not utilities.blank(self.options['condor_config'].value):
            buf = utilities.add_or_replace_setting(
                buf, 'condor_config', self.options['condor_config'].value)

        if not utilities.atomic_write(CondorConfiguration.GRAM_CONFIG_FILE,
                                      buf):
            return False

        return True
    def get_attributes(self, converter=str):
        """
        Get attributes for the osg attributes file using the dict in self.options

        Returns a dictionary of ATTRIBUTE => value mappings

        Need to override parent class method since two options may map to OSG_SITE_NAME
        """

        self.log("%s.get_attributes started" % self.__class__)

        attributes = BaseConfiguration.get_attributes(self, converter)
        if attributes == {}:
            self.log("%s.get_attributes completed" % self.__class__)
            return attributes

        if ('OSG_SITE_NAME' in attributes and
                    self.options['resource'].value is not None and
                not utilities.blank(self.options['resource'].value)):
            attributes['OSG_SITE_NAME'] = self.options['resource'].value

        self.log("%s.get_attributes completed" % self.__class__)
        return attributes
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('SiteInformation.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('SiteInformation.check_attributes completed')
            return attributes_ok

        # OSG_GROUP must be either OSG or OSG-ITB
        group = self.opt_val("group")
        if group not in ('OSG', 'OSG-ITB'):
            self.log("The group setting must be either OSG or OSG-ITB, got: %s" %
                     group,
                     option='group',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        host_name = self.opt_val("host_name")
        # host_name must be a valid dns name, check this by getting it's ip adddress
        if not utilities.blank(host_name) and not validation.valid_domain(host_name, True):
            self.log("hostname %s can't be resolved" % host_name,
                     option='host_name',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        if not utilities.blank(self.opt_val("site_name")):
            self.log("The site_name setting has been deprecated in favor of the"
                     " resource and resource_group settings and will be removed",
                     section=self.config_section,
                     option="site_name",
                     level=logging.WARNING)

        latitude = self.opt_val("latitude")
        if not utilities.blank(latitude) and not -90 < latitude < 90:
            self.log("Latitude must be between -90 and 90, got %s" %
                     latitude,
                     section=self.config_section,
                     option='latitude',
                     level=logging.ERROR)
            attributes_ok = False

        longitude = self.opt_val("longitude")
        if not utilities.blank(longitude) and not -180 < longitude < 180:
            self.log("Longitude must be between -180 and 180, got %s" %
                     longitude,
                     section=self.config_section,
                     option='longitude',
                     level=logging.ERROR)
            attributes_ok = False

        email = self.opt_val("email")
        # make sure the email address has the correct format
        if not utilities.blank(email) and not validation.valid_email(email):
            self.log("Invalid email address in site information: %s" %
                     email,
                     section=self.config_section,
                     option='email',
                     level=logging.ERROR)
            attributes_ok = False

        sponsor = self.opt_val("sponsor")
        if not utilities.blank(sponsor):
            attributes_ok &= self.check_sponsor(sponsor)

        self.log('SiteInformation.check_attributes completed')
        return attributes_ok
Beispiel #27
0
    def parse_configuration(self, configuration):
        """
        Try to get configuration information from ConfigParser or SafeConfigParser
        object given by configuration and write recognized settings to attributes
        dict
        """

        self.log('GratiaConfiguration.parse_configuration started')

        self.check_config(configuration)

        if (not configuration.has_section(self.config_section) and requirements_are_installed()):
            self.log('CE probes installed but no Gratia section, auto-configuring gratia')
            self._auto_configure(configuration)
            self.log('GratiaConfiguration.parse_configuration completed')
            return True
        elif not configuration.has_section(self.config_section):
            self.enabled = False
            self.log("%s section not in config file" % self.config_section)
            self.log('Gratia.parse_configuration completed')
            return

        if not self.set_status(configuration):
            self.log('GratiaConfiguration.parse_configuration completed')
            return True

        # set the appropriate defaults if we're on a CE
        if requirements_are_installed():
            if configuration.has_option('Site Information', 'group'):
                self.grid_group = configuration.get('Site Information', 'group')

            if self.grid_group == 'OSG':
                self.options['probes'].default_value = \
                    self._production_defaults['probes']
            elif self.grid_group == 'OSG-ITB':
                self.options['probes'].default_value = \
                    self._itb_defaults['probes']

            # grab configuration information for various jobmanagers
            probes = self.get_installed_probes()
            for probe in probes:
                if probe == 'condor':
                    self._probe_config['condor'] = {'condor_location':
                                                         CondorConfiguration.get_condor_location(configuration),
                                                     'condor_config':
                                                         CondorConfiguration.get_condor_config(configuration)}
                elif probe == 'pbs':
                    if BaseConfiguration.section_disabled(configuration, 'PBS'):
                        # if the PBS jobmanager is disabled, the CE is probably using LSF
                        # in any case, setting up the pbs gratia probe is not useful
                        continue
                    log_option = configfile.Option(name='log_directory',
                                                   required=configfile.Option.OPTIONAL,
                                                   default_value='')
                    configfile.get_option(configuration, 'PBS', log_option)
                    self._probe_config['pbs'] = {'log_directory': log_option.value}

                    accounting_log_option = configfile.Option(name='accounting_log_directory',
                                                              required=configfile.Option.OPTIONAL,
                                                              default_value='')
                    configfile.get_option(configuration, 'PBS', accounting_log_option)
                    self._probe_config['pbs'] = {'accounting_log_directory': accounting_log_option.value}
                elif probe == 'lsf':
                    if BaseConfiguration.section_disabled(configuration, 'LSF'):
                        # if the LSF jobmanager is disabled, the CE is probably using PBS
                        # in any case, setting up the pbs gratia probe is not useful
                        continue
                    lsf_location = configfile.Option(name='lsf_location',
                                                     default_value='/usr/bin')
                    configfile.get_option(configuration, 'LSF', lsf_location)
                    self._probe_config['lsf'] = {'lsf_location': lsf_location.value}

                    log_option = configfile.Option(name='log_directory',
                                                   required=configfile.Option.OPTIONAL,
                                                   default_value='')
                    configfile.get_option(configuration, 'LSF', log_option)
                    self._probe_config['lsf']['log_directory'] = log_option.value
                elif probe == 'sge':
                    if BaseConfiguration.section_disabled(configuration, 'SGE'):
                        # if section is disabled then the following code won't work
                        # since the parse_configuration will short circuit, so
                        # give a warning and then move on
                        self.log("Skipping SGE gratia probe configuration since SGE is disabled",
                                 level=logging.WARNING)
                        continue
                    sge_config = SGEConfiguration(logger=self.logger)
                    sge_config.parse_configuration(configuration)
                    self._probe_config['sge'] = {'sge_accounting_file': sge_config.get_accounting_file()}
                elif probe == 'slurm':
                    if BaseConfiguration.section_disabled(configuration, 'SLURM'):
                        # if section is disabled then the following code won't work
                        # since the parse_configuration will short circuit, so
                        # give a warning and then move on
                        self.log("Skipping Slurm gratia probe configuration since Slurm is disabled",
                                 level=logging.WARNING)
                        continue
                    slurm_config = SlurmConfiguration(logger=self.logger)
                    slurm_config.parse_configuration(configuration)
                    self._probe_config['slurm'] = {'db_host': slurm_config.get_db_host(),
                                                    'db_port': slurm_config.get_db_port(),
                                                    'db_user': slurm_config.get_db_user(),
                                                    'db_pass': slurm_config.get_db_pass(),
                                                    'db_name': slurm_config.get_db_name(),
                                                    'cluster': slurm_config.get_slurm_cluster(),
                                                    'location': slurm_config.get_location()}

        self.get_options(configuration,
                        ignore_options=['itb-jobmanager-gratia',
                                        'itb-gridftp-gratia',
                                        'osg-jobmanager-gratia',
                                        'osg-gridftp-gratia',
                                        'enabled'])

        if utilities.blank(self.options['probes'].value):
            self.log('GratiaConfiguration.parse_configuration completed')
            return

        self._parse_probes(self.options['probes'].value)
        self.log('GratiaConfiguration.parse_configuration completed')
Beispiel #28
0
    def configure(self, attributes):
        """Configure installation using attributes"""
        self.log("GratiaConfiguration.configure started")

        if self.ignored:
            self.log("%s configuration ignored" % self.config_section,
                     level=logging.WARNING)
            self.log("GratiaConfiguration.configure completed")
            return True

        # disable all gratia services
        # if gratia is enabled, probes will get enabled below
        if not self.enabled:
            self.log("Not enabled")
            self.log("GratiaConfiguration.configure completed")
            return True

        if utilities.blank(self.options['resource'].value):
            if 'OSG_SITE_NAME' not in attributes:
                self.log('No resource found for gratia reporting. You must give it '
                         'using the resource option in the Gratia section or specify '
                         'it in the Site Information section',
                         level=logging.ERROR)
                self.log("GratiaConfiguration.configure completed")
                return False
            else:
                self.options['resource'].value = attributes['OSG_SITE_NAME']

        if 'OSG_HOSTNAME' not in attributes:
            self.log('Hostname of this machine not specified. Please give this '
                     'in the host_name option in the Site Information section',
                     level=logging.ERROR)
            self.log("GratiaConfiguration.configure completed")
            return False

        hostname = attributes['OSG_HOSTNAME']
        probe_list = self.get_installed_probes()
        for probe in probe_list:
            if probe in self._job_managers:
                if probe not in self._probe_config:
                    # Probe is installed but we don't have configuration for it
                    # might be due to pbs-lsf probe sharing or relevant job
                    # manager is not shared
                    continue

                if 'jobmanager' in self.enabled_probe_settings:
                    probe_host = self.enabled_probe_settings['jobmanager']
                else:
                    continue
            else:
                if probe in self.enabled_probe_settings:
                    probe_host = self.enabled_probe_settings[probe]
                else:
                    continue

            self._make_subscription(probe,
                                    probe_list[probe],
                                    probe_host,
                                    self.options['resource'].value,
                                    hostname)
            if probe == 'condor':
                self._configure_condor_probe()
            elif probe == 'pbs':
                self._configure_pbs_probe()
            elif probe == 'lsf':
                self._configure_lsf_probe()
            elif probe == 'sge':
                self._configure_sge_probe()
            elif probe == 'slurm':
                self._configure_slurm_probe()

        self.log("GratiaConfiguration.configure completed")
        return True
Beispiel #29
0
def exclude_blank(item_list):
    """Return a copy of a list with blank values removed"""
    return [item for item in item_list if not utilities.blank(item)]
Beispiel #30
0
    def parse_configuration(self, configuration):
        """
        Try to get configuration information from ConfigParser or SafeConfigParser
        object given by configuration and write recognized settings to attributes
        dict
        """

        self.log('GratiaConfiguration.parse_configuration started')

        self.check_config(configuration)

        if (not configuration.has_section(self.config_section)
                and requirements_are_installed()):
            self.log(
                'CE probes installed but no Gratia section, auto-configuring gratia'
            )
            self._auto_configure(configuration)
            self.log('GratiaConfiguration.parse_configuration completed')
            return True
        elif not configuration.has_section(self.config_section):
            self.enabled = False
            self.log("%s section not in config file" % self.config_section)
            self.log('Gratia.parse_configuration completed')
            return

        if not self.set_status(configuration):
            self.log('GratiaConfiguration.parse_configuration completed')
            return True

        # set the appropriate defaults if we're on a CE
        if requirements_are_installed():
            if configuration.has_option('Site Information', 'group'):
                self.grid_group = configuration.get('Site Information',
                                                    'group')

            if self.grid_group == 'OSG':
                self.options['probes'].default_value = \
                    self._production_defaults['probes']
            elif self.grid_group == 'OSG-ITB':
                self.options['probes'].default_value = \
                    self._itb_defaults['probes']

            # grab configuration information for various jobmanagers
            probes = self.get_installed_probes()
            for probe in probes:
                if probe == 'condor':
                    self._probe_config['condor'] = {
                        'condor_location':
                        CondorConfiguration.get_condor_location(configuration),
                        'condor_config':
                        CondorConfiguration.get_condor_config(configuration)
                    }
                elif probe == 'pbs':
                    if BaseConfiguration.section_disabled(
                            configuration, 'PBS'):
                        # if the PBS jobmanager is disabled, the CE is probably using LSF
                        # in any case, setting up the pbs gratia probe is not useful
                        continue
                    log_option = configfile.Option(
                        name='log_directory',
                        required=configfile.Option.OPTIONAL,
                        default_value='')
                    configfile.get_option(configuration, 'PBS', log_option)
                    self._probe_config['pbs'] = {
                        'log_directory': log_option.value
                    }

                    accounting_log_option = configfile.Option(
                        name='accounting_log_directory',
                        required=configfile.Option.OPTIONAL,
                        default_value='')
                    configfile.get_option(configuration, 'PBS',
                                          accounting_log_option)
                    self._probe_config['pbs'] = {
                        'accounting_log_directory': accounting_log_option.value
                    }
                elif probe == 'lsf':
                    if BaseConfiguration.section_disabled(
                            configuration, 'LSF'):
                        # if the LSF jobmanager is disabled, the CE is probably using PBS
                        # in any case, setting up the pbs gratia probe is not useful
                        continue
                    lsf_location = configfile.Option(name='lsf_location',
                                                     default_value='/usr/bin')
                    configfile.get_option(configuration, 'LSF', lsf_location)
                    self._probe_config['lsf'] = {
                        'lsf_location': lsf_location.value
                    }

                    log_option = configfile.Option(
                        name='log_directory',
                        required=configfile.Option.OPTIONAL,
                        default_value='')
                    configfile.get_option(configuration, 'LSF', log_option)
                    self._probe_config['lsf'][
                        'log_directory'] = log_option.value
                elif probe == 'sge':
                    if BaseConfiguration.section_disabled(
                            configuration, 'SGE'):
                        # if section is disabled then the following code won't work
                        # since the parse_configuration will short circuit, so
                        # give a warning and then move on
                        self.log(
                            "Skipping SGE gratia probe configuration since SGE is disabled",
                            level=logging.WARNING)
                        continue
                    sge_config = SGEConfiguration(logger=self.logger)
                    sge_config.parse_configuration(configuration)
                    self._probe_config['sge'] = {
                        'sge_accounting_file':
                        sge_config.get_accounting_file()
                    }
                elif probe == 'slurm':
                    if BaseConfiguration.section_disabled(
                            configuration, 'SLURM'):
                        # if section is disabled then the following code won't work
                        # since the parse_configuration will short circuit, so
                        # give a warning and then move on
                        self.log(
                            "Skipping Slurm gratia probe configuration since Slurm is disabled",
                            level=logging.WARNING)
                        continue
                    slurm_config = SlurmConfiguration(logger=self.logger)
                    slurm_config.parse_configuration(configuration)
                    self._probe_config['slurm'] = {
                        'db_host': slurm_config.get_db_host(),
                        'db_port': slurm_config.get_db_port(),
                        'db_user': slurm_config.get_db_user(),
                        'db_pass': slurm_config.get_db_pass(),
                        'db_name': slurm_config.get_db_name(),
                        'cluster': slurm_config.get_slurm_cluster(),
                        'location': slurm_config.get_location()
                    }
                elif probe == 'htcondor-ce':
                    self._probe_config['htcondor-ce'] = {}

        self.get_options(configuration,
                         ignore_options=[
                             'itb-jobmanager-gratia', 'itb-gridftp-gratia',
                             'osg-jobmanager-gratia', 'osg-gridftp-gratia',
                             'enabled'
                         ])

        if utilities.blank(self.options['probes'].value):
            self.log('GratiaConfiguration.parse_configuration completed')
            return

        self._parse_probes(self.options['probes'].value)
        self.log('GratiaConfiguration.parse_configuration completed')
Beispiel #31
0
    def configure(self, attributes):
        """Configure installation using attributes"""
        self.log("GratiaConfiguration.configure started")

        if self.ignored:
            self.log("%s configuration ignored" % self.config_section,
                     level=logging.WARNING)
            self.log("GratiaConfiguration.configure completed")
            return True

        # disable all gratia services
        # if gratia is enabled, probes will get enabled below
        if not self.enabled:
            self.log("Not enabled")
            self.log("GratiaConfiguration.configure completed")
            return True

        if utilities.blank(self.options['resource'].value):
            if 'OSG_SITE_NAME' not in attributes:
                self.log(
                    'No resource found for gratia reporting. You must give it '
                    'using the resource option in the Gratia section or specify '
                    'it in the Site Information section',
                    level=logging.ERROR)
                self.log("GratiaConfiguration.configure completed")
                return False
            else:
                self.options['resource'].value = attributes['OSG_SITE_NAME']

        if 'OSG_HOSTNAME' not in attributes:
            self.log(
                'Hostname of this machine not specified. Please give this '
                'in the host_name option in the Site Information section',
                level=logging.ERROR)
            self.log("GratiaConfiguration.configure completed")
            return False

        hostname = attributes['OSG_HOSTNAME']
        probe_list = self.get_installed_probes()
        for probe in probe_list:
            if probe in self._job_managers:
                if probe not in self._probe_config:
                    # Probe is installed but we don't have configuration for it
                    # might be due to pbs-lsf probe sharing or relevant job
                    # manager is not shared
                    continue

                if 'jobmanager' in self.enabled_probe_settings:
                    probe_host = self.enabled_probe_settings['jobmanager']
                else:
                    continue
            else:
                if probe in self.enabled_probe_settings:
                    probe_host = self.enabled_probe_settings[probe]
                else:
                    continue

            self._make_subscription(probe, probe_list[probe], probe_host,
                                    self.options['resource'].value, hostname)
            if probe == 'condor':
                self._configure_condor_probe()
            elif probe == 'pbs':
                self._configure_pbs_probe()
            elif probe == 'lsf':
                self._configure_lsf_probe()
            elif probe == 'sge':
                self._configure_sge_probe()
            elif probe == 'slurm':
                self._configure_slurm_probe()
            elif probe == 'htcondor-ce':
                self._configure_htcondor_ce_probe()

        self.log("GratiaConfiguration.configure completed")
        return True
Beispiel #32
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('SiteAttributes.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('SiteAttributes.check_attributes completed')
            return attributes_ok



        # OSG_GROUP must be either OSG or OSG-ITB
        if self.options['group'].value not in ('OSG', 'OSG-ITB'):
            self.log("The group setting must be either OSG or OSG-ITB, got: %s" %
                     self.options['group'].value,
                     option='group',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        # host_name must be different from the default setting
        if self.options['host_name'].value == 'my.domain.name':
            self.log("Setting left at default value: my.domain.name",
                     option='host_name',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        # host_name must be a valid dns name, check this by getting it's ip adddress
        if not validation.valid_domain(self.options['host_name'].value, True):
            self.log("hostname %s can't be resolved" % self.options['host_name'].value,
                     option='host_name',
                     section=self.config_section,
                     level=logging.ERROR)
            attributes_ok = False

        # site_name or resource/resource_group must be specified not both
        if (not utilities.blank(self.options['site_name'].value) and
                (not utilities.blank(self.options['resource'].value) or
                     not utilities.blank(self.options['resource_group'].value))):
            self.log("In section '%s', site_name and " % self.config_section +
                     "resource or resource_group given at the same time, " +
                     "you should use just the resource and resource_group settings.",
                     level=logging.WARNING)

        if self.options['latitude'].value > 90 or self.options['latitude'].value < -90:
            self.log("Latitude must be between -90 and 90, got %s" %
                     self.options['latitude'].value,
                     section=self.config_section,
                     option='latitude',
                     level=logging.ERROR)
            attributes_ok = False

        if self.options['longitude'].value > 180 or self.options['longitude'].value < -180:
            self.log("Longitude must be between -180 and 180, got %s" %
                     self.options['longitude'].value,
                     section=self.config_section,
                     option='longitude',
                     level=logging.ERROR)
            attributes_ok = False


        # make sure that the email address is different from the default value
        if self.options['email'] == '*****@*****.**':
            self.log("The email setting must be changed from the default",
                     section=self.config_section,
                     option='email',
                     level=logging.ERROR)
            attributes_ok = False

        # make sure the email address has the correct format and that the domain portion is
        # resolvable
        if not validation.valid_email(self.options['email'].value):
            self.log("Invalid email address in site information: %s" %
                     self.options['email'].value,
                     section=self.config_section,
                     option='email',
                     level=logging.ERROR)
            attributes_ok = False
        else:
            match = re.match(r'(?:[a-zA-Z\-_+0-9.]+)@([a-zA-Z0-9_\-]+(?:\.[a-zA-Z0-9_\-]+)+)',
                             self.options['email'].value)
            try:
                socket.gethostbyname(match.group(1))
            except socket.herror:
                self.log("Can't resolve domain in email: %s" % self.options['email'].value,
                         section=self.config_section,
                         option='email',
                         level=logging.WARNING,
                         exception=True)
            except socket.gaierror:
                self.log("Can't resolve domain in email: %s" % self.options['email'].value,
                         section=self.config_section,
                         option='email',
                         level=logging.WARNING,
                         exception=True)

        vo_list = self.options['sponsor'].value
        percentage = 0
        vo_names = utilities.get_vos(None)
        if vo_names == []:
            map_file_present = False
        else:
            map_file_present = True
        vo_names.append('usatlas')  # usatlas is a valid vo name
        vo_names.append('uscms')  # uscms is a valid vo name
        vo_names.append('local')  # local is a valid vo name

        cap_vo_names = [vo.upper() for vo in vo_names]
        for vo in re.split(r'\s*,?\s*', vo_list):
            vo_name = vo.split(':')[0]
            if vo_name not in vo_names:
                if vo_name.upper() in cap_vo_names:
                    self.log("VO name %s has the wrong capitialization" % vo_name,
                             section=self.config_section,
                             option='sponsor',
                             level=logging.WARNING)
                    vo_mesg = "Valid VO names are as follows:\n"
                    for name in vo_names:
                        vo_mesg += name + "\n"
                    self.log(vo_mesg, level=logging.WARNING)
                else:
                    if map_file_present:
                        self.log("In %s section, problem with sponsor setting" % \
                                 self.config_section)
                        self.log("VO name %s not found" % vo_name,
                                 section=self.config_section,
                                 option='sponsor',
                                 level=logging.ERROR)
                        vo_mesg = "Valid VO names are as follows:\n"
                        for name in vo_names:
                            vo_mesg += name + "\n"
                        self.log(vo_mesg, level=logging.ERROR)
                        attributes_ok = False
                    else:
                        self.log("Can't currently check VOs in sponsor setting because " +
                                 "the /var/lib/osg/user-vo-map is empty. If you are " +
                                 "configuring osg components, this may be resolved when " +
                                 "osg-configure runs the appropriate script to generate " +
                                 "this file later in the configuration process",
                                 section=self.config_section,
                                 option='sponsor',
                                 level=logging.WARNING)

            if len(vo.split(':')) == 1:
                percentage += 100
            elif len(vo.split(':')) == 2:
                vo_percentage = vo.split(':')[1]
                try:
                    percentage += int(vo_percentage)
                except ValueError:
                    self.log("VO percentage (%s) in sponsor field (%s) not an integer" \
                             % (vo_percentage, vo),
                             section=self.config_section,
                             option='sponsor',
                             level=logging.ERROR,
                             exception=True)
                    attributes_ok = False
            else:
                self.log("VO sponsor field is not formated correctly: %s" % vo,
                         section=self.config_section,
                         option='sponsor',
                         level=logging.ERROR)
                self.log("Sponsors should be given as sponsor:percentage "
                         "separated by a space or comma")

        if percentage != 100:
            self.log("VO percentages in sponsor field do not add up to 100, got %s" \
                     % percentage,
                     section=self.config_section,
                     option='sponsor',
                     level=logging.ERROR)
            attributes_ok = False
        self.log('SiteAttributes.check_attributes completed')
        return attributes_ok
Beispiel #33
0
    def _check_auth_settings(self):
        """ Check authorization/certificate settings and make sure that they are valid """

        check_value = True

        # Do not allow both the service cert settings and proxy settings
        # first create some helper variables
        blank_service_vals = (
            utilities.blank(self.options['service_cert'].value)
            and utilities.blank(self.options['service_key'].value)
            and utilities.blank(self.options['service_proxy'].value))

        default_service_vals = (self.options['service_cert'].value ==
                                self.options['service_cert'].default_value)
        default_service_vals &= (self.options['service_key'].value ==
                                 self.options['service_key'].default_value)
        default_service_vals &= (self.options['service_proxy'].value ==
                                 self.options['service_proxy'].default_value)

        blank_user_proxy = utilities.blank(self.options['user_proxy'].value)

        if (not blank_user_proxy and default_service_vals):
            self.log('User proxy specified and service_cert, service_key, ' +
                     'service_proxy at default values, assuming user_proxy ' +
                     'takes precedence in ' + self.config_section + ' section')
            self.use_service_cert = False
        elif not blank_user_proxy and not blank_service_vals:
            self.log(
                "You cannot specify user_proxy with any of (service_cert, " +
                "service_key, service_proxy).  They are mutually exclusive " +
                "options in %s section." % self.config_section,
                level=logging.ERROR)
            check_value = False

        # Make sure that either a service cert or user cert is selected
        if not ((self.options['service_cert'].value
                 and self.options['service_key'].value
                 and self.options['service_proxy'].value)
                or self.options['user_proxy'].value):
            self.log("You must specify either service_cert/service_key/" +
                     "service_proxy *or* user_proxy in order to provide " +
                     "credentials for RSV to run jobs in " +
                     " %s section" % self.config_section,
                     level=logging.ERROR)
            check_value = False

        if not blank_user_proxy:
            # if not using a service certificate, make sure that the proxy file exists
            value = self.options['user_proxy'].value
            if utilities.blank(value) or not validation.valid_file(value):
                self.log("user_proxy does not point to an existing file: %s" %
                         value,
                         section=self.config_section,
                         option='user_proxy',
                         level=logging.ERROR)
                check_value = False
        else:
            for optname in 'service_cert', 'service_key':
                value = self.options[optname].value
                if utilities.blank(value):
                    self.log("%s must have a valid location" % optname,
                             section=self.config_section,
                             option=optname,
                             level=logging.ERROR)
                    check_value = False
                elif not self.copy_host_cert_for_service_cert and not validation.valid_file(
                        value):
                    self.log("%s must point to an existing file" % optname,
                             section=self.config_section,
                             option=optname,
                             level=logging.ERROR)
                    check_value = False

            value = self.options['service_proxy'].value
            if utilities.blank(value):
                self.log("service_proxy must have a valid location: %s" %
                         value,
                         section=self.config_section,
                         option='service_proxy',
                         level=logging.ERROR)
                check_value = False

            value = os.path.dirname(self.options['service_proxy'].value)
            if not validation.valid_location(value):
                self.log("service_proxy must be located in a valid " +
                         "directory: %s" % value,
                         section=self.config_section,
                         option='service_proxy',
                         level=logging.ERROR)
                check_value = False

        return check_value
Beispiel #34
0
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log('SiteInformation.check_attributes started')
        attributes_ok = True

        if not self.enabled:
            self.log('Not enabled, returning True')
            self.log('SiteInformation.check_attributes completed')
            return attributes_ok

        # OSG_GROUP must be either OSG or OSG-ITB
        group = self.opt_val("group")
        if group not in ('OSG', 'OSG-ITB'):
            self.log(
                "The group setting must be either OSG or OSG-ITB, got: %s" %
                group,
                option='group',
                section=self.config_section,
                level=logging.ERROR)
            attributes_ok = False

        host_name = self.opt_val("host_name")
        if not utilities.blank(host_name):
            if not validation.valid_domain(host_name, resolve=False):
                self.log("%s is not a valid domain",
                         host_name,
                         option="host_name",
                         section=self.config_section,
                         level=logging.ERROR)
                attributes_ok = False
            elif not validation.valid_domain(host_name, resolve=True):
                self.log("%s is a valid domain but can't be resolved",
                         host_name,
                         option="host_name",
                         section=self.config_section,
                         level=logging.WARNING)

        latitude = self.opt_val("latitude")
        if not utilities.blank(latitude) and not -90 <= latitude <= 90:
            self.log("Latitude must be between -90 and 90, got %s" % latitude,
                     section=self.config_section,
                     option='latitude',
                     level=logging.ERROR)
            attributes_ok = False

        longitude = self.opt_val("longitude")
        if not utilities.blank(longitude) and not -180 <= longitude <= 180:
            self.log("Longitude must be between -180 and 180, got %s" %
                     longitude,
                     section=self.config_section,
                     option='longitude',
                     level=logging.ERROR)
            attributes_ok = False

        email = self.opt_val("email")
        # make sure the email address has the correct format
        if not utilities.blank(email) and not validation.valid_email(email):
            self.log("Invalid email address in site information: %s" % email,
                     section=self.config_section,
                     option='email',
                     level=logging.ERROR)
            attributes_ok = False

        sponsor = self.opt_val("sponsor")
        if not utilities.blank(sponsor):
            attributes_ok &= self.check_sponsor(sponsor)

        self.log('SiteInformation.check_attributes completed')
        return attributes_ok
    def check_attributes(self, attributes):
        """Check attributes currently stored and make sure that they are consistent"""
        self.log("SiteAttributes.check_attributes started")
        attributes_ok = True

        if not self.enabled:
            self.log("Not enabled, returning True")
            self.log("SiteAttributes.check_attributes completed")
            return attributes_ok

        # OSG_GROUP must be either OSG or OSG-ITB
        if self.options["group"].value not in ("OSG", "OSG-ITB"):
            self.log(
                "The group setting must be either OSG or OSG-ITB, got: %s" % self.options["group"].value,
                option="group",
                section=self.config_section,
                level=logging.ERROR,
            )
            attributes_ok = False

        # host_name must be different from the default setting
        if self.options["host_name"].value == "my.domain.name":
            self.log(
                "Setting left at default value: my.domain.name",
                option="host_name",
                section=self.config_section,
                level=logging.ERROR,
            )
            attributes_ok = False

        # host_name must be a valid dns name, check this by getting it's ip adddress
        if not validation.valid_domain(self.options["host_name"].value, True):
            self.log(
                "hostname %s can't be resolved" % self.options["host_name"].value,
                option="host_name",
                section=self.config_section,
                level=logging.ERROR,
            )
            attributes_ok = False

        # site_name or resource/resource_group must be specified not both
        if not utilities.blank(self.options["site_name"].value) and (
            not utilities.blank(self.options["resource"].value)
            or not utilities.blank(self.options["resource_group"].value)
        ):
            self.log(
                "In section '%s', site_name and " % self.config_section
                + "resource or resource_group given at the same time, "
                + "you should use just the resource and resource_group settings.",
                level=logging.WARNING,
            )

        if self.options["latitude"].value > 90 or self.options["latitude"].value < -90:
            self.log(
                "Latitude must be between -90 and 90, got %s" % self.options["latitude"].value,
                section=self.config_section,
                option="latitude",
                level=logging.ERROR,
            )
            attributes_ok = False

        if self.options["longitude"].value > 180 or self.options["longitude"].value < -180:
            self.log(
                "Longitude must be between -180 and 180, got %s" % self.options["longitude"].value,
                section=self.config_section,
                option="longitude",
                level=logging.ERROR,
            )
            attributes_ok = False

        # make sure that the email address is different from the default value
        if self.options["email"] == "*****@*****.**":
            self.log(
                "The email setting must be changed from the default",
                section=self.config_section,
                option="email",
                level=logging.ERROR,
            )
            attributes_ok = False

        # make sure the email address has the correct format
        if not validation.valid_email(self.options["email"].value):
            self.log(
                "Invalid email address in site information: %s" % self.options["email"].value,
                section=self.config_section,
                option="email",
                level=logging.ERROR,
            )
            attributes_ok = False

        vo_list = self.options["sponsor"].value
        percentage = 0
        vo_names = utilities.get_vos(None)
        if vo_names == []:
            map_file_present = False
        else:
            map_file_present = True
        vo_names.append("usatlas")  # usatlas is a valid vo name
        vo_names.append("uscms")  # uscms is a valid vo name
        vo_names.append("local")  # local is a valid vo name

        cap_vo_names = [vo.upper() for vo in vo_names]
        for vo in re.split(r"\s*,?\s*", vo_list):
            vo_name = vo.split(":")[0]
            if vo_name not in vo_names:
                if vo_name.upper() in cap_vo_names:
                    self.log(
                        "VO name %s has the wrong capitialization" % vo_name,
                        section=self.config_section,
                        option="sponsor",
                        level=logging.WARNING,
                    )
                    vo_mesg = "Valid VO names are as follows:\n"
                    for name in vo_names:
                        vo_mesg += name + "\n"
                    self.log(vo_mesg, level=logging.WARNING)
                else:
                    if map_file_present:
                        self.log("In %s section, problem with sponsor setting" % self.config_section)
                        self.log(
                            "VO name %s not found" % vo_name,
                            section=self.config_section,
                            option="sponsor",
                            level=logging.ERROR,
                        )
                        vo_mesg = "Valid VO names are as follows:\n"
                        for name in vo_names:
                            vo_mesg += name + "\n"
                        self.log(vo_mesg, level=logging.ERROR)
                        attributes_ok = False
                    else:
                        self.log(
                            "Can't currently check VOs in sponsor setting because "
                            + "the /var/lib/osg/user-vo-map is empty. If you are "
                            + "configuring osg components, this may be resolved when "
                            + "osg-configure runs the appropriate script to generate "
                            + "this file later in the configuration process",
                            section=self.config_section,
                            option="sponsor",
                            level=logging.WARNING,
                        )

            if len(vo.split(":")) == 1:
                percentage += 100
            elif len(vo.split(":")) == 2:
                vo_percentage = vo.split(":")[1]
                try:
                    percentage += int(vo_percentage)
                except ValueError:
                    self.log(
                        "VO percentage (%s) in sponsor field (%s) not an integer" % (vo_percentage, vo),
                        section=self.config_section,
                        option="sponsor",
                        level=logging.ERROR,
                        exception=True,
                    )
                    attributes_ok = False
            else:
                self.log(
                    "VO sponsor field is not formated correctly: %s" % vo,
                    section=self.config_section,
                    option="sponsor",
                    level=logging.ERROR,
                )
                self.log("Sponsors should be given as sponsor:percentage " "separated by a space or comma")

        if percentage != 100:
            self.log(
                "VO percentages in sponsor field do not add up to 100, got %s" % percentage,
                section=self.config_section,
                option="sponsor",
                level=logging.ERROR,
            )
            attributes_ok = False
        self.log("SiteAttributes.check_attributes completed")
        return attributes_ok